diff options
Diffstat (limited to '')
39 files changed, 9448 insertions, 0 deletions
diff --git a/src/yajl/CMakeLists.txt b/src/yajl/CMakeLists.txt new file mode 100644 index 0000000..82fe532 --- /dev/null +++ b/src/yajl/CMakeLists.txt @@ -0,0 +1,27 @@ +add_library(yajl STATIC + api/yajl_common.h + api/yajl_gen.h + api/yajl_parse.h + api/yajl_tree.h + + yajl_common.h + yajl_alloc.h + yajl_buf.h + yajl_bytestack.h + yajl_encode.h + yajl_lex.h + yajl_parser.h + yajl_version.h + + yajl.c + yajl_alloc.c + yajl_buf.c + yajl_encode.c + yajl_gen.c + yajl_lex.c + yajl_parser.c + yajl_tree.c + yajl_version.c + ) +target_include_directories(yajl PUBLIC api) +target_include_directories(yajl PRIVATE ..) diff --git a/src/yajl/Makefile.am b/src/yajl/Makefile.am new file mode 100644 index 0000000..629185e --- /dev/null +++ b/src/yajl/Makefile.am @@ -0,0 +1,37 @@ + +AM_CPPFLAGS = -I$(top_srcdir)/src + +noinst_LIBRARIES = libyajl.a + +noinst_HEADERS = \ + api/yajl_common.h \ + api/yajl_gen.h \ + api/yajl_parse.h \ + api/yajl_tree.h \ + yajl_alloc.h \ + yajl_buf.h \ + yajl_bytestack.h \ + yajl_common.h \ + yajl_encode.h \ + yajl_lex.h \ + yajl_parser.h \ + yajl_version.h + +if USE_INCLUDED_YAJL +libyajl_a_SOURCES = \ + yajl.c \ + yajl_alloc.c \ + yajl_alloc.h \ + yajl_buf.c \ + yajl_buf.h \ + yajl_bytestack.h \ + yajl_encode.c \ + yajl_encode.h \ + yajl_gen.c \ + yajl_lex.c \ + yajl_lex.h \ + yajl_parser.c \ + yajl_parser.h \ + yajl_tree.c \ + yajl_version.c +endif diff --git a/src/yajl/api/yajl_common.h b/src/yajl/api/yajl_common.h new file mode 100644 index 0000000..9596ef9 --- /dev/null +++ b/src/yajl/api/yajl_common.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __YAJL_COMMON_H__ +#define __YAJL_COMMON_H__ + +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define YAJL_MAX_DEPTH 128 + +/* msft dll export gunk. To build a DLL on windows, you + * must define WIN32, YAJL_SHARED, and YAJL_BUILD. To use a shared + * DLL, you must define YAJL_SHARED and WIN32 */ +#if (defined(_WIN32) || defined(WIN32)) && defined(YAJL_SHARED) +# ifdef YAJL_BUILD +# define YAJL_API __declspec(dllexport) +# else +# define YAJL_API __declspec(dllimport) +# endif +#else +# if defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__) >= 303 +# define YAJL_API __attribute__ ((visibility("default"))) +# else +# define YAJL_API +# endif +#endif + +/** pointer to a malloc function, supporting client overriding memory + * allocation routines */ +typedef void * (*yajl_malloc_func)(void *ctx, size_t sz); + +/** pointer to a free function, supporting client overriding memory + * allocation routines */ +typedef void (*yajl_free_func)(void *ctx, void * ptr); + +/** pointer to a realloc function which can resize an allocation. */ +typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, size_t sz); + +/** A structure which can be passed to yajl_*_alloc routines to allow the + * client to specify memory allocation functions to be used. */ +typedef struct +{ + /** pointer to a function that can allocate uninitialized memory */ + yajl_malloc_func malloc; + /** pointer to a function that can resize memory allocations */ + yajl_realloc_func realloc; + /** pointer to a function that can free memory allocated using + * reallocFunction or mallocFunction */ + yajl_free_func free; + /** a context pointer that will be passed to above allocation routines */ + void * ctx; +} yajl_alloc_funcs; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/yajl/api/yajl_gen.h b/src/yajl/api/yajl_gen.h new file mode 100644 index 0000000..ddd1527 --- /dev/null +++ b/src/yajl/api/yajl_gen.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/** + * \file yajl_gen.h + * Interface to YAJL's JSON generation facilities. + */ + +#include <yajl/yajl_common.h> + +#ifndef __YAJL_GEN_H__ +#define __YAJL_GEN_H__ + +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + /** generator status codes */ + typedef enum { + /** no error */ + yajl_gen_status_ok = 0, + /** at a point where a map key is generated, a function other than + * yajl_gen_string was called */ + yajl_gen_keys_must_be_strings, + /** YAJL's maximum generation depth was exceeded. see + * YAJL_MAX_DEPTH */ + yajl_max_depth_exceeded, + /** A generator function (yajl_gen_XXX) was called while in an error + * state */ + yajl_gen_in_error_state, + /** A complete JSON document has been generated */ + yajl_gen_generation_complete, + /** yajl_gen_double was passed an invalid floating point value + * (infinity or NaN). */ + yajl_gen_invalid_number, + /** A print callback was passed in, so there is no internal + * buffer to get from */ + yajl_gen_no_buf, + /** returned from yajl_gen_string() when the yajl_gen_validate_utf8 + * option is enabled and an invalid was passed by client code. + */ + yajl_gen_invalid_string + } yajl_gen_status; + + /** an opaque handle to a generator */ + typedef struct yajl_gen_t * yajl_gen; + + /** a callback used for "printing" the results. */ + typedef void (*yajl_print_t)(void * ctx, + const char * str, + size_t len); + + /** configuration parameters for the parser, these may be passed to + * yajl_gen_config() along with option specific argument(s). In general, + * all configuration parameters default to *off*. */ + typedef enum { + /** generate indented (beautiful) output */ + yajl_gen_beautify = 0x01, + /** + * Set an indent string which is used when yajl_gen_beautify + * is enabled. Maybe something like \\t or some number of + * spaces. The default is four spaces ' '. + */ + yajl_gen_indent_string = 0x02, + /** + * Set a function and context argument that should be used to + * output generated json. the function should conform to the + * yajl_print_t prototype while the context argument is a + * void * of your choosing. + * + * example: + * yajl_gen_config(g, yajl_gen_print_callback, myFunc, myVoidPtr); + */ + yajl_gen_print_callback = 0x04, + /** + * Normally the generator does not validate that strings you + * pass to it via yajl_gen_string() are valid UTF8. Enabling + * this option will cause it to do so. + */ + yajl_gen_validate_utf8 = 0x08, + /** + * the forward solidus (slash or '/' in human) is not required to be + * escaped in json text. By default, YAJL will not escape it in the + * iterest of saving bytes. Setting this flag will cause YAJL to + * always escape '/' in generated JSON strings. + */ + yajl_gen_escape_solidus = 0x10 + } yajl_gen_option; + + /** allow the modification of generator options subsequent to handle + * allocation (via yajl_alloc) + * \returns zero in case of errors, non-zero otherwise + */ + YAJL_API int yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...); + + /** allocate a generator handle + * \param allocFuncs an optional pointer to a structure which allows + * the client to overide the memory allocation + * used by yajl. May be NULL, in which case + * malloc/free/realloc will be used. + * + * \returns an allocated handle on success, NULL on failure (bad params) + */ + YAJL_API yajl_gen yajl_gen_alloc(const yajl_alloc_funcs * allocFuncs); + + /** free a generator handle */ + YAJL_API void yajl_gen_free(yajl_gen handle); + + YAJL_API yajl_gen_status yajl_gen_integer(yajl_gen hand, long long int number); + /** generate a floating point number. number may not be infinity or + * NaN, as these have no representation in JSON. In these cases the + * generator will return 'yajl_gen_invalid_number' */ + YAJL_API yajl_gen_status yajl_gen_double(yajl_gen hand, double number); + YAJL_API yajl_gen_status yajl_gen_number(yajl_gen hand, + const char * num, + size_t len); + YAJL_API yajl_gen_status yajl_gen_string(yajl_gen hand, + const unsigned char * str, + size_t len); + YAJL_API yajl_gen_status yajl_gen_null(yajl_gen hand); + YAJL_API yajl_gen_status yajl_gen_bool(yajl_gen hand, int boolean); + YAJL_API yajl_gen_status yajl_gen_map_open(yajl_gen hand); + YAJL_API yajl_gen_status yajl_gen_map_close(yajl_gen hand); + YAJL_API yajl_gen_status yajl_gen_array_open(yajl_gen hand); + YAJL_API yajl_gen_status yajl_gen_array_close(yajl_gen hand); + + /** access the null terminated generator buffer. If incrementally + * outputing JSON, one should call yajl_gen_clear to clear the + * buffer. This allows stream generation. */ + YAJL_API yajl_gen_status yajl_gen_get_buf(yajl_gen hand, + const unsigned char ** buf, + size_t * len); + + /** clear yajl's output buffer, but maintain all internal generation + * state. This function will not "reset" the generator state, and is + * intended to enable incremental JSON outputing. */ + YAJL_API void yajl_gen_clear(yajl_gen hand); + + /** Reset the generator state. Allows a client to generate multiple + * json entities in a stream. The "sep" string will be inserted to + * separate the previously generated entity from the current, + * NULL means *no separation* of entites (clients beware, generating + * multiple JSON numbers, for instance, will result in inscrutable + * output) */ + YAJL_API void yajl_gen_reset(yajl_gen hand, const char * sep); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/yajl/api/yajl_parse.h b/src/yajl/api/yajl_parse.h new file mode 100644 index 0000000..eea8d9a --- /dev/null +++ b/src/yajl/api/yajl_parse.h @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/** + * \file yajl_parse.h + * Interface to YAJL's JSON stream parsing facilities. + */ + +#include <yajl/yajl_common.h> + +#ifndef __YAJL_PARSE_H__ +#define __YAJL_PARSE_H__ + +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + /** error codes returned from this interface */ + typedef enum { + /** no error was encountered */ + yajl_status_ok, + /** a client callback returned zero, stopping the parse */ + yajl_status_client_canceled, + /** An error occured during the parse. Call yajl_get_error for + * more information about the encountered error */ + yajl_status_error + } yajl_status; + + /** attain a human readable, english, string for an error */ + YAJL_API const char * yajl_status_to_string(yajl_status code); + + /** an opaque handle to a parser */ + typedef struct yajl_handle_t * yajl_handle; + + /** yajl is an event driven parser. this means as json elements are + * parsed, you are called back to do something with the data. The + * functions in this table indicate the various events for which + * you will be called back. Each callback accepts a "context" + * pointer, this is a void * that is passed into the yajl_parse + * function which the client code may use to pass around context. + * + * All callbacks return an integer. If non-zero, the parse will + * continue. If zero, the parse will be canceled and + * yajl_status_client_canceled will be returned from the parse. + * + * \attention { + * A note about the handling of numbers: + * + * yajl will only convert numbers that can be represented in a + * double or a 64 bit (long long) int. All other numbers will + * be passed to the client in string form using the yajl_number + * callback. Furthermore, if yajl_number is not NULL, it will + * always be used to return numbers, that is yajl_integer and + * yajl_double will be ignored. If yajl_number is NULL but one + * of yajl_integer or yajl_double are defined, parsing of a + * number larger than is representable in a double or 64 bit + * integer will result in a parse error. + * } + */ + typedef struct { + int (* yajl_null)(void * ctx); + int (* yajl_boolean)(void * ctx, int boolVal); + int (* yajl_integer)(void * ctx, long long integerVal); + int (* yajl_double)(void * ctx, double doubleVal); + /** A callback which passes the string representation of the number + * back to the client. Will be used for all numbers when present */ + int (* yajl_number)(void * ctx, const char * numberVal, + size_t numberLen); + + /** strings are returned as pointers into the JSON text when, + * possible, as a result, they are _not_ null padded */ + int (* yajl_string)(void * ctx, const unsigned char * stringVal, + size_t stringLen); + + int (* yajl_start_map)(void * ctx); + int (* yajl_map_key)(void * ctx, const unsigned char * key, + size_t stringLen); + int (* yajl_end_map)(void * ctx); + + int (* yajl_start_array)(void * ctx); + int (* yajl_end_array)(void * ctx); + } yajl_callbacks; + + /** allocate a parser handle + * \param callbacks a yajl callbacks structure specifying the + * functions to call when different JSON entities + * are encountered in the input text. May be NULL, + * which is only useful for validation. + * \param afs memory allocation functions, may be NULL for to use + * C runtime library routines (malloc and friends) + * \param ctx a context pointer that will be passed to callbacks. + */ + YAJL_API yajl_handle yajl_alloc(const yajl_callbacks * callbacks, + yajl_alloc_funcs * afs, + void * ctx); + + + /** configuration parameters for the parser, these may be passed to + * yajl_config() along with option specific argument(s). In general, + * all configuration parameters default to *off*. */ + typedef enum { + /** Ignore javascript style comments present in + * JSON input. Non-standard, but rather fun + * arguments: toggled off with integer zero, on otherwise. + * + * example: + * yajl_config(h, yajl_allow_comments, 1); // turn comment support on + */ + yajl_allow_comments = 0x01, + /** + * When set the parser will verify that all strings in JSON input are + * valid UTF8 and will emit a parse error if this is not so. When set, + * this option makes parsing slightly more expensive (~7% depending + * on processor and compiler in use) + * + * example: + * yajl_config(h, yajl_dont_validate_strings, 1); // disable utf8 checking + */ + yajl_dont_validate_strings = 0x02, + /** + * By default, upon calls to yajl_complete_parse(), yajl will + * ensure the entire input text was consumed and will raise an error + * otherwise. Enabling this flag will cause yajl to disable this + * check. This can be useful when parsing json out of a that contains more + * than a single JSON document. + */ + yajl_allow_trailing_garbage = 0x04, + /** + * Allow multiple values to be parsed by a single handle. The + * entire text must be valid JSON, and values can be seperated + * by any kind of whitespace. This flag will change the + * behavior of the parser, and cause it continue parsing after + * a value is parsed, rather than transitioning into a + * complete state. This option can be useful when parsing multiple + * values from an input stream. + */ + yajl_allow_multiple_values = 0x08, + /** + * When yajl_complete_parse() is called the parser will + * check that the top level value was completely consumed. I.E., + * if called whilst in the middle of parsing a value + * yajl will enter an error state (premature EOF). Setting this + * flag suppresses that check and the corresponding error. + */ + yajl_allow_partial_values = 0x10 + } yajl_option; + + /** allow the modification of parser options subsequent to handle + * allocation (via yajl_alloc) + * \returns zero in case of errors, non-zero otherwise + */ + YAJL_API int yajl_config(yajl_handle h, yajl_option opt, ...); + + YAJL_API void yajl_reset(yajl_handle handle); + + /** free a parser handle */ + YAJL_API void yajl_free(yajl_handle handle); + + /** Parse some json! + * \param hand - a handle to the json parser allocated with yajl_alloc + * \param jsonText - a pointer to the UTF8 json text to be parsed + * \param jsonTextLength - the length, in bytes, of input text + */ + YAJL_API yajl_status yajl_parse(yajl_handle hand, + const unsigned char * jsonText, + size_t jsonTextLength); + + /** Parse any remaining buffered json. + * Since yajl is a stream-based parser, without an explicit end of + * input, yajl sometimes can't decide if content at the end of the + * stream is valid or not. For example, if "1" has been fed in, + * yajl can't know whether another digit is next or some character + * that would terminate the integer token. + * + * \param hand - a handle to the json parser allocated with yajl_alloc + */ + YAJL_API yajl_status yajl_complete_parse(yajl_handle hand); + + /** get an error string describing the state of the + * parse. + * + * If verbose is non-zero, the message will include the JSON + * text where the error occured, along with an arrow pointing to + * the specific char. + * + * \returns A dynamically allocated string will be returned which should + * be freed with yajl_free_error + */ + YAJL_API unsigned char * yajl_get_error(yajl_handle hand, int verbose, + const unsigned char * jsonText, + size_t jsonTextLength); + + /** + * get the amount of data consumed from the last chunk passed to YAJL. + * + * In the case of a successful parse this can help you understand if + * the entire buffer was consumed (which will allow you to handle + * "junk at end of input"). + * + * In the event an error is encountered during parsing, this function + * affords the client a way to get the offset into the most recent + * chunk where the error occured. 0 will be returned if no error + * was encountered. + */ + YAJL_API size_t yajl_get_bytes_consumed(yajl_handle hand); + + /** free an error returned from yajl_get_error */ + YAJL_API void yajl_free_error(yajl_handle hand, unsigned char * str); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/yajl/api/yajl_tree.h b/src/yajl/api/yajl_tree.h new file mode 100644 index 0000000..1c1e06a --- /dev/null +++ b/src/yajl/api/yajl_tree.h @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2010-2011 Florian Forster <ff at octo.it> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/** + * \file yajl_tree.h + * + * Parses JSON data and returns the data in tree form. + * + * \author Florian Forster + * \date August 2010 + * + * This interface makes quick parsing and extraction of + * smallish JSON docs trivial: + * + * \include example/parse_config.c + */ + +#ifndef YAJL_TREE_H +#define YAJL_TREE_H 1 + +#include <yajl/yajl_common.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** possible data types that a yajl_val_s can hold */ +typedef enum { + yajl_t_string = 1, + yajl_t_number = 2, + yajl_t_object = 3, + yajl_t_array = 4, + yajl_t_true = 5, + yajl_t_false = 6, + yajl_t_null = 7, + /** The any type isn't valid for yajl_val_s.type, but can be + * used as an argument to routines like yajl_tree_get(). + */ + yajl_t_any = 8 +} yajl_type; + +#define YAJL_NUMBER_INT_VALID 0x01 +#define YAJL_NUMBER_DOUBLE_VALID 0x02 + +/** A pointer to a node in the parse tree */ +typedef struct yajl_val_s * yajl_val; + +/** + * A JSON value representation capable of holding one of the seven + * types above. For "string", "number", "object", and "array" + * additional data is available in the union. The "YAJL_IS_*" + * and "YAJL_GET_*" macros below allow type checking and convenient + * value extraction. + */ +struct yajl_val_s +{ + /** Type of the value contained. Use the "YAJL_IS_*" macros to check for a + * specific type. */ + yajl_type type; + /** Type-specific data. You may use the "YAJL_GET_*" macros to access these + * members. */ + union + { + char * string; + struct { + long long i; /*< integer value, if representable. */ + double d; /*< double value, if representable. */ + char *r; /*< unparsed number in string form. */ + /** Signals whether the \em i and \em d members are + * valid. See \c YAJL_NUMBER_INT_VALID and + * \c YAJL_NUMBER_DOUBLE_VALID. */ + unsigned int flags; + } number; + struct { + const char **keys; /*< Array of keys */ + yajl_val *values; /*< Array of values. */ + size_t len; /*< Number of key-value-pairs. */ + } object; + struct { + yajl_val *values; /*< Array of elements. */ + size_t len; /*< Number of elements. */ + } array; + } u; +}; + +/** + * Parse a string. + * + * Parses an null-terminated string containing JSON data and returns a pointer + * to the top-level value (root of the parse tree). + * + * \param input Pointer to a null-terminated utf8 string containing + * JSON data. + * \param error_buffer Pointer to a buffer in which an error message will + * be stored if \em yajl_tree_parse fails, or + * \c NULL. The buffer will be initialized before + * parsing, so its content will be destroyed even if + * \em yajl_tree_parse succeeds. + * \param error_buffer_size Size of the memory area pointed to by + * \em error_buffer_size. If \em error_buffer_size is + * \c NULL, this argument is ignored. + * + * \returns Pointer to the top-level value or \c NULL on error. The memory + * pointed to must be freed using \em yajl_tree_free. In case of an error, a + * null terminated message describing the error in more detail is stored in + * \em error_buffer if it is not \c NULL. + */ +YAJL_API yajl_val yajl_tree_parse (const char *input, + char *error_buffer, size_t error_buffer_size); + + +/** + * Free a parse tree returned by "yajl_tree_parse". + * + * \param v Pointer to a JSON value returned by "yajl_tree_parse". Passing NULL + * is valid and results in a no-op. + */ +YAJL_API void yajl_tree_free (yajl_val v); + +/** + * Access a nested value inside a tree. + * + * \param parent the node under which you'd like to extract values. + * \param path A null terminated array of strings, each the name of an object key + * \param type the yajl_type of the object you seek, or yajl_t_any if any will do. + * + * \returns a pointer to the found value, or NULL if we came up empty. + * + * Future Ideas: it'd be nice to move path to a string and implement support for + * a teeny tiny micro language here, so you can extract array elements, do things + * like .first and .last, even .length. Inspiration from JSONPath and css selectors? + * No it wouldn't be fast, but that's not what this API is about. + */ +YAJL_API yajl_val yajl_tree_get(yajl_val parent, const char ** path, yajl_type type); + +/* Various convenience macros to check the type of a `yajl_val` */ +#define YAJL_IS_STRING(v) (((v) != NULL) && ((v)->type == yajl_t_string)) +#define YAJL_IS_NUMBER(v) (((v) != NULL) && ((v)->type == yajl_t_number)) +#define YAJL_IS_INTEGER(v) (YAJL_IS_NUMBER(v) && ((v)->u.number.flags & YAJL_NUMBER_INT_VALID)) +#define YAJL_IS_DOUBLE(v) (YAJL_IS_NUMBER(v) && ((v)->u.number.flags & YAJL_NUMBER_DOUBLE_VALID)) +#define YAJL_IS_OBJECT(v) (((v) != NULL) && ((v)->type == yajl_t_object)) +#define YAJL_IS_ARRAY(v) (((v) != NULL) && ((v)->type == yajl_t_array )) +#define YAJL_IS_TRUE(v) (((v) != NULL) && ((v)->type == yajl_t_true )) +#define YAJL_IS_FALSE(v) (((v) != NULL) && ((v)->type == yajl_t_false )) +#define YAJL_IS_NULL(v) (((v) != NULL) && ((v)->type == yajl_t_null )) + +/** Given a yajl_val_string return a ptr to the bare string it contains, + * or NULL if the value is not a string. */ +#define YAJL_GET_STRING(v) (YAJL_IS_STRING(v) ? (v)->u.string : NULL) + +/** Get the string representation of a number. You should check type first, + * perhaps using YAJL_IS_NUMBER */ +#define YAJL_GET_NUMBER(v) ((v)->u.number.r) + +/** Get the double representation of a number. You should check type first, + * perhaps using YAJL_IS_DOUBLE */ +#define YAJL_GET_DOUBLE(v) ((v)->u.number.d) + +/** Get the 64bit (long long) integer representation of a number. You should + * check type first, perhaps using YAJL_IS_INTEGER */ +#define YAJL_GET_INTEGER(v) ((v)->u.number.i) + +/** Get a pointer to a yajl_val_object or NULL if the value is not an object. */ +#define YAJL_GET_OBJECT(v) (YAJL_IS_OBJECT(v) ? &(v)->u.object : NULL) + +/** Get a pointer to a yajl_val_array or NULL if the value is not an object. */ +#define YAJL_GET_ARRAY(v) (YAJL_IS_ARRAY(v) ? &(v)->u.array : NULL) + +#ifdef __cplusplus +} +#endif + +#endif /* YAJL_TREE_H */ diff --git a/src/yajl/yajl.c b/src/yajl/yajl.c new file mode 100644 index 0000000..346ab48 --- /dev/null +++ b/src/yajl/yajl.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "api/yajl_parse.h" +#include "yajl_lex.h" +#include "yajl_parser.h" +#include "yajl_alloc.h" + +#include <stdlib.h> +#include <string.h> +#include <stdarg.h> +#include <assert.h> + +const char * +yajl_status_to_string(yajl_status stat) +{ + const char * statStr = "unknown"; + switch (stat) { + case yajl_status_ok: + statStr = "ok, no error"; + break; + case yajl_status_client_canceled: + statStr = "client canceled parse"; + break; + case yajl_status_error: + statStr = "parse error"; + break; + } + return statStr; +} + +yajl_handle +yajl_alloc(const yajl_callbacks * callbacks, + yajl_alloc_funcs * afs, + void * ctx) +{ + yajl_handle hand = NULL; + yajl_alloc_funcs afsBuffer; + + /* first order of business is to set up memory allocation routines */ + if (afs != NULL) { + if (afs->malloc == NULL || afs->realloc == NULL || afs->free == NULL) + { + return NULL; + } + } else { + yajl_set_default_alloc_funcs(&afsBuffer); + afs = &afsBuffer; + } + + hand = (yajl_handle) YA_MALLOC(afs, sizeof(struct yajl_handle_t)); + + /* copy in pointers to allocation routines */ + memcpy((void *) &(hand->alloc), (void *) afs, sizeof(yajl_alloc_funcs)); + + hand->callbacks = callbacks; + hand->ctx = ctx; + hand->lexer = NULL; + hand->bytesConsumed = 0; + hand->decodeBuf = yajl_buf_alloc(&(hand->alloc)); + hand->flags = 0; + yajl_bs_init(hand->stateStack, &(hand->alloc)); + yajl_bs_push(hand->stateStack, yajl_state_start); + + return hand; +} + +int +yajl_config(yajl_handle h, yajl_option opt, ...) +{ + int rv = 1; + va_list ap; + va_start(ap, opt); + + switch(opt) { + case yajl_allow_comments: + case yajl_dont_validate_strings: + case yajl_allow_trailing_garbage: + case yajl_allow_multiple_values: + case yajl_allow_partial_values: + if (va_arg(ap, int)) h->flags |= opt; + else h->flags &= ~opt; + break; + default: + rv = 0; + } + va_end(ap); + + return rv; +} + +void +yajl_reset(yajl_handle handle) +{ + handle->bytesConsumed = 0; + if (handle->stateStack.used != 0) { + handle->stateStack.used = 0; + if (handle->lexer != NULL) { + yajl_lex_free(handle->lexer); + handle->lexer = NULL; + } + } + yajl_bs_push(handle->stateStack, yajl_state_start); +} + +void +yajl_free(yajl_handle handle) +{ + yajl_bs_free(handle->stateStack); + yajl_buf_free(handle->decodeBuf); + if (handle->lexer) { + yajl_lex_free(handle->lexer); + handle->lexer = NULL; + } + YA_FREE(&(handle->alloc), handle); +} + +yajl_status +yajl_parse(yajl_handle hand, const unsigned char * jsonText, + size_t jsonTextLen) +{ + yajl_status status; + + /* lazy allocation of the lexer */ + if (hand->lexer == NULL) { + hand->lexer = yajl_lex_alloc(&(hand->alloc), + hand->flags & yajl_allow_comments, + !(hand->flags & yajl_dont_validate_strings)); + } + + status = yajl_do_parse(hand, jsonText, jsonTextLen); + return status; +} + + +yajl_status +yajl_complete_parse(yajl_handle hand) +{ + /* The lexer is lazy allocated in the first call to parse. if parse is + * never called, then no data was provided to parse at all. This is a + * "premature EOF" error unless yajl_allow_partial_values is specified. + * allocating the lexer now is the simplest possible way to handle this + * case while preserving all the other semantics of the parser + * (multiple values, partial values, etc). */ + if (hand->lexer == NULL) { + hand->lexer = yajl_lex_alloc(&(hand->alloc), + hand->flags & yajl_allow_comments, + !(hand->flags & yajl_dont_validate_strings)); + } + + return yajl_do_finish(hand); +} + +unsigned char * +yajl_get_error(yajl_handle hand, int verbose, + const unsigned char * jsonText, size_t jsonTextLen) +{ + return yajl_render_error_string(hand, jsonText, jsonTextLen, verbose); +} + +size_t +yajl_get_bytes_consumed(yajl_handle hand) +{ + if (!hand) return 0; + else return hand->bytesConsumed; +} + + +void +yajl_free_error(yajl_handle hand, unsigned char * str) +{ + /* use memory allocation functions if set */ + YA_FREE(&(hand->alloc), str); +} + +/* XXX: add utility routines to parse from file */ diff --git a/src/yajl/yajl_alloc.c b/src/yajl/yajl_alloc.c new file mode 100644 index 0000000..96ad1d3 --- /dev/null +++ b/src/yajl/yajl_alloc.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/** + * \file yajl_alloc.h + * default memory allocation routines for yajl which use malloc/realloc and + * free + */ + +#include "yajl_alloc.h" +#include <stdlib.h> + +static void * yajl_internal_malloc(void *ctx, size_t sz) +{ + (void)ctx; + return malloc(sz); +} + +static void * yajl_internal_realloc(void *ctx, void * previous, + size_t sz) +{ + (void)ctx; + return realloc(previous, sz); +} + +static void yajl_internal_free(void *ctx, void * ptr) +{ + (void)ctx; + free(ptr); +} + +void yajl_set_default_alloc_funcs(yajl_alloc_funcs * yaf) +{ + yaf->malloc = yajl_internal_malloc; + yaf->free = yajl_internal_free; + yaf->realloc = yajl_internal_realloc; + yaf->ctx = NULL; +} + diff --git a/src/yajl/yajl_alloc.h b/src/yajl/yajl_alloc.h new file mode 100644 index 0000000..203c2f9 --- /dev/null +++ b/src/yajl/yajl_alloc.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/** + * \file yajl_alloc.h + * default memory allocation routines for yajl which use malloc/realloc and + * free + */ + +#ifndef __YAJL_ALLOC_H__ +#define __YAJL_ALLOC_H__ + +#include "api/yajl_common.h" + +#define YA_MALLOC(afs, sz) (afs)->malloc((afs)->ctx, (sz)) +#define YA_FREE(afs, ptr) (afs)->free((afs)->ctx, (ptr)) +#define YA_REALLOC(afs, ptr, sz) (afs)->realloc((afs)->ctx, (ptr), (sz)) + +void yajl_set_default_alloc_funcs(yajl_alloc_funcs * yaf); + +#endif diff --git a/src/yajl/yajl_buf.c b/src/yajl/yajl_buf.c new file mode 100644 index 0000000..1aeafde --- /dev/null +++ b/src/yajl/yajl_buf.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "yajl_buf.h" + +#include <assert.h> +#include <stdlib.h> +#include <string.h> + +#define YAJL_BUF_INIT_SIZE 2048 + +struct yajl_buf_t { + size_t len; + size_t used; + unsigned char * data; + yajl_alloc_funcs * alloc; +}; + +static +void yajl_buf_ensure_available(yajl_buf buf, size_t want) +{ + size_t need; + + assert(buf != NULL); + + /* first call */ + if (buf->data == NULL) { + buf->len = YAJL_BUF_INIT_SIZE; + buf->data = (unsigned char *) YA_MALLOC(buf->alloc, buf->len); + buf->data[0] = 0; + } + + need = buf->len; + + while (want >= (need - buf->used)) need <<= 1; + + if (need != buf->len) { + buf->data = (unsigned char *) YA_REALLOC(buf->alloc, buf->data, need); + buf->len = need; + } +} + +yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc) +{ + yajl_buf b = YA_MALLOC(alloc, sizeof(struct yajl_buf_t)); + memset((void *) b, 0, sizeof(struct yajl_buf_t)); + b->alloc = alloc; + return b; +} + +void yajl_buf_free(yajl_buf buf) +{ + assert(buf != NULL); + if (buf->data) YA_FREE(buf->alloc, buf->data); + YA_FREE(buf->alloc, buf); +} + +void yajl_buf_append(yajl_buf buf, const void * data, size_t len) +{ + yajl_buf_ensure_available(buf, len); + if (len > 0) { + assert(data != NULL); + memcpy(buf->data + buf->used, data, len); + buf->used += len; + buf->data[buf->used] = 0; + } +} + +void yajl_buf_clear(yajl_buf buf) +{ + buf->used = 0; + if (buf->data) buf->data[buf->used] = 0; +} + +const unsigned char * yajl_buf_data(yajl_buf buf) +{ + return buf->data; +} + +size_t yajl_buf_len(yajl_buf buf) +{ + return buf->used; +} + +void +yajl_buf_truncate(yajl_buf buf, size_t len) +{ + assert(len <= buf->used); + buf->used = len; +} diff --git a/src/yajl/yajl_buf.h b/src/yajl/yajl_buf.h new file mode 100644 index 0000000..a358246 --- /dev/null +++ b/src/yajl/yajl_buf.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __YAJL_BUF_H__ +#define __YAJL_BUF_H__ + +#include "api/yajl_common.h" +#include "yajl_alloc.h" + +/* + * Implementation/performance notes. If this were moved to a header + * only implementation using #define's where possible we might be + * able to sqeeze a little performance out of the guy by killing function + * call overhead. YMMV. + */ + +/** + * yajl_buf is a buffer with exponential growth. the buffer ensures that + * you are always null padded. + */ +typedef struct yajl_buf_t * yajl_buf; + +/* allocate a new buffer */ +yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc); + +/* free the buffer */ +void yajl_buf_free(yajl_buf buf); + +/* append a number of bytes to the buffer */ +void yajl_buf_append(yajl_buf buf, const void * data, size_t len); + +/* empty the buffer */ +void yajl_buf_clear(yajl_buf buf); + +/* get a pointer to the beginning of the buffer */ +const unsigned char * yajl_buf_data(yajl_buf buf); + +/* get the length of the buffer */ +size_t yajl_buf_len(yajl_buf buf); + +/* truncate the buffer */ +void yajl_buf_truncate(yajl_buf buf, size_t len); + +#endif diff --git a/src/yajl/yajl_bytestack.h b/src/yajl/yajl_bytestack.h new file mode 100644 index 0000000..9ea7d15 --- /dev/null +++ b/src/yajl/yajl_bytestack.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * A header only implementation of a simple stack of bytes, used in YAJL + * to maintain parse state. + */ + +#ifndef __YAJL_BYTESTACK_H__ +#define __YAJL_BYTESTACK_H__ + +#include "api/yajl_common.h" + +#define YAJL_BS_INC 128 + +typedef struct yajl_bytestack_t +{ + unsigned char * stack; + size_t size; + size_t used; + yajl_alloc_funcs * yaf; +} yajl_bytestack; + +/* initialize a bytestack */ +#define yajl_bs_init(obs, _yaf) { \ + (obs).stack = NULL; \ + (obs).size = 0; \ + (obs).used = 0; \ + (obs).yaf = (_yaf); \ + } \ + + +/* initialize a bytestack */ +#define yajl_bs_free(obs) \ + if ((obs).stack) (obs).yaf->free((obs).yaf->ctx, (obs).stack); + +#define yajl_bs_current(obs) \ + (assert((obs).used > 0), (obs).stack[(obs).used - 1]) + +#define yajl_bs_push(obs, byte) { \ + if (((obs).size - (obs).used) == 0) { \ + (obs).size += YAJL_BS_INC; \ + (obs).stack = (obs).yaf->realloc((obs).yaf->ctx,\ + (void *) (obs).stack, (obs).size);\ + } \ + (obs).stack[((obs).used)++] = (byte); \ +} + +/* removes the top item of the stack, returns nothing */ +#define yajl_bs_pop(obs) { ((obs).used)--; } + +#define yajl_bs_set(obs, byte) \ + (obs).stack[((obs).used) - 1] = (byte); + + +#endif diff --git a/src/yajl/yajl_common.h b/src/yajl/yajl_common.h new file mode 100644 index 0000000..49ca3a5 --- /dev/null +++ b/src/yajl/yajl_common.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __YAJL_COMMON_H__ +#define __YAJL_COMMON_H__ + +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define YAJL_MAX_DEPTH 128 + +/* msft dll export gunk. To build a DLL on windows, you + * must define WIN32, YAJL_SHARED, and YAJL_BUILD. To use a shared + * DLL, you must define YAJL_SHARED and WIN32 */ +#if defined(WIN32) && defined(YAJL_SHARED) +# ifdef YAJL_BUILD +# define YAJL_API __declspec(dllexport) +# else +# define YAJL_API __declspec(dllimport) +# endif +#else +# if defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__) >= 303 +# define YAJL_API __attribute__ ((visibility("default"))) +# else +# define YAJL_API +# endif +#endif + +/** pointer to a malloc function, supporting client overriding memory + * allocation routines */ +typedef void * (*yajl_malloc_func)(void *ctx, size_t sz); + +/** pointer to a free function, supporting client overriding memory + * allocation routines */ +typedef void (*yajl_free_func)(void *ctx, void * ptr); + +/** pointer to a realloc function which can resize an allocation. */ +typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, size_t sz); + +/** A structure which can be passed to yajl_*_alloc routines to allow the + * client to specify memory allocation functions to be used. */ +typedef struct +{ + /** pointer to a function that can allocate uninitialized memory */ + yajl_malloc_func malloc; + /** pointer to a function that can resize memory allocations */ + yajl_realloc_func realloc; + /** pointer to a function that can free memory allocated using + * reallocFunction or mallocFunction */ + yajl_free_func free; + /** a context pointer that will be passed to above allocation routines */ + void * ctx; +} yajl_alloc_funcs; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/yajl/yajl_encode.c b/src/yajl/yajl_encode.c new file mode 100644 index 0000000..db859a2 --- /dev/null +++ b/src/yajl/yajl_encode.c @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "yajl_encode.h" + +#include <assert.h> +#include <stdlib.h> +#include <string.h> +#include <stdio.h> + +static void CharToHex(unsigned char c, char * hexBuf) +{ + const char * hexchar = "0123456789ABCDEF"; + hexBuf[0] = hexchar[c >> 4]; + hexBuf[1] = hexchar[c & 0x0F]; +} + +void +yajl_string_encode(const yajl_print_t print, + void * ctx, + const unsigned char * str, + size_t len, + int escape_solidus) +{ + size_t beg = 0; + size_t end = 0; + char hexBuf[7]; + hexBuf[0] = '\\'; hexBuf[1] = 'u'; hexBuf[2] = '0'; hexBuf[3] = '0'; + hexBuf[6] = 0; + + while (end < len) { + const char * escaped = NULL; + switch (str[end]) { + case '\r': escaped = "\\r"; break; + case '\n': escaped = "\\n"; break; + case '\\': escaped = "\\\\"; break; + /* it is not required to escape a solidus in JSON: + * read sec. 2.5: http://www.ietf.org/rfc/rfc4627.txt + * specifically, this production from the grammar: + * unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + */ + case '/': if (escape_solidus) escaped = "\\/"; break; + case '"': escaped = "\\\""; break; + case '\f': escaped = "\\f"; break; + case '\b': escaped = "\\b"; break; + case '\t': escaped = "\\t"; break; + default: + if ((unsigned char) str[end] < 32) { + CharToHex(str[end], hexBuf + 4); + escaped = hexBuf; + } + break; + } + if (escaped != NULL) { + print(ctx, (const char *) (str + beg), end - beg); + print(ctx, escaped, (unsigned int)strlen(escaped)); + beg = ++end; + } else { + ++end; + } + } + print(ctx, (const char *) (str + beg), end - beg); +} + +static void hexToDigit(unsigned int * val, const unsigned char * hex) +{ + unsigned int i; + for (i=0;i<4;i++) { + unsigned char c = hex[i]; + if (c >= 'A') c = (c & ~0x20) - 7; + c -= '0'; + assert(!(c & 0xF0)); + *val = (*val << 4) | c; + } +} + +static void Utf32toUtf8(unsigned int codepoint, char * utf8Buf) +{ + if (codepoint < 0x80) { + utf8Buf[0] = (char) codepoint; + utf8Buf[1] = 0; + } else if (codepoint < 0x0800) { + utf8Buf[0] = (char) ((codepoint >> 6) | 0xC0); + utf8Buf[1] = (char) ((codepoint & 0x3F) | 0x80); + utf8Buf[2] = 0; + } else if (codepoint < 0x10000) { + utf8Buf[0] = (char) ((codepoint >> 12) | 0xE0); + utf8Buf[1] = (char) (((codepoint >> 6) & 0x3F) | 0x80); + utf8Buf[2] = (char) ((codepoint & 0x3F) | 0x80); + utf8Buf[3] = 0; + } else if (codepoint < 0x200000) { + utf8Buf[0] =(char)((codepoint >> 18) | 0xF0); + utf8Buf[1] =(char)(((codepoint >> 12) & 0x3F) | 0x80); + utf8Buf[2] =(char)(((codepoint >> 6) & 0x3F) | 0x80); + utf8Buf[3] =(char)((codepoint & 0x3F) | 0x80); + utf8Buf[4] = 0; + } else { + utf8Buf[0] = '?'; + utf8Buf[1] = 0; + } +} + +void yajl_string_decode(yajl_buf buf, const unsigned char * str, + size_t len) +{ + size_t beg = 0; + size_t end = 0; + + while (end < len) { + if (str[end] == '\\') { + char utf8Buf[5]; + const char * unescaped = "?"; + yajl_buf_append(buf, str + beg, end - beg); + switch (str[++end]) { + case 'r': unescaped = "\r"; break; + case 'n': unescaped = "\n"; break; + case '\\': unescaped = "\\"; break; + case '/': unescaped = "/"; break; + case '"': unescaped = "\""; break; + case 'f': unescaped = "\f"; break; + case 'b': unescaped = "\b"; break; + case 't': unescaped = "\t"; break; + case 'u': { + unsigned int codepoint = 0; + hexToDigit(&codepoint, str + ++end); + end+=3; + /* check if this is a surrogate */ + if ((codepoint & 0xFC00) == 0xD800) { + if (str[end + 1] == '\\' && str[end + 2] == 'u') { + end += 1; + unsigned int surrogate = 0; + hexToDigit(&surrogate, str + end + 2); + codepoint + = (((codepoint & 0x3F) << 10) + | ((((codepoint >> 6) & 0xF) + 1) << 16) + | (surrogate & 0x3FF)); + end += 5; + } else { + unescaped = "?"; + break; + } + } + + Utf32toUtf8(codepoint, utf8Buf); + unescaped = utf8Buf; + + if (codepoint == 0) { + yajl_buf_append(buf, unescaped, 1); + beg = ++end; + continue; + } + + break; + } + default: + assert("this should never happen" == NULL); + } + yajl_buf_append(buf, unescaped, (unsigned int)strlen(unescaped)); + beg = ++end; + } else { + end++; + } + } + yajl_buf_append(buf, str + beg, end - beg); +} + +#define ADV_PTR s++; if (!(len--)) return 0; + +int yajl_string_validate_utf8(const unsigned char * s, size_t len) +{ + if (!len) return 1; + if (!s) return 0; + + while (len--) { + /* single byte */ + if (*s <= 0x7f) { + /* noop */ + } + /* two byte */ + else if ((*s >> 5) == 0x6) { + ADV_PTR; + if (!((*s >> 6) == 0x2)) return 0; + } + /* three byte */ + else if ((*s >> 4) == 0x0e) { + ADV_PTR; + if (!((*s >> 6) == 0x2)) return 0; + ADV_PTR; + if (!((*s >> 6) == 0x2)) return 0; + } + /* four byte */ + else if ((*s >> 3) == 0x1e) { + ADV_PTR; + if (!((*s >> 6) == 0x2)) return 0; + ADV_PTR; + if (!((*s >> 6) == 0x2)) return 0; + ADV_PTR; + if (!((*s >> 6) == 0x2)) return 0; + } else { + return 0; + } + + s++; + } + + return 1; +} diff --git a/src/yajl/yajl_encode.h b/src/yajl/yajl_encode.h new file mode 100644 index 0000000..853a1a7 --- /dev/null +++ b/src/yajl/yajl_encode.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __YAJL_ENCODE_H__ +#define __YAJL_ENCODE_H__ + +#include "yajl_buf.h" +#include "api/yajl_gen.h" + +void yajl_string_encode(const yajl_print_t printer, + void * ctx, + const unsigned char * str, + size_t length, + int escape_solidus); + +void yajl_string_decode(yajl_buf buf, const unsigned char * str, + size_t length); + +int yajl_string_validate_utf8(const unsigned char * s, size_t len); + +#endif diff --git a/src/yajl/yajl_gen.c b/src/yajl/yajl_gen.c new file mode 100644 index 0000000..0f5c68e --- /dev/null +++ b/src/yajl/yajl_gen.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "api/yajl_gen.h" +#include "yajl_buf.h" +#include "yajl_encode.h" + +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <math.h> +#include <stdarg.h> + +typedef enum { + yajl_gen_start, + yajl_gen_map_start, + yajl_gen_map_key, + yajl_gen_map_val, + yajl_gen_array_start, + yajl_gen_in_array, + yajl_gen_complete, + yajl_gen_error +} yajl_gen_state; + +struct yajl_gen_t +{ + unsigned int flags; + unsigned int depth; + const char * indentString; + yajl_gen_state state[YAJL_MAX_DEPTH]; + yajl_print_t print; + void * ctx; /* yajl_buf */ + /* memory allocation routines */ + yajl_alloc_funcs alloc; +}; + +int +yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...) +{ + int rv = 1; + va_list ap; + va_start(ap, opt); + + switch(opt) { + case yajl_gen_beautify: + case yajl_gen_validate_utf8: + case yajl_gen_escape_solidus: + if (va_arg(ap, int)) g->flags |= opt; + else g->flags &= ~opt; + break; + case yajl_gen_indent_string: { + const char *indent = va_arg(ap, const char *); + g->indentString = indent; + for (; *indent; indent++) { + if (*indent != '\n' + && *indent != '\v' + && *indent != '\f' + && *indent != '\t' + && *indent != '\r' + && *indent != ' ') + { + g->indentString = NULL; + rv = 0; + } + } + break; + } + case yajl_gen_print_callback: + yajl_buf_free(g->ctx); + g->print = va_arg(ap, const yajl_print_t); + g->ctx = va_arg(ap, void *); + break; + default: + rv = 0; + } + + va_end(ap); + + return rv; +} + + + +yajl_gen +yajl_gen_alloc(const yajl_alloc_funcs * afs) +{ + yajl_gen g = NULL; + yajl_alloc_funcs afsBuffer; + + /* first order of business is to set up memory allocation routines */ + if (afs != NULL) { + if (afs->malloc == NULL || afs->realloc == NULL || afs->free == NULL) + { + return NULL; + } + } else { + yajl_set_default_alloc_funcs(&afsBuffer); + afs = &afsBuffer; + } + + g = (yajl_gen) YA_MALLOC(afs, sizeof(struct yajl_gen_t)); + if (!g) return NULL; + + memset((void *) g, 0, sizeof(struct yajl_gen_t)); + /* copy in pointers to allocation routines */ + memcpy((void *) &(g->alloc), (void *) afs, sizeof(yajl_alloc_funcs)); + + g->print = (yajl_print_t)&yajl_buf_append; + g->ctx = yajl_buf_alloc(&(g->alloc)); + g->indentString = " "; + + return g; +} + +void +yajl_gen_reset(yajl_gen g, const char * sep) +{ + g->depth = 0; + memset((void *) &(g->state), 0, sizeof(g->state)); + if (sep != NULL) g->print(g->ctx, sep, strlen(sep)); +} + +void +yajl_gen_free(yajl_gen g) +{ + if (g->print == (yajl_print_t)&yajl_buf_append) yajl_buf_free((yajl_buf)g->ctx); + YA_FREE(&(g->alloc), g); +} + +#define INSERT_SEP \ + if (g->state[g->depth] == yajl_gen_map_key || \ + g->state[g->depth] == yajl_gen_in_array) { \ + g->print(g->ctx, ",", 1); \ + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1); \ + } else if (g->state[g->depth] == yajl_gen_map_val) { \ + g->print(g->ctx, ":", 1); \ + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, " ", 1); \ + } + +#define INSERT_WHITESPACE \ + if ((g->flags & yajl_gen_beautify)) { \ + if (g->state[g->depth] != yajl_gen_map_val) { \ + unsigned int _i; \ + for (_i=0;_i<g->depth;_i++) \ + g->print(g->ctx, \ + g->indentString, \ + (unsigned int)strlen(g->indentString)); \ + } \ + } + +#define ENSURE_NOT_KEY \ + if (g->state[g->depth] == yajl_gen_map_key || \ + g->state[g->depth] == yajl_gen_map_start) { \ + return yajl_gen_keys_must_be_strings; \ + } \ + +/* check that we're not complete, or in error state. in a valid state + * to be generating */ +#define ENSURE_VALID_STATE \ + if (g->state[g->depth] == yajl_gen_error) { \ + return yajl_gen_in_error_state;\ + } else if (g->state[g->depth] == yajl_gen_complete) { \ + return yajl_gen_generation_complete; \ + } + +#define INCREMENT_DEPTH \ + if (++(g->depth) >= YAJL_MAX_DEPTH) return yajl_max_depth_exceeded; + +#define DECREMENT_DEPTH \ + if (--(g->depth) >= YAJL_MAX_DEPTH) return yajl_gen_generation_complete; + +#define APPENDED_ATOM \ + switch (g->state[g->depth]) { \ + case yajl_gen_start: \ + g->state[g->depth] = yajl_gen_complete; \ + break; \ + case yajl_gen_map_start: \ + case yajl_gen_map_key: \ + g->state[g->depth] = yajl_gen_map_val; \ + break; \ + case yajl_gen_array_start: \ + g->state[g->depth] = yajl_gen_in_array; \ + break; \ + case yajl_gen_map_val: \ + g->state[g->depth] = yajl_gen_map_key; \ + break; \ + default: \ + break; \ + } \ + +#define FINAL_NEWLINE \ + if ((g->flags & yajl_gen_beautify) && g->state[g->depth] == yajl_gen_complete) \ + g->print(g->ctx, "\n", 1); + +yajl_gen_status +yajl_gen_integer(yajl_gen g, long long int number) +{ + char i[32]; + ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE; + sprintf(i, "%lld", number); + g->print(g->ctx, i, (unsigned int)strlen(i)); + APPENDED_ATOM; + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +#if defined(_WIN32) || defined(WIN32) +#include <float.h> +#define isnan _isnan +#define isinf !_finite +#endif + +yajl_gen_status +yajl_gen_double(yajl_gen g, double number) +{ + char i[32]; + ENSURE_VALID_STATE; ENSURE_NOT_KEY; + if (isnan(number) || isinf(number)) return yajl_gen_invalid_number; + INSERT_SEP; INSERT_WHITESPACE; + sprintf(i, "%.20g", number); + if (strspn(i, "0123456789-") == strlen(i)) { + strcat(i, ".0"); + } + g->print(g->ctx, i, (unsigned int)strlen(i)); + APPENDED_ATOM; + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_number(yajl_gen g, const char * s, size_t l) +{ + ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE; + g->print(g->ctx, s, l); + APPENDED_ATOM; + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_string(yajl_gen g, const unsigned char * str, + size_t len) +{ + // if validation is enabled, check that the string is valid utf8 + // XXX: This checking could be done a little faster, in the same pass as + // the string encoding + if (g->flags & yajl_gen_validate_utf8) { + if (!yajl_string_validate_utf8(str, len)) { + return yajl_gen_invalid_string; + } + } + ENSURE_VALID_STATE; INSERT_SEP; INSERT_WHITESPACE; + g->print(g->ctx, "\"", 1); + yajl_string_encode(g->print, g->ctx, str, len, g->flags & yajl_gen_escape_solidus); + g->print(g->ctx, "\"", 1); + APPENDED_ATOM; + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_null(yajl_gen g) +{ + ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE; + g->print(g->ctx, "null", strlen("null")); + APPENDED_ATOM; + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_bool(yajl_gen g, int boolean) +{ + const char * val = boolean ? "true" : "false"; + + ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE; + g->print(g->ctx, val, (unsigned int)strlen(val)); + APPENDED_ATOM; + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_map_open(yajl_gen g) +{ + ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE; + INCREMENT_DEPTH; + + g->state[g->depth] = yajl_gen_map_start; + g->print(g->ctx, "{", 1); + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1); + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_map_close(yajl_gen g) +{ + ENSURE_VALID_STATE; + DECREMENT_DEPTH; + + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1); + APPENDED_ATOM; + INSERT_WHITESPACE; + g->print(g->ctx, "}", 1); + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_array_open(yajl_gen g) +{ + ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE; + INCREMENT_DEPTH; + g->state[g->depth] = yajl_gen_array_start; + g->print(g->ctx, "[", 1); + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1); + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_array_close(yajl_gen g) +{ + ENSURE_VALID_STATE; + DECREMENT_DEPTH; + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1); + APPENDED_ATOM; + INSERT_WHITESPACE; + g->print(g->ctx, "]", 1); + FINAL_NEWLINE; + return yajl_gen_status_ok; +} + +yajl_gen_status +yajl_gen_get_buf(yajl_gen g, const unsigned char ** buf, + size_t * len) +{ + if (g->print != (yajl_print_t)&yajl_buf_append) return yajl_gen_no_buf; + *buf = yajl_buf_data((yajl_buf)g->ctx); + *len = yajl_buf_len((yajl_buf)g->ctx); + return yajl_gen_status_ok; +} + +void +yajl_gen_clear(yajl_gen g) +{ + if (g->print == (yajl_print_t)&yajl_buf_append) yajl_buf_clear((yajl_buf)g->ctx); +} diff --git a/src/yajl/yajl_lex.c b/src/yajl/yajl_lex.c new file mode 100644 index 0000000..0b6f7cc --- /dev/null +++ b/src/yajl/yajl_lex.c @@ -0,0 +1,763 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "yajl_lex.h" +#include "yajl_buf.h" + +#include <stdlib.h> +#include <stdio.h> +#include <assert.h> +#include <string.h> + +#ifdef YAJL_LEXER_DEBUG +static const char * +tokToStr(yajl_tok tok) +{ + switch (tok) { + case yajl_tok_bool: return "bool"; + case yajl_tok_colon: return "colon"; + case yajl_tok_comma: return "comma"; + case yajl_tok_eof: return "eof"; + case yajl_tok_error: return "error"; + case yajl_tok_left_brace: return "brace"; + case yajl_tok_left_bracket: return "bracket"; + case yajl_tok_null: return "null"; + case yajl_tok_integer: return "integer"; + case yajl_tok_double: return "double"; + case yajl_tok_right_brace: return "brace"; + case yajl_tok_right_bracket: return "bracket"; + case yajl_tok_string: return "string"; + case yajl_tok_string_with_escapes: return "string_with_escapes"; + } + return "unknown"; +} +#endif + +/* Impact of the stream parsing feature on the lexer: + * + * YAJL support stream parsing. That is, the ability to parse the first + * bits of a chunk of JSON before the last bits are available (still on + * the network or disk). This makes the lexer more complex. The + * responsibility of the lexer is to handle transparently the case where + * a chunk boundary falls in the middle of a token. This is + * accomplished is via a buffer and a character reading abstraction. + * + * Overview of implementation + * + * When we lex to end of input string before end of token is hit, we + * copy all of the input text composing the token into our lexBuf. + * + * Every time we read a character, we do so through the readChar function. + * readChar's responsibility is to handle pulling all chars from the buffer + * before pulling chars from input text + */ + +struct yajl_lexer_t { + /* the overal line and char offset into the data */ + size_t lineOff; + size_t charOff; + + /* error */ + yajl_lex_error error; + + /* a input buffer to handle the case where a token is spread over + * multiple chunks */ + yajl_buf buf; + + /* in the case where we have data in the lexBuf, bufOff holds + * the current offset into the lexBuf. */ + size_t bufOff; + + /* are we using the lex buf? */ + unsigned int bufInUse; + + /* shall we allow comments? */ + unsigned int allowComments; + + /* shall we validate utf8 inside strings? */ + unsigned int validateUTF8; + + yajl_alloc_funcs * alloc; +}; + +#define readChar(lxr, txt, off) \ + (((lxr)->bufInUse && yajl_buf_len((lxr)->buf) && lxr->bufOff < yajl_buf_len((lxr)->buf)) ? \ + (*((const unsigned char *) yajl_buf_data((lxr)->buf) + ((lxr)->bufOff)++)) : \ + ((txt)[(*(off))++])) + +#define unreadChar(lxr, off) ((*(off) > 0) ? (*(off))-- : ((lxr)->bufOff--)) + +yajl_lexer +yajl_lex_alloc(yajl_alloc_funcs * alloc, + unsigned int allowComments, unsigned int validateUTF8) +{ + yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t)); + memset((void *) lxr, 0, sizeof(struct yajl_lexer_t)); + lxr->buf = yajl_buf_alloc(alloc); + lxr->allowComments = allowComments; + lxr->validateUTF8 = validateUTF8; + lxr->alloc = alloc; + return lxr; +} + +void +yajl_lex_free(yajl_lexer lxr) +{ + yajl_buf_free(lxr->buf); + YA_FREE(lxr->alloc, lxr); + return; +} + +/* a lookup table which lets us quickly determine three things: + * VEC - valid escaped control char + * note. the solidus '/' may be escaped or not. + * IJC - invalid json char + * VHC - valid hex char + * NFP - needs further processing (from a string scanning perspective) + * NUC - needs utf8 checking when enabled (from a string scanning perspective) + */ +#define VEC 0x01 +#define IJC 0x02 +#define VHC 0x04 +#define NFP 0x08 +#define NUC 0x10 + +static const char charLookupTable[256] = +{ +/*00*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , +/*08*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , +/*10*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , +/*18*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , + +/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , 0 , 0 , +/*28*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , VEC , +/*30*/ VHC , VHC , VHC , VHC , VHC , VHC , VHC , VHC , +/*38*/ VHC , VHC , 0 , 0 , 0 , 0 , 0 , 0 , + +/*40*/ 0 , VHC , VHC , VHC , VHC , VHC , VHC , 0 , +/*48*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , +/*50*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , +/*58*/ 0 , 0 , 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , + +/*60*/ 0 , VHC , VEC|VHC, VHC , VHC , VHC , VEC|VHC, 0 , +/*68*/ 0 , 0 , 0 , 0 , 0 , 0 , VEC , 0 , +/*70*/ 0 , 0 , VEC , 0 , VEC , 0 , 0 , 0 , +/*78*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , + + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , + NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC +}; + +/** process a variable length utf8 encoded codepoint. + * + * returns: + * yajl_tok_string - if valid utf8 char was parsed and offset was + * advanced + * yajl_tok_eof - if end of input was hit before validation could + * complete + * yajl_tok_error - if invalid utf8 was encountered + * + * NOTE: on error the offset will point to the first char of the + * invalid utf8 */ +#define UTF8_CHECK_EOF if (*offset >= jsonTextLen) { return yajl_tok_eof; } + +static yajl_tok +yajl_lex_utf8_char(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + unsigned char curChar) +{ + if (curChar <= 0x7f) { + /* single byte */ + return yajl_tok_string; + } else if ((curChar >> 5) == 0x6) { + /* two byte */ + UTF8_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if ((curChar >> 6) == 0x2) return yajl_tok_string; + } else if ((curChar >> 4) == 0x0e) { + /* three byte */ + UTF8_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if ((curChar >> 6) == 0x2) { + UTF8_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if ((curChar >> 6) == 0x2) return yajl_tok_string; + } + } else if ((curChar >> 3) == 0x1e) { + /* four byte */ + UTF8_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if ((curChar >> 6) == 0x2) { + UTF8_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if ((curChar >> 6) == 0x2) { + UTF8_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if ((curChar >> 6) == 0x2) return yajl_tok_string; + } + } + } + + return yajl_tok_error; +} + +/* lex a string. input is the lexer, pointer to beginning of + * json text, and start of string (offset). + * a token is returned which has the following meanings: + * yajl_tok_string: lex of string was successful. offset points to + * terminating '"'. + * yajl_tok_eof: end of text was encountered before we could complete + * the lex. + * yajl_tok_error: embedded in the string were unallowable chars. offset + * points to the offending char + */ +#define STR_CHECK_EOF \ +if (*offset >= jsonTextLen) { \ + tok = yajl_tok_eof; \ + goto finish_string_lex; \ +} + +/** scan a string for interesting characters that might need further + * review. return the number of chars that are uninteresting and can + * be skipped. + * (lth) hi world, any thoughts on how to make this routine faster? */ +static size_t +yajl_string_scan(const unsigned char * buf, size_t len, int utf8check) +{ + unsigned char mask = IJC|NFP|(utf8check ? NUC : 0); + size_t skip = 0; + while (skip < len && !(charLookupTable[*buf] & mask)) + { + skip++; + buf++; + } + return skip; +} + +static yajl_tok +yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset) +{ + yajl_tok tok = yajl_tok_error; + int hasEscapes = 0; + + for (;;) { + unsigned char curChar; + + /* now jump into a faster scanning routine to skip as much + * of the buffers as possible */ + { + const unsigned char * p; + size_t len; + + if ((lexer->bufInUse && yajl_buf_len(lexer->buf) && + lexer->bufOff < yajl_buf_len(lexer->buf))) + { + p = ((const unsigned char *) yajl_buf_data(lexer->buf) + + (lexer->bufOff)); + len = yajl_buf_len(lexer->buf) - lexer->bufOff; + lexer->bufOff += yajl_string_scan(p, len, lexer->validateUTF8); + } + else if (*offset < jsonTextLen) + { + p = jsonText + *offset; + len = jsonTextLen - *offset; + *offset += yajl_string_scan(p, len, lexer->validateUTF8); + } + } + + STR_CHECK_EOF; + + curChar = readChar(lexer, jsonText, offset); + + /* quote terminates */ + if (curChar == '"') { + tok = yajl_tok_string; + break; + } + /* backslash escapes a set of control chars, */ + else if (curChar == '\\') { + hasEscapes = 1; + STR_CHECK_EOF; + + /* special case \u */ + curChar = readChar(lexer, jsonText, offset); + if (curChar == 'u') { + unsigned int i = 0; + + for (i=0;i<4;i++) { + STR_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if (!(charLookupTable[curChar] & VHC)) { + /* back up to offending char */ + unreadChar(lexer, offset); + lexer->error = yajl_lex_string_invalid_hex_char; + goto finish_string_lex; + } + } + } else if (!(charLookupTable[curChar] & VEC)) { + /* back up to offending char */ + unreadChar(lexer, offset); + lexer->error = yajl_lex_string_invalid_escaped_char; + goto finish_string_lex; + } + } + /* when not validating UTF8 it's a simple table lookup to determine + * if the present character is invalid */ + else if(charLookupTable[curChar] & IJC) { + /* back up to offending char */ + unreadChar(lexer, offset); + lexer->error = yajl_lex_string_invalid_json_char; + goto finish_string_lex; + } + /* when in validate UTF8 mode we need to do some extra work */ + else if (lexer->validateUTF8) { + yajl_tok t = yajl_lex_utf8_char(lexer, jsonText, jsonTextLen, + offset, curChar); + + if (t == yajl_tok_eof) { + tok = yajl_tok_eof; + goto finish_string_lex; + } else if (t == yajl_tok_error) { + lexer->error = yajl_lex_string_invalid_utf8; + goto finish_string_lex; + } + } + /* accept it, and move on */ + } + finish_string_lex: + /* tell our buddy, the parser, wether he needs to process this string + * again */ + if (hasEscapes && tok == yajl_tok_string) { + tok = yajl_tok_string_with_escapes; + } + + return tok; +} + +#define RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof; + +static yajl_tok +yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset) +{ + /** XXX: numbers are the only entities in json that we must lex + * _beyond_ in order to know that they are complete. There + * is an ambiguous case for integers at EOF. */ + + unsigned char c; + + yajl_tok tok = yajl_tok_integer; + + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + + /* optional leading minus */ + if (c == '-') { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } + + /* a single zero, or a series of integers */ + if (c == '0') { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } else if (c >= '1' && c <= '9') { + do { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } while (c >= '0' && c <= '9'); + } else { + unreadChar(lexer, offset); + lexer->error = yajl_lex_missing_integer_after_minus; + return yajl_tok_error; + } + + /* optional fraction (indicates this is floating point) */ + if (c == '.') { + int numRd = 0; + + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + + while (c >= '0' && c <= '9') { + numRd++; + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } + + if (!numRd) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_missing_integer_after_decimal; + return yajl_tok_error; + } + tok = yajl_tok_double; + } + + /* optional exponent (indicates this is floating point) */ + if (c == 'e' || c == 'E') { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + + /* optional sign */ + if (c == '+' || c == '-') { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } + + if (c >= '0' && c <= '9') { + do { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } while (c >= '0' && c <= '9'); + } else { + unreadChar(lexer, offset); + lexer->error = yajl_lex_missing_integer_after_exponent; + return yajl_tok_error; + } + tok = yajl_tok_double; + } + + /* we always go "one too far" */ + unreadChar(lexer, offset); + + return tok; +} + +static yajl_tok +yajl_lex_comment(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset) +{ + unsigned char c; + + yajl_tok tok = yajl_tok_comment; + + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + + /* either slash or star expected */ + if (c == '/') { + /* now we throw away until end of line */ + do { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } while (c != '\n'); + } else if (c == '*') { + /* now we throw away until end of comment */ + for (;;) { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + if (c == '*') { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + if (c == '/') { + break; + } else { + unreadChar(lexer, offset); + } + } + } + } else { + lexer->error = yajl_lex_invalid_char; + tok = yajl_tok_error; + } + + return tok; +} + +yajl_tok +yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + const unsigned char ** outBuf, size_t * outLen) +{ + yajl_tok tok = yajl_tok_error; + unsigned char c; + size_t startOffset = *offset; + + *outBuf = NULL; + *outLen = 0; + + for (;;) { + assert(*offset <= jsonTextLen); + + if (*offset >= jsonTextLen) { + tok = yajl_tok_eof; + goto lexed; + } + + c = readChar(lexer, jsonText, offset); + + switch (c) { + case '{': + tok = yajl_tok_left_bracket; + goto lexed; + case '}': + tok = yajl_tok_right_bracket; + goto lexed; + case '[': + tok = yajl_tok_left_brace; + goto lexed; + case ']': + tok = yajl_tok_right_brace; + goto lexed; + case ',': + tok = yajl_tok_comma; + goto lexed; + case ':': + tok = yajl_tok_colon; + goto lexed; + case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': + startOffset++; + break; + case 't': { + const char * want = "rue"; + do { + if (*offset >= jsonTextLen) { + tok = yajl_tok_eof; + goto lexed; + } + c = readChar(lexer, jsonText, offset); + if (c != *want) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_invalid_string; + tok = yajl_tok_error; + goto lexed; + } + } while (*(++want)); + tok = yajl_tok_bool; + goto lexed; + } + case 'f': { + const char * want = "alse"; + do { + if (*offset >= jsonTextLen) { + tok = yajl_tok_eof; + goto lexed; + } + c = readChar(lexer, jsonText, offset); + if (c != *want) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_invalid_string; + tok = yajl_tok_error; + goto lexed; + } + } while (*(++want)); + tok = yajl_tok_bool; + goto lexed; + } + case 'n': { + const char * want = "ull"; + do { + if (*offset >= jsonTextLen) { + tok = yajl_tok_eof; + goto lexed; + } + c = readChar(lexer, jsonText, offset); + if (c != *want) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_invalid_string; + tok = yajl_tok_error; + goto lexed; + } + } while (*(++want)); + tok = yajl_tok_null; + goto lexed; + } + case '"': { + tok = yajl_lex_string(lexer, (const unsigned char *) jsonText, + jsonTextLen, offset); + goto lexed; + } + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + /* integer parsing wants to start from the beginning */ + unreadChar(lexer, offset); + tok = yajl_lex_number(lexer, (const unsigned char *) jsonText, + jsonTextLen, offset); + goto lexed; + } + case '/': + /* hey, look, a probable comment! If comments are disabled + * it's an error. */ + if (!lexer->allowComments) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_unallowed_comment; + tok = yajl_tok_error; + goto lexed; + } + /* if comments are enabled, then we should try to lex + * the thing. possible outcomes are + * - successful lex (tok_comment, which means continue), + * - malformed comment opening (slash not followed by + * '*' or '/') (tok_error) + * - eof hit. (tok_eof) */ + tok = yajl_lex_comment(lexer, (const unsigned char *) jsonText, + jsonTextLen, offset); + if (tok == yajl_tok_comment) { + /* "error" is silly, but that's the initial + * state of tok. guilty until proven innocent. */ + tok = yajl_tok_error; + yajl_buf_clear(lexer->buf); + lexer->bufInUse = 0; + startOffset = *offset; + break; + } + /* hit error or eof, bail */ + goto lexed; + default: + lexer->error = yajl_lex_invalid_char; + tok = yajl_tok_error; + goto lexed; + } + } + + + lexed: + /* need to append to buffer if the buffer is in use or + * if it's an EOF token */ + if (tok == yajl_tok_eof || lexer->bufInUse) { + if (!lexer->bufInUse) yajl_buf_clear(lexer->buf); + lexer->bufInUse = 1; + yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset); + lexer->bufOff = 0; + + if (tok != yajl_tok_eof) { + *outBuf = yajl_buf_data(lexer->buf); + *outLen = yajl_buf_len(lexer->buf); + lexer->bufInUse = 0; + } + } else if (tok != yajl_tok_error) { + *outBuf = jsonText + startOffset; + *outLen = *offset - startOffset; + } + + /* special case for strings. skip the quotes. */ + if (tok == yajl_tok_string || tok == yajl_tok_string_with_escapes) + { + assert(*outLen >= 2); + (*outBuf)++; + *outLen -= 2; + } + + +#ifdef YAJL_LEXER_DEBUG + if (tok == yajl_tok_error) { + printf("lexical error: %s\n", + yajl_lex_error_to_string(yajl_lex_get_error(lexer))); + } else if (tok == yajl_tok_eof) { + printf("EOF hit\n"); + } else { + printf("lexed %s: '", tokToStr(tok)); + fwrite(*outBuf, 1, *outLen, stdout); + printf("'\n"); + } +#endif + + return tok; +} + +const char * +yajl_lex_error_to_string(yajl_lex_error error) +{ + switch (error) { + case yajl_lex_e_ok: + return "ok, no error"; + case yajl_lex_string_invalid_utf8: + return "invalid bytes in UTF8 string."; + case yajl_lex_string_invalid_escaped_char: + return "inside a string, '\\' occurs before a character " + "which it may not."; + case yajl_lex_string_invalid_json_char: + return "invalid character inside string."; + case yajl_lex_string_invalid_hex_char: + return "invalid (non-hex) character occurs after '\\u' inside " + "string."; + case yajl_lex_invalid_char: + return "invalid char in json text."; + case yajl_lex_invalid_string: + return "invalid string in json text."; + case yajl_lex_missing_integer_after_exponent: + return "malformed number, a digit is required after the exponent."; + case yajl_lex_missing_integer_after_decimal: + return "malformed number, a digit is required after the " + "decimal point."; + case yajl_lex_missing_integer_after_minus: + return "malformed number, a digit is required after the " + "minus sign."; + case yajl_lex_unallowed_comment: + return "probable comment found in input text, comments are " + "not enabled."; + } + return "unknown error code"; +} + + +/** allows access to more specific information about the lexical + * error when yajl_lex_lex returns yajl_tok_error. */ +yajl_lex_error +yajl_lex_get_error(yajl_lexer lexer) +{ + if (lexer == NULL) return (yajl_lex_error) -1; + return lexer->error; +} + +size_t yajl_lex_current_line(yajl_lexer lexer) +{ + return lexer->lineOff; +} + +size_t yajl_lex_current_char(yajl_lexer lexer) +{ + return lexer->charOff; +} + +yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t offset) +{ + const unsigned char * outBuf; + size_t outLen; + size_t bufLen = yajl_buf_len(lexer->buf); + size_t bufOff = lexer->bufOff; + unsigned int bufInUse = lexer->bufInUse; + yajl_tok tok; + + tok = yajl_lex_lex(lexer, jsonText, jsonTextLen, &offset, + &outBuf, &outLen); + + lexer->bufOff = bufOff; + lexer->bufInUse = bufInUse; + yajl_buf_truncate(lexer->buf, bufLen); + + return tok; +} diff --git a/src/yajl/yajl_lex.h b/src/yajl/yajl_lex.h new file mode 100644 index 0000000..fd17c00 --- /dev/null +++ b/src/yajl/yajl_lex.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __YAJL_LEX_H__ +#define __YAJL_LEX_H__ + +#include "api/yajl_common.h" + +typedef enum { + yajl_tok_bool, + yajl_tok_colon, + yajl_tok_comma, + yajl_tok_eof, + yajl_tok_error, + yajl_tok_left_brace, + yajl_tok_left_bracket, + yajl_tok_null, + yajl_tok_right_brace, + yajl_tok_right_bracket, + + /* we differentiate between integers and doubles to allow the + * parser to interpret the number without re-scanning */ + yajl_tok_integer, + yajl_tok_double, + + /* we differentiate between strings which require further processing, + * and strings that do not */ + yajl_tok_string, + yajl_tok_string_with_escapes, + + /* comment tokens are not currently returned to the parser, ever */ + yajl_tok_comment +} yajl_tok; + +typedef struct yajl_lexer_t * yajl_lexer; + +yajl_lexer yajl_lex_alloc(yajl_alloc_funcs * alloc, + unsigned int allowComments, + unsigned int validateUTF8); + +void yajl_lex_free(yajl_lexer lexer); + +/** + * run/continue a lex. "offset" is an input/output parameter. + * It should be initialized to zero for a + * new chunk of target text, and upon subsetquent calls with the same + * target text should passed with the value of the previous invocation. + * + * the client may be interested in the value of offset when an error is + * returned from the lexer. This allows the client to render useful + * error messages. + * + * When you pass the next chunk of data, context should be reinitialized + * to zero. + * + * Finally, the output buffer is usually just a pointer into the jsonText, + * however in cases where the entity being lexed spans multiple chunks, + * the lexer will buffer the entity and the data returned will be + * a pointer into that buffer. + * + * This behavior is abstracted from client code except for the performance + * implications which require that the client choose a reasonable chunk + * size to get adequate performance. + */ +yajl_tok yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + const unsigned char ** outBuf, size_t * outLen); + +/** have a peek at the next token, but don't move the lexer forward */ +yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t offset); + + +typedef enum { + yajl_lex_e_ok = 0, + yajl_lex_string_invalid_utf8, + yajl_lex_string_invalid_escaped_char, + yajl_lex_string_invalid_json_char, + yajl_lex_string_invalid_hex_char, + yajl_lex_invalid_char, + yajl_lex_invalid_string, + yajl_lex_missing_integer_after_decimal, + yajl_lex_missing_integer_after_exponent, + yajl_lex_missing_integer_after_minus, + yajl_lex_unallowed_comment +} yajl_lex_error; + +const char * yajl_lex_error_to_string(yajl_lex_error error); + +/** allows access to more specific information about the lexical + * error when yajl_lex_lex returns yajl_tok_error. */ +yajl_lex_error yajl_lex_get_error(yajl_lexer lexer); + +/** get the current offset into the most recently lexed json string. */ +size_t yajl_lex_current_offset(yajl_lexer lexer); + +/** get the number of lines lexed by this lexer instance */ +size_t yajl_lex_current_line(yajl_lexer lexer); + +/** get the number of chars lexed by this lexer instance since the last + * \n or \r */ +size_t yajl_lex_current_char(yajl_lexer lexer); + +#endif diff --git a/src/yajl/yajl_parser.c b/src/yajl/yajl_parser.c new file mode 100644 index 0000000..c0c538e --- /dev/null +++ b/src/yajl/yajl_parser.c @@ -0,0 +1,498 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "api/yajl_parse.h" +#include "yajl_lex.h" +#include "yajl_parser.h" +#include "yajl_encode.h" +#include "yajl_bytestack.h" + +#include <stdlib.h> +#include <limits.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <ctype.h> +#include <assert.h> +#include <math.h> + +#define MAX_VALUE_TO_MULTIPLY ((LLONG_MAX / 10) + (LLONG_MAX % 10)) + + /* same semantics as strtol */ +long long +yajl_parse_integer(const unsigned char *number, unsigned int length) +{ + long long ret = 0; + long sign = 1; + const unsigned char *pos = number; + if (*pos == '-') { pos++; sign = -1; } + if (*pos == '+') { pos++; } + + while (pos < number + length) { + if ( ret > MAX_VALUE_TO_MULTIPLY ) { + errno = ERANGE; + return sign == 1 ? LLONG_MAX : LLONG_MIN; + } + ret *= 10; + if (LLONG_MAX - ret < (*pos - '0')) { + errno = ERANGE; + return sign == 1 ? LLONG_MAX : LLONG_MIN; + } + if (*pos < '0' || *pos > '9') { + errno = ERANGE; + return sign == 1 ? LLONG_MAX : LLONG_MIN; + } + ret += (*pos++ - '0'); + } + + return sign * ret; +} + +unsigned char * +yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText, + size_t jsonTextLen, int verbose) +{ + size_t offset = hand->bytesConsumed; + unsigned char * str; + const char * errorType = NULL; + const char * errorText = NULL; + char text[72]; + const char * arrow = " (right here) ------^\n"; + + if (yajl_bs_current(hand->stateStack) == yajl_state_parse_error) { + errorType = "parse"; + errorText = hand->parseError; + } else if (yajl_bs_current(hand->stateStack) == yajl_state_lexical_error) { + errorType = "lexical"; + errorText = yajl_lex_error_to_string(yajl_lex_get_error(hand->lexer)); + } else { + errorType = "unknown"; + } + + { + size_t memneeded = 0; + memneeded += strlen(errorType); + memneeded += strlen(" error"); + if (errorText != NULL) { + memneeded += strlen(": "); + memneeded += strlen(errorText); + } + str = (unsigned char *) YA_MALLOC(&(hand->alloc), memneeded + 2); + if (!str) return NULL; + str[0] = 0; + strcat((char *) str, errorType); + strcat((char *) str, " error"); + if (errorText != NULL) { + strcat((char *) str, ": "); + strcat((char *) str, errorText); + } + strcat((char *) str, "\n"); + } + + /* now we append as many spaces as needed to make sure the error + * falls at char 41, if verbose was specified */ + if (verbose) { + size_t start, end, i; + size_t spacesNeeded; + + spacesNeeded = (offset < 30 ? 40 - offset : 10); + start = (offset >= 30 ? offset - 30 : 0); + end = (offset + 30 > jsonTextLen ? jsonTextLen : offset + 30); + + for (i=0;i<spacesNeeded;i++) text[i] = ' '; + + for (;start < end;start++, i++) { + if (jsonText[start] != '\n' && jsonText[start] != '\r' && jsonText[start] != '\t') + { + text[i] = jsonText[start]; + } + else + { + text[i] = ' '; + } + } + assert(i <= 71); + text[i++] = '\n'; + text[i] = 0; + { + char * newStr = (char *) + YA_MALLOC(&(hand->alloc), (unsigned int)(strlen((char *) str) + + strlen((char *) text) + + strlen(arrow) + 1)); + if (newStr) { + newStr[0] = 0; + strcat((char *) newStr, (char *) str); + strcat((char *) newStr, text); + strcat((char *) newStr, arrow); + } + YA_FREE(&(hand->alloc), str); + str = (unsigned char *) newStr; + } + } + return str; +} + +/* check for client cancelation */ +#define _CC_CHK(x) \ + if (!(x)) { \ + yajl_bs_set(hand->stateStack, yajl_state_parse_error); \ + hand->parseError = \ + "client cancelled parse via callback return value"; \ + return yajl_status_client_canceled; \ + } + + +yajl_status +yajl_do_finish(yajl_handle hand) +{ + yajl_status stat; + stat = yajl_do_parse(hand,(const unsigned char *) " ",1); + + if (stat != yajl_status_ok) return stat; + + switch(yajl_bs_current(hand->stateStack)) + { + case yajl_state_parse_error: + case yajl_state_lexical_error: + return yajl_status_error; + case yajl_state_got_value: + case yajl_state_parse_complete: + return yajl_status_ok; + default: + if (!(hand->flags & yajl_allow_partial_values)) + { + yajl_bs_set(hand->stateStack, yajl_state_parse_error); + hand->parseError = "premature EOF"; + return yajl_status_error; + } + return yajl_status_ok; + } +} + +yajl_status +yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, + size_t jsonTextLen) +{ + yajl_tok tok; + const unsigned char * buf; + size_t bufLen; + size_t * offset = &(hand->bytesConsumed); + + *offset = 0; + + around_again: + switch (yajl_bs_current(hand->stateStack)) { + case yajl_state_parse_complete: + if (hand->flags & yajl_allow_multiple_values) { + yajl_bs_set(hand->stateStack, yajl_state_got_value); + goto around_again; + } + if (!(hand->flags & yajl_allow_trailing_garbage)) { + if (*offset != jsonTextLen) { + tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + offset, &buf, &bufLen); + if (tok != yajl_tok_eof) { + yajl_bs_set(hand->stateStack, yajl_state_parse_error); + hand->parseError = "trailing garbage"; + } + goto around_again; + } + } + return yajl_status_ok; + case yajl_state_lexical_error: + case yajl_state_parse_error: + return yajl_status_error; + case yajl_state_start: + case yajl_state_got_value: + case yajl_state_map_need_val: + case yajl_state_array_need_val: + case yajl_state_array_start: { + /* for arrays and maps, we advance the state for this + * depth, then push the state of the next depth. + * If an error occurs during the parsing of the nesting + * enitity, the state at this level will not matter. + * a state that needs pushing will be anything other + * than state_start */ + + yajl_state stateToPush = yajl_state_start; + + tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + offset, &buf, &bufLen); + + switch (tok) { + case yajl_tok_eof: + return yajl_status_ok; + case yajl_tok_error: + yajl_bs_set(hand->stateStack, yajl_state_lexical_error); + goto around_again; + case yajl_tok_string: + if (hand->callbacks && hand->callbacks->yajl_string) { + _CC_CHK(hand->callbacks->yajl_string(hand->ctx, + buf, bufLen)); + } + break; + case yajl_tok_string_with_escapes: + if (hand->callbacks && hand->callbacks->yajl_string) { + yajl_buf_clear(hand->decodeBuf); + yajl_string_decode(hand->decodeBuf, buf, bufLen); + _CC_CHK(hand->callbacks->yajl_string( + hand->ctx, yajl_buf_data(hand->decodeBuf), + yajl_buf_len(hand->decodeBuf))); + } + break; + case yajl_tok_bool: + if (hand->callbacks && hand->callbacks->yajl_boolean) { + _CC_CHK(hand->callbacks->yajl_boolean(hand->ctx, + *buf == 't')); + } + break; + case yajl_tok_null: + if (hand->callbacks && hand->callbacks->yajl_null) { + _CC_CHK(hand->callbacks->yajl_null(hand->ctx)); + } + break; + case yajl_tok_left_bracket: + if (hand->callbacks && hand->callbacks->yajl_start_map) { + _CC_CHK(hand->callbacks->yajl_start_map(hand->ctx)); + } + stateToPush = yajl_state_map_start; + break; + case yajl_tok_left_brace: + if (hand->callbacks && hand->callbacks->yajl_start_array) { + _CC_CHK(hand->callbacks->yajl_start_array(hand->ctx)); + } + stateToPush = yajl_state_array_start; + break; + case yajl_tok_integer: + if (hand->callbacks) { + if (hand->callbacks->yajl_number) { + _CC_CHK(hand->callbacks->yajl_number( + hand->ctx,(const char *) buf, bufLen)); + } else if (hand->callbacks->yajl_integer) { + long long int i = 0; + errno = 0; + i = yajl_parse_integer(buf, bufLen); + if ((i == LLONG_MIN || i == LLONG_MAX) && + errno == ERANGE) + { + yajl_bs_set(hand->stateStack, + yajl_state_parse_error); + hand->parseError = "integer overflow" ; + /* try to restore error offset */ + if (*offset >= bufLen) *offset -= bufLen; + else *offset = 0; + goto around_again; + } + _CC_CHK(hand->callbacks->yajl_integer(hand->ctx, + i)); + } + } + break; + case yajl_tok_double: + if (hand->callbacks) { + if (hand->callbacks->yajl_number) { + _CC_CHK(hand->callbacks->yajl_number( + hand->ctx, (const char *) buf, bufLen)); + } else if (hand->callbacks->yajl_double) { + double d = 0.0; + yajl_buf_clear(hand->decodeBuf); + yajl_buf_append(hand->decodeBuf, buf, bufLen); + buf = yajl_buf_data(hand->decodeBuf); + errno = 0; + d = strtod((char *) buf, NULL); + if ((d == HUGE_VAL || d == -HUGE_VAL) && + errno == ERANGE) + { + yajl_bs_set(hand->stateStack, + yajl_state_parse_error); + hand->parseError = "numeric (floating point) " + "overflow"; + /* try to restore error offset */ + if (*offset >= bufLen) *offset -= bufLen; + else *offset = 0; + goto around_again; + } + _CC_CHK(hand->callbacks->yajl_double(hand->ctx, + d)); + } + } + break; + case yajl_tok_right_brace: { + if (yajl_bs_current(hand->stateStack) == + yajl_state_array_start) + { + if (hand->callbacks && + hand->callbacks->yajl_end_array) + { + _CC_CHK(hand->callbacks->yajl_end_array(hand->ctx)); + } + yajl_bs_pop(hand->stateStack); + goto around_again; + } + /* intentional fall-through */ + } + case yajl_tok_colon: + case yajl_tok_comma: + case yajl_tok_right_bracket: + yajl_bs_set(hand->stateStack, yajl_state_parse_error); + hand->parseError = + "unallowed token at this point in JSON text"; + goto around_again; + default: + yajl_bs_set(hand->stateStack, yajl_state_parse_error); + hand->parseError = "invalid token, internal error"; + goto around_again; + } + /* got a value. transition depends on the state we're in. */ + { + yajl_state s = yajl_bs_current(hand->stateStack); + if (s == yajl_state_start || s == yajl_state_got_value) { + yajl_bs_set(hand->stateStack, yajl_state_parse_complete); + } else if (s == yajl_state_map_need_val) { + yajl_bs_set(hand->stateStack, yajl_state_map_got_val); + } else { + yajl_bs_set(hand->stateStack, yajl_state_array_got_val); + } + } + if (stateToPush != yajl_state_start) { + yajl_bs_push(hand->stateStack, stateToPush); + } + + goto around_again; + } + case yajl_state_map_start: + case yajl_state_map_need_key: { + /* only difference between these two states is that in + * start '}' is valid, whereas in need_key, we've parsed + * a comma, and a string key _must_ follow */ + tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + offset, &buf, &bufLen); + switch (tok) { + case yajl_tok_eof: + return yajl_status_ok; + case yajl_tok_error: + yajl_bs_set(hand->stateStack, yajl_state_lexical_error); + goto around_again; + case yajl_tok_string_with_escapes: + if (hand->callbacks && hand->callbacks->yajl_map_key) { + yajl_buf_clear(hand->decodeBuf); + yajl_string_decode(hand->decodeBuf, buf, bufLen); + buf = yajl_buf_data(hand->decodeBuf); + bufLen = yajl_buf_len(hand->decodeBuf); + } + /* intentional fall-through */ + case yajl_tok_string: + if (hand->callbacks && hand->callbacks->yajl_map_key) { + _CC_CHK(hand->callbacks->yajl_map_key(hand->ctx, buf, + bufLen)); + } + yajl_bs_set(hand->stateStack, yajl_state_map_sep); + goto around_again; + case yajl_tok_right_bracket: + if (yajl_bs_current(hand->stateStack) == + yajl_state_map_start) + { + if (hand->callbacks && hand->callbacks->yajl_end_map) { + _CC_CHK(hand->callbacks->yajl_end_map(hand->ctx)); + } + yajl_bs_pop(hand->stateStack); + goto around_again; + } + default: + yajl_bs_set(hand->stateStack, yajl_state_parse_error); + hand->parseError = + "invalid object key (must be a string)"; + goto around_again; + } + } + case yajl_state_map_sep: { + tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + offset, &buf, &bufLen); + switch (tok) { + case yajl_tok_colon: + yajl_bs_set(hand->stateStack, yajl_state_map_need_val); + goto around_again; + case yajl_tok_eof: + return yajl_status_ok; + case yajl_tok_error: + yajl_bs_set(hand->stateStack, yajl_state_lexical_error); + goto around_again; + default: + yajl_bs_set(hand->stateStack, yajl_state_parse_error); + hand->parseError = "object key and value must " + "be separated by a colon (':')"; + goto around_again; + } + } + case yajl_state_map_got_val: { + tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + offset, &buf, &bufLen); + switch (tok) { + case yajl_tok_right_bracket: + if (hand->callbacks && hand->callbacks->yajl_end_map) { + _CC_CHK(hand->callbacks->yajl_end_map(hand->ctx)); + } + yajl_bs_pop(hand->stateStack); + goto around_again; + case yajl_tok_comma: + yajl_bs_set(hand->stateStack, yajl_state_map_need_key); + goto around_again; + case yajl_tok_eof: + return yajl_status_ok; + case yajl_tok_error: + yajl_bs_set(hand->stateStack, yajl_state_lexical_error); + goto around_again; + default: + yajl_bs_set(hand->stateStack, yajl_state_parse_error); + hand->parseError = "after key and value, inside map, " + "I expect ',' or '}'"; + /* try to restore error offset */ + if (*offset >= bufLen) *offset -= bufLen; + else *offset = 0; + goto around_again; + } + } + case yajl_state_array_got_val: { + tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + offset, &buf, &bufLen); + switch (tok) { + case yajl_tok_right_brace: + if (hand->callbacks && hand->callbacks->yajl_end_array) { + _CC_CHK(hand->callbacks->yajl_end_array(hand->ctx)); + } + yajl_bs_pop(hand->stateStack); + goto around_again; + case yajl_tok_comma: + yajl_bs_set(hand->stateStack, yajl_state_array_need_val); + goto around_again; + case yajl_tok_eof: + return yajl_status_ok; + case yajl_tok_error: + yajl_bs_set(hand->stateStack, yajl_state_lexical_error); + goto around_again; + default: + yajl_bs_set(hand->stateStack, yajl_state_parse_error); + hand->parseError = + "after array element, I expect ',' or ']'"; + goto around_again; + } + } + } + + abort(); + return yajl_status_error; +} + diff --git a/src/yajl/yajl_parser.h b/src/yajl/yajl_parser.h new file mode 100644 index 0000000..c79299a --- /dev/null +++ b/src/yajl/yajl_parser.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __YAJL_PARSER_H__ +#define __YAJL_PARSER_H__ + +#include "api/yajl_parse.h" +#include "yajl_bytestack.h" +#include "yajl_buf.h" +#include "yajl_lex.h" + + +typedef enum { + yajl_state_start = 0, + yajl_state_parse_complete, + yajl_state_parse_error, + yajl_state_lexical_error, + yajl_state_map_start, + yajl_state_map_sep, + yajl_state_map_need_val, + yajl_state_map_got_val, + yajl_state_map_need_key, + yajl_state_array_start, + yajl_state_array_got_val, + yajl_state_array_need_val, + yajl_state_got_value, +} yajl_state; + +struct yajl_handle_t { + const yajl_callbacks * callbacks; + void * ctx; + yajl_lexer lexer; + const char * parseError; + /* the number of bytes consumed from the last client buffer, + * in the case of an error this will be an error offset, in the + * case of an error this can be used as the error offset */ + size_t bytesConsumed; + /* temporary storage for decoded strings */ + yajl_buf decodeBuf; + /* a stack of states. access with yajl_state_XXX routines */ + yajl_bytestack stateStack; + /* memory allocation routines */ + yajl_alloc_funcs alloc; + /* bitfield */ + unsigned int flags; +}; + +yajl_status +yajl_do_parse(yajl_handle handle, const unsigned char * jsonText, + size_t jsonTextLen); + +yajl_status +yajl_do_finish(yajl_handle handle); + +unsigned char * +yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText, + size_t jsonTextLen, int verbose); + +/* A little built in integer parsing routine with the same semantics as strtol + * that's unaffected by LOCALE. */ +long long +yajl_parse_integer(const unsigned char *number, unsigned int length); + + +#endif diff --git a/src/yajl/yajl_tree.c b/src/yajl/yajl_tree.c new file mode 100644 index 0000000..3d357a3 --- /dev/null +++ b/src/yajl/yajl_tree.c @@ -0,0 +1,503 @@ +/* + * Copyright (c) 2010-2011 Florian Forster <ff at octo.it> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> +#include <assert.h> + +#include "api/yajl_tree.h" +#include "api/yajl_parse.h" + +#include "yajl_parser.h" + +#if defined(_WIN32) || defined(WIN32) +#define snprintf sprintf_s +#endif + +#define STATUS_CONTINUE 1 +#define STATUS_ABORT 0 + +struct stack_elem_s; +typedef struct stack_elem_s stack_elem_t; +struct stack_elem_s +{ + char * key; + yajl_val value; + stack_elem_t *next; +}; + +struct context_s +{ + stack_elem_t *stack; + yajl_val root; + char *errbuf; + size_t errbuf_size; +}; +typedef struct context_s context_t; + +#define RETURN_ERROR(ctx,retval,...) { \ + if ((ctx)->errbuf != NULL) \ + snprintf ((ctx)->errbuf, (ctx)->errbuf_size, __VA_ARGS__); \ + return (retval); \ + } + +static yajl_val value_alloc (yajl_type type) +{ + yajl_val v; + + v = malloc (sizeof (*v)); + if (v == NULL) return (NULL); + memset (v, 0, sizeof (*v)); + v->type = type; + + return (v); +} + +static void yajl_object_free (yajl_val v) +{ + size_t i; + + if (!YAJL_IS_OBJECT(v)) return; + + for (i = 0; i < v->u.object.len; i++) + { + free((char *) v->u.object.keys[i]); + v->u.object.keys[i] = NULL; + yajl_tree_free (v->u.object.values[i]); + v->u.object.values[i] = NULL; + } + + free((void*) v->u.object.keys); + free(v->u.object.values); + free(v); +} + +static void yajl_array_free (yajl_val v) +{ + size_t i; + + if (!YAJL_IS_ARRAY(v)) return; + + for (i = 0; i < v->u.array.len; i++) + { + yajl_tree_free (v->u.array.values[i]); + v->u.array.values[i] = NULL; + } + + free(v->u.array.values); + free(v); +} + +/* + * Parsing nested objects and arrays is implemented using a stack. When a new + * object or array starts (a curly or a square opening bracket is read), an + * appropriate value is pushed on the stack. When the end of the object is + * reached (an appropriate closing bracket has been read), the value is popped + * off the stack and added to the enclosing object using "context_add_value". + */ +static int context_push(context_t *ctx, yajl_val v) +{ + stack_elem_t *stack; + + stack = malloc (sizeof (*stack)); + if (stack == NULL) + RETURN_ERROR (ctx, ENOMEM, "Out of memory"); + memset (stack, 0, sizeof (*stack)); + + assert ((ctx->stack == NULL) + || YAJL_IS_OBJECT (v) + || YAJL_IS_ARRAY (v)); + + stack->value = v; + stack->next = ctx->stack; + ctx->stack = stack; + + return (0); +} + +static yajl_val context_pop(context_t *ctx) +{ + stack_elem_t *stack; + yajl_val v; + + if (ctx->stack == NULL) + RETURN_ERROR (ctx, NULL, "context_pop: " + "Bottom of stack reached prematurely"); + + stack = ctx->stack; + ctx->stack = stack->next; + + v = stack->value; + + free (stack); + + return (v); +} + +static int object_add_keyval(context_t *ctx, + yajl_val obj, char *key, yajl_val value) +{ + const char **tmpk; + yajl_val *tmpv; + + /* We're checking for NULL in "context_add_value" or its callers. */ + assert (ctx != NULL); + assert (obj != NULL); + assert (key != NULL); + assert (value != NULL); + + /* We're assuring that "obj" is an object in "context_add_value". */ + assert(YAJL_IS_OBJECT(obj)); + + tmpk = realloc((void *) obj->u.object.keys, sizeof(*(obj->u.object.keys)) * (obj->u.object.len + 1)); + if (tmpk == NULL) + RETURN_ERROR(ctx, ENOMEM, "Out of memory"); + obj->u.object.keys = tmpk; + + tmpv = realloc(obj->u.object.values, sizeof (*obj->u.object.values) * (obj->u.object.len + 1)); + if (tmpv == NULL) + RETURN_ERROR(ctx, ENOMEM, "Out of memory"); + obj->u.object.values = tmpv; + + obj->u.object.keys[obj->u.object.len] = key; + obj->u.object.values[obj->u.object.len] = value; + obj->u.object.len++; + + return (0); +} + +static int array_add_value (context_t *ctx, + yajl_val array, yajl_val value) +{ + yajl_val *tmp; + + /* We're checking for NULL pointers in "context_add_value" or its + * callers. */ + assert (ctx != NULL); + assert (array != NULL); + assert (value != NULL); + + /* "context_add_value" will only call us with array values. */ + assert(YAJL_IS_ARRAY(array)); + + tmp = realloc(array->u.array.values, + sizeof(*(array->u.array.values)) * (array->u.array.len + 1)); + if (tmp == NULL) + RETURN_ERROR(ctx, ENOMEM, "Out of memory"); + array->u.array.values = tmp; + array->u.array.values[array->u.array.len] = value; + array->u.array.len++; + + return 0; +} + +/* + * Add a value to the value on top of the stack or the "root" member in the + * context if the end of the parsing process is reached. + */ +static int context_add_value (context_t *ctx, yajl_val v) +{ + /* We're checking for NULL values in all the calling functions. */ + assert (ctx != NULL); + assert (v != NULL); + + /* + * There are three valid states in which this function may be called: + * - There is no value on the stack => This is the only value. This is the + * last step done when parsing a document. We assign the value to the + * "root" member and return. + * - The value on the stack is an object. In this case store the key on the + * stack or, if the key has already been read, add key and value to the + * object. + * - The value on the stack is an array. In this case simply add the value + * and return. + */ + if (ctx->stack == NULL) + { + assert (ctx->root == NULL); + ctx->root = v; + return (0); + } + else if (YAJL_IS_OBJECT (ctx->stack->value)) + { + if (ctx->stack->key == NULL) + { + if (!YAJL_IS_STRING (v)) + RETURN_ERROR (ctx, EINVAL, "context_add_value: " + "Object key is not a string (%#04x)", + v->type); + + ctx->stack->key = v->u.string; + v->u.string = NULL; + free(v); + return (0); + } + else /* if (ctx->key != NULL) */ + { + char * key; + + key = ctx->stack->key; + ctx->stack->key = NULL; + return (object_add_keyval (ctx, ctx->stack->value, key, v)); + } + } + else if (YAJL_IS_ARRAY (ctx->stack->value)) + { + return (array_add_value (ctx, ctx->stack->value, v)); + } + else + { + RETURN_ERROR (ctx, EINVAL, "context_add_value: Cannot add value to " + "a value of type %#04x (not a composite type)", + ctx->stack->value->type); + } +} + +static int handle_string (void *ctx, + const unsigned char *string, size_t string_length) +{ + yajl_val v; + + v = value_alloc (yajl_t_string); + if (v == NULL) + RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory"); + + v->u.string = malloc (string_length + 1); + if (v->u.string == NULL) + { + free (v); + RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory"); + } + memcpy(v->u.string, string, string_length); + v->u.string[string_length] = 0; + + return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT); +} + +static int handle_number (void *ctx, const char *string, size_t string_length) +{ + yajl_val v; + char *endptr; + + v = value_alloc(yajl_t_number); + if (v == NULL) + RETURN_ERROR((context_t *) ctx, STATUS_ABORT, "Out of memory"); + + v->u.number.r = malloc(string_length + 1); + if (v->u.number.r == NULL) + { + free(v); + RETURN_ERROR((context_t *) ctx, STATUS_ABORT, "Out of memory"); + } + memcpy(v->u.number.r, string, string_length); + v->u.number.r[string_length] = 0; + + v->u.number.flags = 0; + + errno = 0; + v->u.number.i = yajl_parse_integer((const unsigned char *) v->u.number.r, + strlen(v->u.number.r)); + if (errno == 0) + v->u.number.flags |= YAJL_NUMBER_INT_VALID; + + endptr = NULL; + errno = 0; + v->u.number.d = strtod(v->u.number.r, &endptr); + if ((errno == 0) && (endptr != NULL) && (*endptr == 0)) + v->u.number.flags |= YAJL_NUMBER_DOUBLE_VALID; + + return ((context_add_value(ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT); +} + +static int handle_start_map (void *ctx) +{ + yajl_val v; + + v = value_alloc(yajl_t_object); + if (v == NULL) + RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory"); + + v->u.object.keys = NULL; + v->u.object.values = NULL; + v->u.object.len = 0; + + return ((context_push (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT); +} + +static int handle_end_map (void *ctx) +{ + yajl_val v; + + v = context_pop (ctx); + if (v == NULL) + return (STATUS_ABORT); + + return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT); +} + +static int handle_start_array (void *ctx) +{ + yajl_val v; + + v = value_alloc(yajl_t_array); + if (v == NULL) + RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory"); + + v->u.array.values = NULL; + v->u.array.len = 0; + + return ((context_push (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT); +} + +static int handle_end_array (void *ctx) +{ + yajl_val v; + + v = context_pop (ctx); + if (v == NULL) + return (STATUS_ABORT); + + return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT); +} + +static int handle_boolean (void *ctx, int boolean_value) +{ + yajl_val v; + + v = value_alloc (boolean_value ? yajl_t_true : yajl_t_false); + if (v == NULL) + RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory"); + + return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT); +} + +static int handle_null (void *ctx) +{ + yajl_val v; + + v = value_alloc (yajl_t_null); + if (v == NULL) + RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory"); + + return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT); +} + +/* + * Public functions + */ +yajl_val yajl_tree_parse (const char *input, + char *error_buffer, size_t error_buffer_size) +{ + static const yajl_callbacks callbacks = + { + /* null = */ handle_null, + /* boolean = */ handle_boolean, + /* integer = */ NULL, + /* double = */ NULL, + /* number = */ handle_number, + /* string = */ handle_string, + /* start map = */ handle_start_map, + /* map key = */ handle_string, + /* end map = */ handle_end_map, + /* start array = */ handle_start_array, + /* end array = */ handle_end_array + }; + + yajl_handle handle; + yajl_status status; + char * internal_err_str; + context_t ctx = { NULL, NULL, NULL, 0 }; + + ctx.errbuf = error_buffer; + ctx.errbuf_size = error_buffer_size; + + if (error_buffer != NULL) + memset (error_buffer, 0, error_buffer_size); + + handle = yajl_alloc (&callbacks, NULL, &ctx); + yajl_config(handle, yajl_allow_comments, 1); + + status = yajl_parse(handle, + (unsigned char *) input, + strlen (input)); + status = yajl_complete_parse (handle); + if (status != yajl_status_ok) { + if (error_buffer != NULL && error_buffer_size > 0) { + internal_err_str = (char *) yajl_get_error(handle, 1, + (const unsigned char *) input, + strlen(input)); + snprintf(error_buffer, error_buffer_size, "%s", internal_err_str); + YA_FREE(&(handle->alloc), internal_err_str); + } + yajl_free (handle); + return NULL; + } + + yajl_free (handle); + return (ctx.root); +} + +yajl_val yajl_tree_get(yajl_val n, const char ** path, yajl_type type) +{ + if (!path) return NULL; + while (n && *path) { + size_t i; + size_t len; + + if (n->type != yajl_t_object) return NULL; + len = n->u.object.len; + for (i = 0; i < len; i++) { + if (!strcmp(*path, n->u.object.keys[i])) { + n = n->u.object.values[i]; + break; + } + } + if (i == len) return NULL; + path++; + } + if (n && type != yajl_t_any && type != n->type) n = NULL; + return n; +} + +void yajl_tree_free (yajl_val v) +{ + if (v == NULL) return; + + if (YAJL_IS_STRING(v)) + { + free(v->u.string); + free(v); + } + else if (YAJL_IS_NUMBER(v)) + { + free(v->u.number.r); + free(v); + } + else if (YAJL_GET_OBJECT(v)) + { + yajl_object_free(v); + } + else if (YAJL_GET_ARRAY(v)) + { + yajl_array_free(v); + } + else /* if (yajl_t_true or yajl_t_false or yajl_t_null) */ + { + free(v); + } +} diff --git a/src/yajl/yajl_version.c b/src/yajl/yajl_version.c new file mode 100644 index 0000000..0671da7 --- /dev/null +++ b/src/yajl/yajl_version.c @@ -0,0 +1,7 @@ +#include <yajl/yajl_version.h> + +int yajl_version(void) +{ + return YAJL_VERSION; +} + diff --git a/src/yajl/yajl_version.h b/src/yajl/yajl_version.h new file mode 100644 index 0000000..0fba9b8 --- /dev/null +++ b/src/yajl/yajl_version.h @@ -0,0 +1,23 @@ +#ifndef YAJL_VERSION_H_ +#define YAJL_VERSION_H_ + +#include <yajl/yajl_common.h> + +#define YAJL_MAJOR 2 +#define YAJL_MINOR 0 +#define YAJL_MICRO 1 + +#define YAJL_VERSION ((YAJL_MAJOR * 10000) + (YAJL_MINOR * 100) + YAJL_MICRO) + +#ifdef __cplusplus +extern "C" { +#endif + +extern int YAJL_API yajl_version(void); + +#ifdef __cplusplus +} +#endif + +#endif /* YAJL_VERSION_H_ */ + diff --git a/src/yajlpp/CMakeLists.txt b/src/yajlpp/CMakeLists.txt new file mode 100644 index 0000000..73168ba --- /dev/null +++ b/src/yajlpp/CMakeLists.txt @@ -0,0 +1,27 @@ +add_library( + yajlpp STATIC + ../config.h.in + json_op.hh + json_ptr.hh + yajlpp.hh + yajlpp_def.hh + + json_op.cc + json_ptr.cc + yajlpp.cc +) + +target_include_directories(yajlpp PUBLIC . .. ../fmtlib + ${CMAKE_CURRENT_BINARY_DIR}/..) +target_link_libraries(yajlpp pcrepp yajl ncurses::libcurses) + +add_executable(test_yajlpp test_yajlpp.cc) +target_link_libraries(test_yajlpp yajlpp base ${lnav_LIBS}) +add_test(NAME test_yajlpp COMMAND test_yajlpp) + +add_executable(test_json_ptr test_json_ptr.cc) +target_link_libraries(test_json_ptr yajlpp base ${lnav_LIBS}) +add_test(NAME test_json_ptr COMMAND test_json_ptr) + +add_executable(drive_json_op drive_json_op.cc) +target_link_libraries(drive_json_op base yajlpp ${lnav_LIBS}) diff --git a/src/yajlpp/Makefile.am b/src/yajlpp/Makefile.am new file mode 100644 index 0000000..cc6bcf6 --- /dev/null +++ b/src/yajlpp/Makefile.am @@ -0,0 +1,81 @@ + +include $(top_srcdir)/aminclude_static.am + +TESTS_ENVIRONMENT = $(SHELL) $(top_builddir)/TESTS_ENVIRONMENT +LOG_COMPILER = $(SHELL) $(top_builddir)/TESTS_ENVIRONMENT + +AM_CPPFLAGS = \ + $(CODE_COVERAGE_CPPFLAGS) \ + -Wall \ + $(LIBARCHIVE_CFLAGS) \ + $(PCRE_CFLAGS) \ + -I$(top_srcdir)/src/ \ + -I$(top_srcdir)/src/fmtlib \ + -I$(top_srcdir)/src/third-party/scnlib/include + +AM_LDFLAGS = \ + $(LIBARCHIVE_LDFLAGS) \ + $(STATIC_LDFLAGS) + +AM_LIBS = $(CODE_COVERAGE_LIBS) +AM_CFLAGS = $(CODE_COVERAGE_CFLAGS) +AM_CXXFLAGS = $(CODE_COVERAGE_CXXFLAGS) + +noinst_LIBRARIES = libyajlpp.a + +noinst_HEADERS = \ + json_op.hh \ + json_ptr.hh \ + yajlpp.hh \ + yajlpp_def.hh + +libyajlpp_a_SOURCES = \ + json_op.cc \ + json_ptr.cc \ + yajlpp.cc + +check_PROGRAMS = \ + drive_json_op \ + drive_json_ptr_walk \ + test_json_ptr \ + test_yajlpp + +drive_json_op_SOURCES = drive_json_op.cc + +drive_json_ptr_walk_SOURCES = drive_json_ptr_walk.cc + +test_json_ptr_SOURCES = test_json_ptr.cc + +test_yajlpp_SOURCES = test_yajlpp.cc + +LDADD = \ + $(LIBARCHIVE_LIBS) \ + libyajlpp.a \ + $(top_builddir)/src/base/libbase.a \ + $(top_builddir)/src/fmtlib/libcppfmt.a \ + $(top_builddir)/src/third-party/scnlib/src/libscnlib.a \ + $(top_builddir)/src/pcrepp/libpcrepp.a \ + $(top_builddir)/src/yajl/libyajl.a + +dist_noinst_SCRIPTS = \ + test_json_op.sh \ + test_json_ptr_walk.sh + +TESTS = \ + test_json_op.sh \ + test_json_ptr \ + test_json_ptr_walk.sh \ + test_yajlpp + +DISTCLEANFILES = \ + *.dat \ + *.err \ + *.db \ + *.dpt \ + *.diff \ + *.index \ + *.tmp \ + *.errbak \ + *.tmpbak \ + *.gz \ + *.bz2 diff --git a/src/yajlpp/README.md b/src/yajlpp/README.md new file mode 100644 index 0000000..3a7650c --- /dev/null +++ b/src/yajlpp/README.md @@ -0,0 +1,5 @@ +# YAJLPP + +This directory contains a C++ adapter for the YAJL JSON library. The adapter +provides a usage model that is based around client code mapping internal data +structures to a JSON schema. diff --git a/src/yajlpp/drive_json_op.cc b/src/yajlpp/drive_json_op.cc new file mode 100644 index 0000000..51ef161 --- /dev/null +++ b/src/yajlpp/drive_json_op.cc @@ -0,0 +1,210 @@ +/** + * Copyright (c) 2014, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY TIMOTHY STACK AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file drive_json_op.cc + */ + +#include <errno.h> +#include <stdlib.h> + +#include "base/lnav_log.hh" +#include "config.h" +#include "yajl/api/yajl_gen.h" +#include "yajlpp/json_op.hh" + +static void +printer(void* ctx, const char* numberVal, size_t numberLen) +{ + log_perror(write(STDOUT_FILENO, numberVal, numberLen)); +} + +static int +handle_start_map(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_map_open(gen); + + return 1; +} + +static int +handle_map_key(void* ctx, const unsigned char* key, size_t len) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_string(gen, key, len); + + return 1; +} + +static int +handle_end_map(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_map_close(gen); + + return 1; +} + +static int +handle_null(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_null(gen); + + return 1; +} + +static int +handle_boolean(void* ctx, int boolVal) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_bool(gen, boolVal); + + return 1; +} + +static int +handle_number(void* ctx, const char* numberVal, size_t numberLen) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_number(gen, numberVal, numberLen); + + return 1; +} + +static int +handle_string(void* ctx, const unsigned char* stringVal, size_t len) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_string(gen, stringVal, len); + + return 1; +} + +static int +handle_start_array(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_array_open(gen); + + return 1; +} + +static int +handle_end_array(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + yajl_gen_array_close(gen); + + return 1; +} + +int +main(int argc, char* argv[]) +{ + int retval = EXIT_SUCCESS; + + log_argv(argc, argv); + + if (argc != 3) { + fprintf(stderr, "error: expecting operation and json-pointer\n"); + retval = EXIT_FAILURE; + } else if (strcmp(argv[1], "get") == 0) { + unsigned char buffer[1024]; + json_ptr jptr(argv[2]); + json_op jo(jptr); + yajl_handle handle; + yajl_status status; + yajl_gen gen; + ssize_t rc; + + gen = yajl_gen_alloc(nullptr); + yajl_gen_config(gen, yajl_gen_print_callback, printer, nullptr); + yajl_gen_config(gen, yajl_gen_beautify, true); + + jo.jo_ptr_callbacks.yajl_start_map = handle_start_map; + jo.jo_ptr_callbacks.yajl_map_key = handle_map_key; + jo.jo_ptr_callbacks.yajl_end_map = handle_end_map; + jo.jo_ptr_callbacks.yajl_start_array = handle_start_array; + jo.jo_ptr_callbacks.yajl_end_array = handle_end_array; + jo.jo_ptr_callbacks.yajl_null = handle_null; + jo.jo_ptr_callbacks.yajl_boolean = handle_boolean; + jo.jo_ptr_callbacks.yajl_number = handle_number; + jo.jo_ptr_callbacks.yajl_string = handle_string; + jo.jo_ptr_data = gen; + + handle = yajl_alloc(&json_op::ptr_callbacks, nullptr, &jo); + while ((rc = read(STDIN_FILENO, buffer, sizeof(buffer))) > 0) { + status = yajl_parse(handle, buffer, rc); + if (status == yajl_status_error) { + auto msg = yajl_get_error(handle, 1, buffer, rc); + + fprintf(stderr, "error:cannot parse JSON input -- %s\n", msg); + retval = EXIT_FAILURE; + yajl_free_error(handle, msg); + break; + } else if (status == yajl_status_client_canceled) { + fprintf(stderr, "client cancel\n"); + break; + } + } + status = yajl_complete_parse(handle); + if (status == yajl_status_error) { + auto msg = yajl_get_error(handle, 1, buffer, rc); + fprintf(stderr, "error:cannot parse JSON input -- %s\n", msg); + yajl_free_error(handle, msg); + retval = EXIT_FAILURE; + } else if (status == yajl_status_client_canceled) { + fprintf(stderr, "client cancel\n"); + } + yajl_free(handle); + } else { + fprintf(stderr, "error: unknown operation -- %s\n", argv[1]); + retval = EXIT_FAILURE; + } + + return retval; +} diff --git a/src/yajlpp/drive_json_ptr_walk.cc b/src/yajlpp/drive_json_ptr_walk.cc new file mode 100644 index 0000000..c6a8f72 --- /dev/null +++ b/src/yajlpp/drive_json_ptr_walk.cc @@ -0,0 +1,103 @@ +/** + * Copyright (c) 2014, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY TIMOTHY STACK AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file drive_json_ptr_dump.cc + */ + +#include <iostream> + +#include <stdlib.h> + +#include "base/lnav_log.hh" +#include "config.h" +#include "json_op.hh" +#include "json_ptr.hh" +#include "yajl/api/yajl_gen.h" +#include "yajlpp.hh" + +int +main(int argc, char* argv[]) +{ + int retval = EXIT_SUCCESS; + yajl_status status; + json_ptr_walk jpw; + + log_argv(argc, argv); + + std::string json_input(std::istreambuf_iterator<char>(std::cin), {}); + + status = jpw.parse(json_input.c_str(), json_input.size()); + if (status == yajl_status_error) { + fprintf(stderr, + "error:cannot parse JSON input -- %s\n", + jpw.jpw_error_msg.c_str()); + return EXIT_FAILURE; + } + + if (status == yajl_status_client_canceled) { + fprintf(stderr, "client cancel\n"); + } + + status = jpw.complete_parse(); + if (status == yajl_status_error) { + fprintf(stderr, + "error:cannot parse JSON input -- %s\n", + jpw.jpw_error_msg.c_str()); + return EXIT_FAILURE; + } else if (status == yajl_status_client_canceled) { + fprintf(stderr, "client cancel\n"); + } + + for (json_ptr_walk::walk_list_t::iterator iter = jpw.jpw_values.begin(); + iter != jpw.jpw_values.end(); + ++iter) + { + printf("%s = %s\n", iter->wt_ptr.c_str(), iter->wt_value.c_str()); + + { + auto_mem<yajl_handle_t> parse_handle(yajl_free); + json_ptr jp(iter->wt_ptr.c_str()); + json_op jo(jp); + yajlpp_gen gen; + + jo.jo_ptr_callbacks = json_op::gen_callbacks; + jo.jo_ptr_data = gen.get_handle(); + parse_handle.reset( + yajl_alloc(&json_op::ptr_callbacks, nullptr, &jo)); + + yajl_parse(parse_handle.in(), + (const unsigned char*) json_input.c_str(), + json_input.size()); + yajl_complete_parse(parse_handle.in()); + + assert(iter->wt_value == gen.to_string_fragment().to_string()); + } + } + + return retval; +} diff --git a/src/yajlpp/json_op.cc b/src/yajlpp/json_op.cc new file mode 100644 index 0000000..046f223 --- /dev/null +++ b/src/yajlpp/json_op.cc @@ -0,0 +1,316 @@ +/** + * Copyright (c) 2014, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY TIMOTHY STACK AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file json_op.cc + */ + +#include "json_op.hh" + +#include "base/lnav_log.hh" +#include "config.h" +#include "yajl/api/yajl_gen.h" + +static int +gen_handle_start_map(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_map_open(gen); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +static int +gen_handle_map_key(void* ctx, const unsigned char* key, size_t len) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_string(gen, key, len); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +static int +gen_handle_end_map(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_map_close(gen); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +static int +gen_handle_null(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_null(gen); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +static int +gen_handle_boolean(void* ctx, int boolVal) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_bool(gen, boolVal); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +static int +gen_handle_number(void* ctx, const char* numberVal, size_t numberLen) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_number(gen, numberVal, numberLen); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +static int +gen_handle_string(void* ctx, const unsigned char* stringVal, size_t len) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_string(gen, stringVal, len); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +static int +gen_handle_start_array(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_array_open(gen); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +static int +gen_handle_end_array(void* ctx) +{ + json_op* jo = (json_op*) ctx; + yajl_gen gen = (yajl_gen) jo->jo_ptr_data; + + jo->jo_ptr_error_code = yajl_gen_array_close(gen); + + return jo->jo_ptr_error_code == yajl_gen_status_ok; +} + +const yajl_callbacks json_op::gen_callbacks = { + gen_handle_null, + gen_handle_boolean, + nullptr, + nullptr, + gen_handle_number, + gen_handle_string, + gen_handle_start_map, + gen_handle_map_key, + gen_handle_end_map, + gen_handle_start_array, + gen_handle_end_array, +}; + +const yajl_callbacks json_op::ptr_callbacks = { + handle_null, + handle_boolean, + nullptr, + nullptr, + handle_number, + handle_string, + handle_start_map, + handle_map_key, + handle_end_map, + handle_start_array, + handle_end_array, +}; + +int +json_op::handle_null(void* ctx) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index()) { + if (jo->jo_ptr_callbacks.yajl_null != nullptr) { + retval = jo->jo_ptr_callbacks.yajl_null(ctx); + } + } + + return retval; +} + +int +json_op::handle_boolean(void* ctx, int boolVal) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index()) { + if (jo->jo_ptr_callbacks.yajl_boolean != nullptr) { + retval = jo->jo_ptr_callbacks.yajl_boolean(ctx, boolVal); + } + } + + return retval; +} + +int +json_op::handle_number(void* ctx, const char* numberVal, size_t numberLen) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index()) { + if (jo->jo_ptr_callbacks.yajl_number != nullptr) { + retval + = jo->jo_ptr_callbacks.yajl_number(ctx, numberVal, numberLen); + } + } + + return retval; +} + +int +json_op::handle_string(void* ctx, + const unsigned char* stringVal, + size_t stringLen) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index()) { + if (jo->jo_ptr_callbacks.yajl_string != nullptr) { + retval + = jo->jo_ptr_callbacks.yajl_string(ctx, stringVal, stringLen); + } + } + + return retval; +} + +int +json_op::handle_start_map(void* ctx) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index(false)) { + if (jo->jo_ptr_callbacks.yajl_start_map != nullptr) { + retval = jo->jo_ptr_callbacks.yajl_start_map(ctx); + } + } + + if (!jo->jo_ptr.expect_map(jo->jo_depth, jo->jo_array_index)) { + retval = 0; + } + + return retval; +} + +int +json_op::handle_map_key(void* ctx, const unsigned char* key, size_t len) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index(false)) { + if (jo->jo_ptr_callbacks.yajl_map_key != nullptr) { + retval = jo->jo_ptr_callbacks.yajl_map_key(ctx, key, len); + } + } + + if (!jo->jo_ptr.at_key(jo->jo_depth, (const char*) key, len)) { + retval = 0; + } + + return retval; +} + +int +json_op::handle_end_map(void* ctx) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index()) { + if (jo->jo_ptr_callbacks.yajl_end_map != nullptr) { + retval = jo->jo_ptr_callbacks.yajl_end_map(ctx); + } + } + + jo->jo_ptr.exit_container(jo->jo_depth, jo->jo_array_index); + + return retval; +} + +int +json_op::handle_start_array(void* ctx) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index(false)) { + if (jo->jo_ptr_callbacks.yajl_start_array != nullptr) { + retval = jo->jo_ptr_callbacks.yajl_start_array(ctx); + } + } + + jo->jo_ptr.expect_array(jo->jo_depth, jo->jo_array_index); + + return retval; +} + +int +json_op::handle_end_array(void* ctx) +{ + json_op* jo = (json_op*) ctx; + int retval = 1; + + if (jo->check_index()) { + if (jo->jo_ptr_callbacks.yajl_end_array != nullptr) { + retval = jo->jo_ptr_callbacks.yajl_end_array(ctx); + } + } + + jo->jo_ptr.exit_container(jo->jo_depth, jo->jo_array_index); + + return retval; +} diff --git a/src/yajlpp/json_op.hh b/src/yajlpp/json_op.hh new file mode 100644 index 0000000..4f3399b --- /dev/null +++ b/src/yajlpp/json_op.hh @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2014, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY TIMOTHY STACK AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file json_op.hh + */ + +#ifndef json_op_hh +#define json_op_hh + +#include <string> + +#include <sys/types.h> + +#include "yajl/api/yajl_parse.h" +#include "yajlpp/json_ptr.hh" + +class json_op { + static int handle_null(void* ctx); + static int handle_boolean(void* ctx, int boolVal); + static int handle_number(void* ctx, + const char* numberVal, + size_t numberLen); + static int handle_string(void* ctx, + const unsigned char* stringVal, + size_t len); + static int handle_start_map(void* ctx); + static int handle_map_key(void* ctx, const unsigned char* key, size_t len); + static int handle_end_map(void* ctx); + static int handle_start_array(void* ctx); + static int handle_end_array(void* ctx); + +public: + const static yajl_callbacks gen_callbacks; + const static yajl_callbacks ptr_callbacks; + + explicit json_op(const json_ptr& ptr) + : jo_ptr(ptr), jo_ptr_callbacks(gen_callbacks) + { + } + + bool check_index(bool primitive = true) + { + return this->jo_ptr.at_index( + this->jo_depth, this->jo_array_index, primitive); + } + + int jo_depth{0}; + int jo_array_index{-1}; + + json_ptr jo_ptr; + yajl_callbacks jo_ptr_callbacks; + void* jo_ptr_data{nullptr}; + std::string jo_ptr_error; + int jo_ptr_error_code{0}; +}; + +#endif diff --git a/src/yajlpp/json_ptr.cc b/src/yajlpp/json_ptr.cc new file mode 100644 index 0000000..4cc0273 --- /dev/null +++ b/src/yajlpp/json_ptr.cc @@ -0,0 +1,521 @@ +/** + * Copyright (c) 2014, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY TIMOTHY STACK AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file json_ptr.cc + */ + +#ifdef __CYGWIN__ +# include <alloca.h> +#endif + +#include "config.h" +#include "fmt/format.h" +#include "yajl/api/yajl_gen.h" +#include "yajlpp/json_ptr.hh" + +static int +handle_null(void* ctx) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + + jpw->jpw_values.emplace_back(jpw->current_ptr(), yajl_t_null, "null"); + jpw->inc_array_index(); + + return 1; +} + +static int +handle_boolean(void* ctx, int boolVal) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + + jpw->jpw_values.emplace_back(jpw->current_ptr(), + boolVal ? yajl_t_true : yajl_t_false, + boolVal ? "true" : "false"); + jpw->inc_array_index(); + + return 1; +} + +static int +handle_number(void* ctx, const char* numberVal, size_t numberLen) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + + jpw->jpw_values.emplace_back( + jpw->current_ptr(), yajl_t_number, std::string(numberVal, numberLen)); + jpw->inc_array_index(); + + return 1; +} + +static void +appender(void* ctx, const char* strVal, size_t strLen) +{ + std::string& str = *(std::string*) ctx; + + str.append(strVal, strLen); +} + +static int +handle_string(void* ctx, const unsigned char* stringVal, size_t len) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + auto_mem<yajl_gen_t> gen(yajl_gen_free); + std::string str; + + gen = yajl_gen_alloc(nullptr); + yajl_gen_config(gen.in(), yajl_gen_print_callback, appender, &str); + yajl_gen_string(gen.in(), stringVal, len); + jpw->jpw_values.emplace_back(jpw->current_ptr(), yajl_t_string, str); + jpw->inc_array_index(); + + return 1; +} + +static int +handle_start_map(void* ctx) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + + jpw->jpw_keys.emplace_back(""); + jpw->jpw_array_indexes.push_back(-1); + + return 1; +} + +static int +handle_map_key(void* ctx, const unsigned char* key, size_t len) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + char partially_encoded_key[len + 32]; + size_t required_len; + + jpw->jpw_keys.pop_back(); + + required_len = json_ptr::encode(partially_encoded_key, + sizeof(partially_encoded_key), + (const char*) key, + len); + if (required_len < sizeof(partially_encoded_key)) { + jpw->jpw_keys.emplace_back(&partially_encoded_key[0], required_len); + } else { + auto fully_encoded_key = (char*) alloca(required_len); + + json_ptr::encode( + fully_encoded_key, required_len, (const char*) key, len); + jpw->jpw_keys.emplace_back(&fully_encoded_key[0], required_len); + } + + return 1; +} + +static int +handle_end_map(void* ctx) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + + jpw->jpw_keys.pop_back(); + jpw->jpw_array_indexes.pop_back(); + + jpw->inc_array_index(); + + return 1; +} + +static int +handle_start_array(void* ctx) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + + jpw->jpw_keys.emplace_back(""); + jpw->jpw_array_indexes.push_back(0); + + return 1; +} + +static int +handle_end_array(void* ctx) +{ + json_ptr_walk* jpw = (json_ptr_walk*) ctx; + + jpw->jpw_keys.pop_back(); + jpw->jpw_array_indexes.pop_back(); + jpw->inc_array_index(); + + return 1; +} + +const yajl_callbacks json_ptr_walk::callbacks = {handle_null, + handle_boolean, + nullptr, + nullptr, + handle_number, + handle_string, + handle_start_map, + handle_map_key, + handle_end_map, + handle_start_array, + handle_end_array}; + +size_t +json_ptr::encode(char* dst, size_t dst_len, const char* src, size_t src_len) +{ + size_t retval = 0; + + if (src_len == (size_t) -1) { + src_len = strlen(src); + } + + for (size_t lpc = 0; lpc < src_len; lpc++) { + switch (src[lpc]) { + case '/': + case '~': + case '#': + if (retval < dst_len) { + dst[retval] = '~'; + retval += 1; + if (src[lpc] == '~') { + dst[retval] = '0'; + } else if (src[lpc] == '#') { + dst[retval] = '2'; + } else { + dst[retval] = '1'; + } + } else { + retval += 1; + } + break; + default: + if (retval < dst_len) { + dst[retval] = src[lpc]; + } + break; + } + retval += 1; + } + if (retval < dst_len) { + dst[retval] = '\0'; + } + + return retval; +} + +size_t +json_ptr::decode(char* dst, const char* src, ssize_t src_len) +{ + size_t retval = 0; + + if (src_len == -1) { + src_len = strlen(src); + } + + for (int lpc = 0; lpc < src_len; lpc++) { + switch (src[lpc]) { + case '~': + if ((lpc + 1) < src_len) { + switch (src[lpc + 1]) { + case '0': + dst[retval++] = '~'; + lpc += 1; + break; + case '1': + dst[retval++] = '/'; + lpc += 1; + break; + case '2': + dst[retval++] = '#'; + lpc += 1; + break; + default: + break; + } + } + break; + default: + dst[retval++] = src[lpc]; + break; + } + } + + dst[retval] = '\0'; + + return retval; +} + +bool +json_ptr::expect_map(int32_t& depth, int32_t& index) +{ + bool retval; + + if (this->jp_state == match_state_t::DONE) { + retval = true; + } else if (depth != this->jp_depth) { + retval = true; + } else if (this->reached_end()) { + retval = true; + } else if (this->jp_state == match_state_t::VALUE + && (this->jp_array_index == -1 + || ((index - 1) == this->jp_array_index))) + { + if (this->jp_pos[0] == '/') { + this->jp_pos += 1; + this->jp_depth += 1; + this->jp_state = match_state_t::VALUE; + this->jp_array_index = -1; + index = -1; + } + retval = true; + } else { + retval = true; + } + depth += 1; + + return retval; +} + +bool +json_ptr::at_key(int32_t depth, const char* component, ssize_t len) +{ + const char* component_end; + int lpc; + + if (this->jp_state == match_state_t::DONE || depth != this->jp_depth) { + return true; + } + + if (len == -1) { + len = strlen(component); + } + component_end = component + len; + + for (lpc = 0; component < component_end; lpc++, component++) { + char ch = this->jp_pos[lpc]; + + if (this->jp_pos[lpc] == '~') { + switch (this->jp_pos[lpc + 1]) { + case '0': + ch = '~'; + break; + case '1': + ch = '/'; + break; + default: + this->jp_state = match_state_t::ERR_INVALID_ESCAPE; + return false; + } + lpc += 1; + } else if (this->jp_pos[lpc] == '/') { + ch = '\0'; + } + + if (ch != *component) { + return true; + } + } + + this->jp_pos += lpc; + this->jp_state = match_state_t::VALUE; + + return true; +} + +void +json_ptr::exit_container(int32_t& depth, int32_t& index) +{ + depth -= 1; + if (this->jp_state == match_state_t::VALUE && depth == this->jp_depth + && (index == -1 || (index - 1 == this->jp_array_index)) + && this->reached_end()) + { + this->jp_state = match_state_t::DONE; + index = -1; + } +} + +bool +json_ptr::expect_array(int32_t& depth, int32_t& index) +{ + bool retval; + + if (this->jp_state == match_state_t::DONE) { + retval = true; + } else if (depth != this->jp_depth) { + retval = true; + } else if (this->reached_end()) { + retval = true; + } else if (this->jp_pos[0] == '/' && index == this->jp_array_index) { + int offset; + + this->jp_depth += 1; + + if (sscanf(this->jp_pos, "/%d%n", &this->jp_array_index, &offset) != 1) + { + this->jp_state = match_state_t::ERR_INVALID_INDEX; + retval = true; + } else if (this->jp_pos[offset] != '\0' && this->jp_pos[offset] != '/') + { + this->jp_state = match_state_t::ERR_INVALID_INDEX; + retval = true; + } else { + index = 0; + this->jp_pos += offset; + this->jp_state = match_state_t::VALUE; + retval = true; + } + } else { + this->jp_state = match_state_t::ERR_NO_SLASH; + retval = true; + } + + depth += 1; + + return retval; +} + +bool +json_ptr::at_index(int32_t& depth, int32_t& index, bool primitive) +{ + bool retval; + + if (this->jp_state == match_state_t::DONE) { + retval = false; + } else if (depth < this->jp_depth) { + retval = false; + } else if (depth == this->jp_depth) { + if (index == -1) { + if (this->jp_array_index == -1) { + retval = this->reached_end(); + if (primitive && retval) { + this->jp_state = match_state_t::DONE; + } + } else { + retval = false; + } + } else if (index == this->jp_array_index) { + retval = this->reached_end(); + this->jp_array_index = -1; + index = -1; + if (primitive && retval) { + this->jp_state = match_state_t::DONE; + } + } else { + index += 1; + retval = false; + } + } else if (index == -1) { + retval = this->reached_end(); + } else { + retval = false; + } + + return retval; +} + +std::string +json_ptr::error_msg() const +{ + switch (this->jp_state) { + case match_state_t::ERR_INVALID_ESCAPE: + return fmt::format(FMT_STRING("invalid escape sequence near -- {}"), + this->jp_pos); + case match_state_t::ERR_INVALID_INDEX: + return fmt::format(FMT_STRING("expecting array index at -- {}"), + this->jp_pos); + case match_state_t::ERR_INVALID_TYPE: + return fmt::format(FMT_STRING("expecting container at -- {}"), + this->jp_pos); + default: + break; + } + + return ""; +} + +std::string +json_ptr_walk::current_ptr() +{ + std::string retval; + + for (size_t lpc = 0; lpc < this->jpw_array_indexes.size(); lpc++) { + retval.append("/"); + if (this->jpw_array_indexes[lpc] == -1) { + retval.append(this->jpw_keys[lpc]); + } else { + fmt::format_to(std::back_inserter(retval), + FMT_STRING("{}"), + this->jpw_array_indexes[lpc]); + } + } + + this->jpw_max_ptr_len = std::max(this->jpw_max_ptr_len, retval.size()); + + return retval; +} + +void +json_ptr_walk::update_error_msg(yajl_status status, + const char* buffer, + ssize_t len) +{ + switch (status) { + case yajl_status_ok: + break; + case yajl_status_client_canceled: + this->jpw_error_msg = "internal error"; + break; + case yajl_status_error: { + auto* msg = yajl_get_error( + this->jpw_handle, 1, (const unsigned char*) buffer, len); + this->jpw_error_msg = std::string((const char*) msg); + + yajl_free_error(this->jpw_handle, msg); + break; + } + } +} + +yajl_status +json_ptr_walk::complete_parse() +{ + yajl_status retval; + + retval = yajl_complete_parse(this->jpw_handle); + this->update_error_msg(retval, nullptr, -1); + return retval; +} + +yajl_status +json_ptr_walk::parse(const char* buffer, ssize_t len) +{ + yajl_status retval; + + retval = yajl_parse(this->jpw_handle, (const unsigned char*) buffer, len); + this->update_error_msg(retval, buffer, len); + return retval; +} diff --git a/src/yajlpp/json_ptr.hh b/src/yajlpp/json_ptr.hh new file mode 100644 index 0000000..f7822ab --- /dev/null +++ b/src/yajlpp/json_ptr.hh @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2014-2019, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY TIMOTHY STACK AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file json_ptr.hh + */ + +#ifndef json_ptr_hh +#define json_ptr_hh + +#include <string> +#include <vector> + +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <sys/types.h> + +#include "base/auto_mem.hh" +#include "yajl/api/yajl_parse.h" +#include "yajl/api/yajl_tree.h" + +class json_ptr_walk { +public: + const static yajl_callbacks callbacks; + + json_ptr_walk() + { + this->jpw_handle = yajl_alloc(&callbacks, nullptr, this); + } + + yajl_status parse(const char* buffer, ssize_t len); + + yajl_status complete_parse(); + + void update_error_msg(yajl_status status, const char* buffer, ssize_t len); + + void clear() { this->jpw_values.clear(); } + + void inc_array_index() + { + if (!this->jpw_array_indexes.empty() + && this->jpw_array_indexes.back() != -1) { + this->jpw_array_indexes.back() += 1; + } + } + + std::string current_ptr(); + + struct walk_triple { + walk_triple(std::string ptr, yajl_type type, std::string value) + : wt_ptr(std::move(ptr)), wt_type(type), wt_value(std::move(value)) + { + } + + std::string wt_ptr; + yajl_type wt_type; + std::string wt_value; + }; + + using walk_list_t = std::vector<walk_triple>; + + auto_mem<yajl_handle_t> jpw_handle{yajl_free}; + std::string jpw_error_msg; + walk_list_t jpw_values; + std::vector<std::string> jpw_keys; + std::vector<int32_t> jpw_array_indexes; + size_t jpw_max_ptr_len{0}; +}; + +class json_ptr { +public: + enum class match_state_t { + DONE, + VALUE, + ERR_INVALID_TYPE, + ERR_NO_SLASH, + ERR_INVALID_ESCAPE, + ERR_INVALID_INDEX, + }; + + static size_t encode(char* dst, + size_t dst_len, + const char* src, + size_t src_len = -1); + + static size_t decode(char* dst, const char* src, ssize_t src_len = -1); + + json_ptr(const char* value) : jp_value(value), jp_pos(value) {} + + bool expect_map(int32_t& depth, int32_t& index); + + bool at_key(int32_t depth, const char* component, ssize_t len = -1); + + void exit_container(int32_t& depth, int32_t& index); + + bool expect_array(int32_t& depth, int32_t& index); + + bool at_index(int32_t& depth, int32_t& index, bool primitive = true); + + bool reached_end() const { return this->jp_pos[0] == '\0'; } + + std::string error_msg() const; + + const char* jp_value; + const char* jp_pos; + int32_t jp_depth{0}; + int32_t jp_array_index{-1}; + match_state_t jp_state{match_state_t::VALUE}; +}; + +#endif diff --git a/src/yajlpp/test_json_op.sh b/src/yajlpp/test_json_op.sh new file mode 100755 index 0000000..91a3849 --- /dev/null +++ b/src/yajlpp/test_json_op.sh @@ -0,0 +1,114 @@ +#! /bin/bash + +run_test ./drive_json_op get "" <<EOF +3 +EOF + +check_output "cannot read root number value" <<EOF +3 +EOF + +run_test ./drive_json_op get "" <<EOF +null +EOF + +check_output "cannot read root null value" <<EOF +null +EOF + +run_test ./drive_json_op get "" <<EOF +true +EOF + +check_output "cannot read root bool value" <<EOF +true +EOF + +run_test ./drive_json_op get "" <<EOF +"str" +EOF + +check_output "cannot read root string value" <<EOF +"str" +EOF + +run_test ./drive_json_op get "" <<EOF +{ "val" : 3, "other" : 2 } +EOF + +check_output "cannot read root map value" <<EOF +{ + "val": 3, + "other": 2 +} +EOF + +run_test ./drive_json_op get /val <<EOF +{ "val" : 3 } +EOF + +check_output "cannot read top-level value" <<EOF +3 +EOF + +run_test ./drive_json_op get /val <<EOF +{ "other" : { "val" : 5 }, "val" : 3 } +EOF + +check_output "read wrong value" <<EOF +3 +EOF + +run_test ./drive_json_op get /other <<EOF +{ "other" : { "val" : 5 }, "val" : 3 } +EOF + +check_output "cannot read map" <<EOF +{ + "val": 5 +} +EOF + +run_test ./drive_json_op get /other/val <<EOF +{ "other" : { "val" : 5 }, "val" : 3 } +EOF + +check_output "cannot read nested map" <<EOF +5 +EOF + + +run_test ./drive_json_op get "" <<EOF +[0, 1] +EOF + +check_output "cannot read root array value" <<EOF +[ + 0, + 1 +] +EOF + +run_test ./drive_json_op get "/6" <<EOF +[null, true, 1, "str", {"sub":[10, 11]}, [21, [33, 34], 66], 2] +EOF + +check_output "cannot read array value" <<EOF +2 +EOF + +run_test ./drive_json_op get "/ID" <<EOF +{"ID":"P1","ProcessID":"P1","Name":"VxWorks","CanSuspend":true,"CanResume":1,"IsContainer":true,"WordSize":4,"CanTerminate":true,"CanDetach":true,"RCGroup":"P1","SymbolsGroup":"P1","CPUGroup":"P1","DiagnosticTestProcess":true} +EOF + +check_output "cannot read key value" <<EOF +"P1" +EOF + +run_test ./drive_json_op get "/arr/1/id" <<EOF +{"arr": [{"id": 1}, {"id": 2}]} +EOF + +check_output "cannot read key value" <<EOF +2 +EOF diff --git a/src/yajlpp/test_json_ptr.cc b/src/yajlpp/test_json_ptr.cc new file mode 100644 index 0000000..0c530c0 --- /dev/null +++ b/src/yajlpp/test_json_ptr.cc @@ -0,0 +1,73 @@ +/** + * Copyright (c) 2014, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY TIMOTHY STACK AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file test_json_ptr.cc + */ + +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> + +#include "config.h" +#include "yajlpp/json_ptr.hh" + +int +main(int argc, const char* argv[]) +{ + int32_t depth, index; + + { + json_ptr jptr(""); + + depth = 0; + index = -1; + assert(jptr.at_index(depth, index)); + } + + { + json_ptr jptr("/"); + + depth = 0; + index = -1; + assert(!jptr.at_index(depth, index)); + assert(jptr.expect_map(depth, index)); + assert(jptr.at_index(depth, index)); + } + + { + json_ptr jptr("/foo/bar"); + + depth = 0; + index = -1; + assert(jptr.expect_map(depth, index)); + assert(jptr.at_key(depth, "foo")); + assert(jptr.expect_map(depth, index)); + assert(jptr.at_key(depth, "bar")); + assert(jptr.at_index(depth, index)); + } +} diff --git a/src/yajlpp/test_json_ptr_walk.sh b/src/yajlpp/test_json_ptr_walk.sh new file mode 100644 index 0000000..bcaf1ec --- /dev/null +++ b/src/yajlpp/test_json_ptr_walk.sh @@ -0,0 +1,77 @@ +#! /bin/bash + +run_test ./drive_json_ptr_walk <<EOF +{ "foo" : 1 } +EOF + +check_output "simple object" <<EOF +/foo = 1 +EOF + +run_test ./drive_json_ptr_walk <<EOF +{ "~tstack/julia" : 1 } +EOF + +check_output "escaped object" <<EOF +/~0tstack~1julia = 1 +EOF + +run_test ./drive_json_ptr_walk <<EOF +1 +EOF + +check_output "root value" <<EOF + = 1 +EOF + +run_test ./drive_json_ptr_walk <<EOF +[1, 2, 3] +EOF + +check_output "array" <<EOF +/0 = 1 +/1 = 2 +/2 = 3 +EOF + +run_test ./drive_json_ptr_walk <<EOF +[1, 2, 3, [4, 5, 6], 7, 8, 9, [10, 11, [12, 13, 14], 15], 16] +EOF + +check_output "nested array" <<EOF +/0 = 1 +/1 = 2 +/2 = 3 +/3/0 = 4 +/3/1 = 5 +/3/2 = 6 +/4 = 7 +/5 = 8 +/6 = 9 +/7/0 = 10 +/7/1 = 11 +/7/2/0 = 12 +/7/2/1 = 13 +/7/2/2 = 14 +/7/3 = 15 +/8 = 16 +EOF + +run_test ./drive_json_ptr_walk <<EOF +[null, true, 123.0, "foo", { "bar" : { "baz" : [1, 2, 3]} }, ["a", null]] +EOF + +check_error_output "" <<EOF +EOF + +check_output "complex" <<EOF +/0 = null +/1 = true +/2 = 123.0 +/3 = "foo" +/4/bar/baz/0 = 1 +/4/bar/baz/1 = 2 +/4/bar/baz/2 = 3 +/5/0 = "a" +/5/1 = null +EOF diff --git a/src/yajlpp/test_yajlpp.cc b/src/yajlpp/test_yajlpp.cc new file mode 100644 index 0000000..afa60a3 --- /dev/null +++ b/src/yajlpp/test_yajlpp.cc @@ -0,0 +1,166 @@ +/** + * Copyright (c) 2013, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file test_yajlpp.cc + */ + +#include <assert.h> +#include <stdio.h> + +#include "yajlpp.hh" +#include "yajlpp_def.hh" + +const char* TEST_DATA = R"([{ "foo": 0 }, 2, { "foo": 1 }])"; + +const char* TEST_OBJ_DATA = "{ \"foo\": 0 }"; + +static int FOO_COUNT = 0; +static int CONST_COUNT = 0; + +static int +read_foo(yajlpp_parse_context* ypc, long long value) +{ + assert(value == FOO_COUNT); + assert(ypc->ypc_array_index.empty() + || ypc->ypc_array_index.back() == FOO_COUNT); + + FOO_COUNT += 1; + + return 1; +} + +static int +read_const(yajlpp_parse_context* ypc, long long value) +{ + CONST_COUNT += 1; + + return 1; +} + +static int +dummy_string_handler(void* ctx, const unsigned char* s, size_t len) +{ + return 1; +} + +int +main(int argc, char* argv[]) +{ + static const auto TEST_SRC = intern_string::lookup("test_data"); + + { + struct dummy {}; + + typed_json_path_container<dummy> dummy_handlers = { + + }; + + std::string in1 = "{\"#\":{\""; + auto parse_res = dummy_handlers.parser_for(TEST_SRC).of(in1); + } + + { + static const char UNICODE_BARF[] = "\"\\udb00\\\\0\"\n"; + + yajl_callbacks cbs; + memset(&cbs, 0, sizeof(cbs)); + cbs.yajl_string = dummy_string_handler; + auto handle = yajl_alloc(&cbs, nullptr, nullptr); + auto rc = yajl_parse(handle, (const unsigned char*) UNICODE_BARF, 12); + assert(rc == yajl_status_ok); + yajl_free(handle); + } + + struct json_path_container test_obj_handler = { + json_path_handler("foo", read_foo), + }; + + { + struct json_path_container test_array_handlers = { + json_path_handler("#") + .add_cb(read_const) + .with_children(test_obj_handler), + }; + + yajlpp_parse_context ypc(TEST_SRC, &test_array_handlers); + auto handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc); + ypc.with_handle(handle); + ypc.parse((const unsigned char*) TEST_DATA, strlen(TEST_DATA)); + yajl_free(handle); + + assert(FOO_COUNT == 2); + assert(CONST_COUNT == 1); + } + + { + FOO_COUNT = 0; + + yajlpp_parse_context ypc(TEST_SRC, &test_obj_handler); + auto handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc); + ypc.with_handle(handle); + + ypc.parse(reinterpret_cast<const unsigned char*>(TEST_OBJ_DATA), + strlen(TEST_OBJ_DATA)); + yajl_free(handle); + + assert(FOO_COUNT == 1); + } + + { + const char* TEST_INPUT = R"({ + "msg": "Hello, World!", + "parent1": { + "child": {} + }, + "parent2": { + "child": {"name": "steve"} + }, + "parent3": { + "child": {}, + "sibling": {"name": "mongoose"} + } +})"; + const std::string EXPECTED_OUTPUT + = "{\"msg\":\"Hello, " + "World!\",\"parent2\":{\"child\":{\"name\":\"steve\"}}," + "\"parent3\":{\"sibling\":{\"name\":\"mongoose\"}}}"; + + char errbuf[1024]; + + auto tree = yajl_tree_parse(TEST_INPUT, errbuf, sizeof(errbuf)); + yajl_cleanup_tree(tree); + + yajlpp_gen gen; + + yajl_gen_tree(gen, tree); + auto actual = gen.to_string_fragment().to_string(); + assert(EXPECTED_OUTPUT == actual); + + yajl_tree_free(tree); + } +} diff --git a/src/yajlpp/yajlpp.cc b/src/yajlpp/yajlpp.cc new file mode 100644 index 0000000..010a7f9 --- /dev/null +++ b/src/yajlpp/yajlpp.cc @@ -0,0 +1,1565 @@ +/** + * Copyright (c) 2015, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file yajlpp.cc + */ + +#include <regex> +#include <utility> + +#include "yajlpp.hh" + +#include "base/fs_util.hh" +#include "base/snippet_highlighters.hh" +#include "config.h" +#include "fmt/format.h" +#include "ghc/filesystem.hpp" +#include "yajl/api/yajl_parse.h" +#include "yajlpp_def.hh" + +const json_path_handler_base::enum_value_t + json_path_handler_base::ENUM_TERMINATOR((const char*) nullptr, 0); + +yajl_gen_status +yajl_gen_tree(yajl_gen hand, yajl_val val) +{ + switch (val->type) { + case yajl_t_string: { + return yajl_gen_string(hand, YAJL_GET_STRING(val)); + } + case yajl_t_number: { + if (YAJL_IS_INTEGER(val)) { + return yajl_gen_integer(hand, YAJL_GET_INTEGER(val)); + } + if (YAJL_IS_DOUBLE(val)) { + return yajl_gen_double(hand, YAJL_GET_DOUBLE(val)); + } + return yajl_gen_number( + hand, YAJL_GET_NUMBER(val), strlen(YAJL_GET_NUMBER(val))); + } + case yajl_t_object: { + auto rc = yajl_gen_map_open(hand); + if (rc != yajl_gen_status_ok) { + return rc; + } + for (size_t lpc = 0; lpc < YAJL_GET_OBJECT(val)->len; lpc++) { + rc = yajl_gen_string(hand, YAJL_GET_OBJECT(val)->keys[lpc]); + if (rc != yajl_gen_status_ok) { + return rc; + } + rc = yajl_gen_tree(hand, YAJL_GET_OBJECT(val)->values[lpc]); + if (rc != yajl_gen_status_ok) { + return rc; + } + } + rc = yajl_gen_map_close(hand); + if (rc != yajl_gen_status_ok) { + return rc; + } + return yajl_gen_status_ok; + } + case yajl_t_array: { + auto rc = yajl_gen_array_open(hand); + if (rc != yajl_gen_status_ok) { + return rc; + } + for (size_t lpc = 0; lpc < YAJL_GET_ARRAY(val)->len; lpc++) { + rc = yajl_gen_tree(hand, YAJL_GET_ARRAY(val)->values[lpc]); + if (rc != yajl_gen_status_ok) { + return rc; + } + } + rc = yajl_gen_array_close(hand); + if (rc != yajl_gen_status_ok) { + return rc; + } + return yajl_gen_status_ok; + } + case yajl_t_true: { + return yajl_gen_bool(hand, true); + } + case yajl_t_false: { + return yajl_gen_bool(hand, false); + } + case yajl_t_null: { + return yajl_gen_null(hand); + } + default: + return yajl_gen_status_ok; + } +} + +void +yajl_cleanup_tree(yajl_val val) +{ + if (YAJL_IS_OBJECT(val)) { + auto* val_as_obj = YAJL_GET_OBJECT(val); + + for (size_t lpc = 0; lpc < val_as_obj->len;) { + auto* child_val = val_as_obj->values[lpc]; + + yajl_cleanup_tree(child_val); + if (YAJL_IS_OBJECT(child_val) + && YAJL_GET_OBJECT(child_val)->len == 0) + { + free((char*) val_as_obj->keys[lpc]); + yajl_tree_free(val_as_obj->values[lpc]); + val_as_obj->len -= 1; + for (auto lpc2 = lpc; lpc2 < val_as_obj->len; lpc2++) { + val_as_obj->keys[lpc2] = val_as_obj->keys[lpc2 + 1]; + val_as_obj->values[lpc2] = val_as_obj->values[lpc2 + 1]; + } + } else { + lpc++; + } + } + } +} + +json_path_handler_base::json_path_handler_base(const std::string& property) + : jph_property(property.back() == '#' + ? property.substr(0, property.size() - 1) + : property), + jph_regex(lnav::pcre2pp::code::from(lnav::pcre2pp::quote(property), + PCRE2_ANCHORED) + .unwrap() + .to_shared()), + jph_is_array(property.back() == '#') +{ + memset(&this->jph_callbacks, 0, sizeof(this->jph_callbacks)); +} + +static std::string +scrub_pattern(const std::string& pattern) +{ + static const std::regex CAPTURE(R"(\(\?\<\w+\>)"); + + return std::regex_replace(pattern, CAPTURE, "("); +} + +json_path_handler_base::json_path_handler_base( + const std::shared_ptr<const lnav::pcre2pp::code>& property) + : jph_property(scrub_pattern(property->get_pattern())), jph_regex(property), + jph_is_array(property->get_pattern().find('#') != std::string::npos), + jph_is_pattern_property(property->get_capture_count() > 0) +{ + memset(&this->jph_callbacks, 0, sizeof(this->jph_callbacks)); +} + +json_path_handler_base::json_path_handler_base( + std::string property, + const std::shared_ptr<const lnav::pcre2pp::code>& property_re) + : jph_property(std::move(property)), jph_regex(property_re), + jph_is_array(property_re->get_pattern().find('#') != std::string::npos) +{ + memset(&this->jph_callbacks, 0, sizeof(this->jph_callbacks)); +} + +yajl_gen_status +json_path_handler_base::gen(yajlpp_gen_context& ygc, yajl_gen handle) const +{ + if (this->jph_is_array) { + auto size = this->jph_size_provider(ygc.ygc_obj_stack.top()); + auto md = lnav::pcre2pp::match_data::unitialized(); + + yajl_gen_string(handle, this->jph_property); + yajl_gen_array_open(handle); + for (size_t index = 0; index < size; index++) { + yajlpp_provider_context ypc{&md, index}; + yajlpp_gen_context elem_ygc(handle, *this->jph_children); + elem_ygc.ygc_depth = 1; + elem_ygc.ygc_obj_stack.push( + this->jph_obj_provider(ypc, ygc.ygc_obj_stack.top())); + + elem_ygc.gen(); + } + yajl_gen_array_close(handle); + + return yajl_gen_status_ok; + } + + std::vector<std::string> local_paths; + + if (this->jph_path_provider) { + this->jph_path_provider(ygc.ygc_obj_stack.top(), local_paths); + } else { + local_paths.emplace_back(this->jph_property); + } + + if (this->jph_children) { + for (const auto& lpath : local_paths) { + std::string full_path = lpath; + if (this->jph_path_provider) { + full_path += "/"; + } + int start_depth = ygc.ygc_depth; + + yajl_gen_string(handle, lpath); + yajl_gen_map_open(handle); + ygc.ygc_depth += 1; + + if (this->jph_obj_provider) { + static thread_local auto md + = lnav::pcre2pp::match_data::unitialized(); + + auto find_res = this->jph_regex->capture_from(full_path) + .into(md) + .matches(); + + ygc.ygc_obj_stack.push(this->jph_obj_provider( + {&md, yajlpp_provider_context::nindex}, + ygc.ygc_obj_stack.top())); + if (!ygc.ygc_default_stack.empty()) { + ygc.ygc_default_stack.push(this->jph_obj_provider( + {&md, yajlpp_provider_context::nindex}, + ygc.ygc_default_stack.top())); + } + } + + for (const auto& jph : this->jph_children->jpc_children) { + yajl_gen_status status = jph.gen(ygc, handle); + + const unsigned char* buf; + size_t len; + yajl_gen_get_buf(handle, &buf, &len); + if (status != yajl_gen_status_ok) { + log_error("yajl_gen failure for: %s -- %d", + jph.jph_property.c_str(), + status); + return status; + } + } + + if (this->jph_obj_provider) { + ygc.ygc_obj_stack.pop(); + if (!ygc.ygc_default_stack.empty()) { + ygc.ygc_default_stack.pop(); + } + } + + while (ygc.ygc_depth > start_depth) { + yajl_gen_map_close(handle); + ygc.ygc_depth -= 1; + } + } + } else if (this->jph_gen_callback != nullptr) { + return this->jph_gen_callback(ygc, *this, handle); + } + + return yajl_gen_status_ok; +} + +const char* const SCHEMA_TYPE_STRINGS[] = { + "any", + "boolean", + "integer", + "number", + "string", + "array", + "object", +}; + +yajl_gen_status +json_path_handler_base::gen_schema(yajlpp_gen_context& ygc) const +{ + if (this->jph_children) { + { + yajlpp_map schema(ygc.ygc_handle); + + if (this->jph_description && this->jph_description[0]) { + schema.gen("description"); + schema.gen(this->jph_description); + } + if (this->jph_is_pattern_property) { + ygc.ygc_path.emplace_back( + fmt::format(FMT_STRING("<{}>"), + this->jph_regex->get_name_for_capture(1))); + } else { + ygc.ygc_path.emplace_back(this->jph_property); + } + if (this->jph_children->jpc_definition_id.empty()) { + schema.gen("title"); + schema.gen(fmt::format(FMT_STRING("/{}"), + fmt::join(ygc.ygc_path, "/"))); + schema.gen("type"); + if (this->jph_is_array) { + if (this->jph_regex->get_pattern().find("#?") + == std::string::npos) + { + schema.gen("array"); + } else { + yajlpp_array type_array(ygc.ygc_handle); + + type_array.gen("array"); + for (auto schema_type : this->get_types()) { + type_array.gen( + SCHEMA_TYPE_STRINGS[(int) schema_type]); + } + } + schema.gen("items"); + yajl_gen_map_open(ygc.ygc_handle); + yajl_gen_string(ygc.ygc_handle, "type"); + this->gen_schema_type(ygc); + } else { + this->gen_schema_type(ygc); + } + this->jph_children->gen_schema(ygc); + if (this->jph_is_array) { + yajl_gen_map_close(ygc.ygc_handle); + } + } else { + schema.gen("title"); + schema.gen(fmt::format(FMT_STRING("/{}"), + fmt::join(ygc.ygc_path, "/"))); + this->jph_children->gen_schema(ygc); + } + ygc.ygc_path.pop_back(); + } + } else { + yajlpp_map schema(ygc.ygc_handle); + + if (this->jph_is_pattern_property) { + ygc.ygc_path.emplace_back(fmt::format( + FMT_STRING("<{}>"), this->jph_regex->get_name_for_capture(1))); + } else { + ygc.ygc_path.emplace_back(this->jph_property); + } + + schema.gen("title"); + schema.gen( + fmt::format(FMT_STRING("/{}"), fmt::join(ygc.ygc_path, "/"))); + if (this->jph_description && this->jph_description[0]) { + schema.gen("description"); + schema.gen(this->jph_description); + } + + schema.gen("type"); + + if (this->jph_is_array) { + if (this->jph_regex->get_pattern().find("#?") == std::string::npos) + { + schema.gen("array"); + } else { + yajlpp_array type_array(ygc.ygc_handle); + + type_array.gen("array"); + for (auto schema_type : this->get_types()) { + type_array.gen(SCHEMA_TYPE_STRINGS[(int) schema_type]); + } + } + yajl_gen_string(ygc.ygc_handle, "items"); + yajl_gen_map_open(ygc.ygc_handle); + yajl_gen_string(ygc.ygc_handle, "type"); + } + + this->gen_schema_type(ygc); + + if (!this->jph_examples.empty()) { + schema.gen("examples"); + + yajlpp_array example_array(ygc.ygc_handle); + for (const auto& ex : this->jph_examples) { + example_array.gen(ex); + } + } + + if (this->jph_is_array) { + yajl_gen_map_close(ygc.ygc_handle); + } + + ygc.ygc_path.pop_back(); + } + + return yajl_gen_status_ok; +} + +yajl_gen_status +json_path_handler_base::gen_schema_type(yajlpp_gen_context& ygc) const +{ + yajlpp_generator schema(ygc.ygc_handle); + + auto types = this->get_types(); + if (types.size() == 1) { + yajl_gen_string(ygc.ygc_handle, SCHEMA_TYPE_STRINGS[(int) types[0]]); + } else { + yajlpp_array type_array(ygc.ygc_handle); + + for (auto schema_type : types) { + type_array.gen(SCHEMA_TYPE_STRINGS[(int) schema_type]); + } + } + + for (auto& schema_type : types) { + switch (schema_type) { + case schema_type_t::STRING: + if (this->jph_min_length > 0) { + schema("minLength"); + schema(this->jph_min_length); + } + if (this->jph_max_length < INT_MAX) { + schema("maxLength"); + schema(this->jph_max_length); + } + if (this->jph_pattern_re) { + schema("pattern"); + schema(this->jph_pattern_re); + } + if (this->jph_enum_values) { + schema("enum"); + + yajlpp_array enum_array(ygc.ygc_handle); + for (int lpc = 0; this->jph_enum_values[lpc].first; lpc++) { + enum_array.gen(this->jph_enum_values[lpc].first); + } + } + break; + case schema_type_t::INTEGER: + case schema_type_t::NUMBER: + if (this->jph_min_value > LLONG_MIN) { + schema("minimum"); + schema(this->jph_min_value); + } + break; + default: + break; + } + } + + return yajl_gen_keys_must_be_strings; +} + +void +json_path_handler_base::walk( + const std::function< + void(const json_path_handler_base&, const std::string&, void*)>& cb, + void* root, + const std::string& base) const +{ + std::vector<std::string> local_paths; + + if (this->jph_path_provider) { + this->jph_path_provider(root, local_paths); + + for (auto& lpath : local_paths) { + cb(*this, + fmt::format(FMT_STRING("{}{}{}"), + base, + lpath, + this->jph_children ? "/" : ""), + nullptr); + } + if (this->jph_obj_deleter) { + local_paths.clear(); + this->jph_path_provider(root, local_paths); + } + } else { + local_paths.emplace_back(this->jph_property); + + std::string full_path = base + this->jph_property; + if (this->jph_children) { + full_path += "/"; + } + cb(*this, full_path, nullptr); + } + + if (this->jph_children) { + for (const auto& lpath : local_paths) { + for (const auto& jph : this->jph_children->jpc_children) { + static const intern_string_t POSS_SRC + = intern_string::lookup("possibilities"); + + std::string full_path = base + lpath; + if (this->jph_children) { + full_path += "/"; + } + json_path_container dummy{ + json_path_handler(this->jph_property, this->jph_regex), + }; + + yajlpp_parse_context ypc(POSS_SRC, &dummy); + void* child_root = root; + + ypc.set_path(full_path).with_obj(root).update_callbacks(); + if (this->jph_obj_provider) { + static thread_local auto md + = lnav::pcre2pp::match_data::unitialized(); + + std::string full_path = lpath + "/"; + + if (!this->jph_regex->capture_from(full_path) + .into(md) + .matches() + .ignore_error()) + { + ensure(false); + } + child_root = this->jph_obj_provider( + {&md, yajlpp_provider_context::nindex}, root); + } + + jph.walk(cb, child_root, full_path); + } + } + } else { + for (auto& lpath : local_paths) { + void* field = nullptr; + + if (this->jph_field_getter) { + field = this->jph_field_getter(root, lpath); + } + cb(*this, base + lpath, field); + } + } +} + +nonstd::optional<int> +json_path_handler_base::to_enum_value(const string_fragment& sf) const +{ + for (int lpc = 0; this->jph_enum_values[lpc].first; lpc++) { + const auto& ev = this->jph_enum_values[lpc]; + + if (sf == ev.first) { + return ev.second; + } + } + + return nonstd::nullopt; +} + +const char* +json_path_handler_base::to_enum_string(int value) const +{ + for (int lpc = 0; this->jph_enum_values[lpc].first; lpc++) { + const auto& ev = this->jph_enum_values[lpc]; + + if (ev.second == value) { + return ev.first; + } + } + + return ""; +} + +std::vector<json_path_handler_base::schema_type_t> +json_path_handler_base::get_types() const +{ + std::vector<schema_type_t> retval; + + if (this->jph_callbacks.yajl_boolean) { + retval.push_back(schema_type_t::BOOLEAN); + } + if (this->jph_callbacks.yajl_integer) { + retval.push_back(schema_type_t::INTEGER); + } + if (this->jph_callbacks.yajl_double || this->jph_callbacks.yajl_number) { + retval.push_back(schema_type_t::NUMBER); + } + if (this->jph_callbacks.yajl_string) { + retval.push_back(schema_type_t::STRING); + } + if (this->jph_children) { + retval.push_back(schema_type_t::OBJECT); + } + if (retval.empty()) { + retval.push_back(schema_type_t::ANY); + } + return retval; +} + +yajlpp_parse_context::yajlpp_parse_context( + intern_string_t source, const struct json_path_container* handlers) + : ypc_source(source), ypc_handlers(handlers) +{ + this->ypc_path.reserve(4096); + this->ypc_path.push_back('/'); + this->ypc_path.push_back('\0'); + this->ypc_callbacks = DEFAULT_CALLBACKS; + memset(&this->ypc_alt_callbacks, 0, sizeof(this->ypc_alt_callbacks)); +} + +int +yajlpp_parse_context::map_start(void* ctx) +{ + yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx; + int retval = 1; + + require(ypc->ypc_path.size() >= 2); + + ypc->ypc_path_index_stack.push_back(ypc->ypc_path.size() - 1); + + if (ypc->ypc_path.size() > 1 + && ypc->ypc_path[ypc->ypc_path.size() - 2] == '#') + { + ypc->ypc_array_index.back() += 1; + } + + if (ypc->ypc_alt_callbacks.yajl_start_map != nullptr) { + retval = ypc->ypc_alt_callbacks.yajl_start_map(ypc); + } + + return retval; +} + +int +yajlpp_parse_context::map_key(void* ctx, const unsigned char* key, size_t len) +{ + yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx; + int retval = 1; + + require(ypc->ypc_path.size() >= 2); + + ypc->ypc_path.resize(ypc->ypc_path_index_stack.back()); + if (ypc->ypc_path.back() != '/') { + ypc->ypc_path.push_back('/'); + } + for (size_t lpc = 0; lpc < len; lpc++) { + switch (key[lpc]) { + case '~': + ypc->ypc_path.push_back('~'); + ypc->ypc_path.push_back('0'); + break; + case '/': + ypc->ypc_path.push_back('~'); + ypc->ypc_path.push_back('1'); + break; + case '#': + ypc->ypc_path.push_back('~'); + ypc->ypc_path.push_back('2'); + break; + default: + ypc->ypc_path.push_back(key[lpc]); + break; + } + } + ypc->ypc_path.push_back('\0'); + + if (ypc->ypc_alt_callbacks.yajl_map_key != nullptr) { + retval = ypc->ypc_alt_callbacks.yajl_map_key(ctx, key, len); + } + + if (ypc->ypc_handlers != nullptr) { + ypc->update_callbacks(); + } + + ensure(ypc->ypc_path.size() >= 2); + + return retval; +} + +void +yajlpp_parse_context::update_callbacks(const json_path_container* orig_handlers, + int child_start) +{ + const json_path_container* handlers = orig_handlers; + + this->ypc_current_handler = nullptr; + + if (this->ypc_handlers == nullptr) { + return; + } + + this->ypc_sibling_handlers = orig_handlers; + this->ypc_callbacks = DEFAULT_CALLBACKS; + + if (handlers == nullptr) { + handlers = this->ypc_handlers; + this->ypc_handler_stack.clear(); + this->ypc_array_handler_count = 0; + } + + if (!this->ypc_active_paths.empty()) { + std::string curr_path(&this->ypc_path[0], this->ypc_path.size() - 1); + + if (this->ypc_active_paths.find(curr_path) + == this->ypc_active_paths.end()) + { + return; + } + } + + if (child_start == 0 && !this->ypc_obj_stack.empty()) { + while (this->ypc_obj_stack.size() > 1) { + this->ypc_obj_stack.pop(); + } + } + + auto path_frag = string_fragment::from_byte_range( + this->ypc_path.data(), 1 + child_start, this->ypc_path.size() - 1); + for (const auto& jph : handlers->jpc_children) { + static thread_local auto md = lnav::pcre2pp::match_data::unitialized(); + + if (jph.jph_regex->capture_from(path_frag) + .into(md) + .matches() + .ignore_error()) + { + auto cap = md[0].value(); + + if (jph.jph_is_array) { + this->ypc_array_handler_count += 1; + } + if (jph.jph_obj_provider) { + auto index = this->ypc_array_handler_count == 0 + ? static_cast<size_t>(-1) + : this->ypc_array_index[this->ypc_array_handler_count - 1]; + + if ((cap.sf_end != (int) this->ypc_path.size() - 1) + && (!jph.is_array() + || index != yajlpp_provider_context::nindex)) + { + this->ypc_obj_stack.push(jph.jph_obj_provider( + {&md, index, this}, this->ypc_obj_stack.top())); + } + } + + if (jph.jph_children) { + this->ypc_handler_stack.emplace_back(&jph); + + if (cap.sf_end != (int) this->ypc_path.size() - 1) { + this->update_callbacks(jph.jph_children, cap.sf_end); + return; + } + } else { + if (cap.sf_end != (int) this->ypc_path.size() - 1) { + continue; + } + + this->ypc_current_handler = &jph; + } + + if (jph.jph_callbacks.yajl_null != nullptr) { + this->ypc_callbacks.yajl_null = jph.jph_callbacks.yajl_null; + } + if (jph.jph_callbacks.yajl_boolean != nullptr) { + this->ypc_callbacks.yajl_boolean + = jph.jph_callbacks.yajl_boolean; + } + if (jph.jph_callbacks.yajl_integer != nullptr) { + this->ypc_callbacks.yajl_integer + = jph.jph_callbacks.yajl_integer; + } + if (jph.jph_callbacks.yajl_double != nullptr) { + this->ypc_callbacks.yajl_double = jph.jph_callbacks.yajl_double; + } + if (jph.jph_callbacks.yajl_string != nullptr) { + this->ypc_callbacks.yajl_string = jph.jph_callbacks.yajl_string; + } + if (jph.jph_is_array) { + this->ypc_array_handler_count -= 1; + } + return; + } + } +} + +int +yajlpp_parse_context::map_end(void* ctx) +{ + yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx; + int retval = 1; + + ypc->ypc_path.resize(ypc->ypc_path_index_stack.back()); + ypc->ypc_path.push_back('\0'); + ypc->ypc_path_index_stack.pop_back(); + + if (ypc->ypc_alt_callbacks.yajl_end_map != nullptr) { + retval = ypc->ypc_alt_callbacks.yajl_end_map(ctx); + } + + ypc->update_callbacks(); + + ensure(ypc->ypc_path.size() >= 2); + + return retval; +} + +int +yajlpp_parse_context::array_start(void* ctx) +{ + yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx; + int retval = 1; + + ypc->ypc_path_index_stack.push_back(ypc->ypc_path.size() - 1); + ypc->ypc_path[ypc->ypc_path.size() - 1] = '#'; + ypc->ypc_path.push_back('\0'); + ypc->ypc_array_index.push_back(-1); + + if (ypc->ypc_alt_callbacks.yajl_start_array != nullptr) { + retval = ypc->ypc_alt_callbacks.yajl_start_array(ctx); + } + + ypc->update_callbacks(); + + ensure(ypc->ypc_path.size() >= 2); + + return retval; +} + +int +yajlpp_parse_context::array_end(void* ctx) +{ + yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx; + int retval = 1; + + ypc->ypc_path.resize(ypc->ypc_path_index_stack.back()); + ypc->ypc_path.push_back('\0'); + ypc->ypc_path_index_stack.pop_back(); + ypc->ypc_array_index.pop_back(); + + if (ypc->ypc_alt_callbacks.yajl_end_array != nullptr) { + retval = ypc->ypc_alt_callbacks.yajl_end_array(ctx); + } + + ypc->update_callbacks(); + + ensure(ypc->ypc_path.size() >= 2); + + return retval; +} + +int +yajlpp_parse_context::handle_unused(void* ctx) +{ + yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx; + + if (ypc->ypc_ignore_unused) { + return 1; + } + + const json_path_handler_base* handler = ypc->ypc_current_handler; + lnav::console::user_message msg; + + if (handler != nullptr && strlen(handler->jph_synopsis) > 0 + && strlen(handler->jph_description) > 0) + { + auto help_text = handler->get_help_text(ypc); + std::vector<std::string> expected_types; + + if (ypc->ypc_callbacks.yajl_boolean + != (int (*)(void*, int)) yajlpp_parse_context::handle_unused) + { + expected_types.emplace_back("boolean"); + } + if (ypc->ypc_callbacks.yajl_integer + != (int (*)(void*, long long)) yajlpp_parse_context::handle_unused) + { + expected_types.emplace_back("integer"); + } + if (ypc->ypc_callbacks.yajl_double + != (int (*)(void*, double)) yajlpp_parse_context::handle_unused) + { + expected_types.emplace_back("float"); + } + if (ypc->ypc_callbacks.yajl_string + != (int (*)(void*, const unsigned char*, size_t)) + yajlpp_parse_context::handle_unused) + { + expected_types.emplace_back("string"); + } + if (!expected_types.empty()) { + help_text.appendf( + FMT_STRING(" expecting one of the following types: {}"), + fmt::join(expected_types, ", ")); + } + msg = lnav::console::user_message::warning( + attr_line_t("unexpected data for property ") + .append_quoted(lnav::roles::symbol( + ypc->get_full_path().to_string()))) + .with_help(help_text); + } else if (ypc->ypc_path[1]) { + msg = lnav::console::user_message::warning( + attr_line_t("unexpected value for property ") + .append_quoted( + lnav::roles::symbol(ypc->get_full_path().to_string()))); + } else { + msg = lnav::console::user_message::error("unexpected JSON value"); + } + + if (handler == nullptr) { + const json_path_container* accepted_handlers; + + if (ypc->ypc_sibling_handlers) { + accepted_handlers = ypc->ypc_sibling_handlers; + } else { + accepted_handlers = ypc->ypc_handlers; + } + + attr_line_t help_text; + + if (accepted_handlers->jpc_children.size() == 1 + && accepted_handlers->jpc_children.front().jph_is_array) + { + const auto& jph = accepted_handlers->jpc_children.front(); + + help_text.append("expecting an array of ") + .append(lnav::roles::variable(jph.jph_synopsis)) + .append(" values"); + } else { + help_text.append(lnav::roles::h2("Available Properties")) + .append("\n"); + for (const auto& jph : accepted_handlers->jpc_children) { + help_text.append(" ") + .append(lnav::roles::symbol(jph.jph_property)) + .append(lnav::roles::symbol( + jph.jph_children != nullptr ? "/" : "")) + .append(" ") + .append(lnav::roles::variable(jph.jph_synopsis)) + .append("\n"); + } + } + msg.with_help(help_text); + } + + msg.with_snippet(ypc->get_snippet()); + + ypc->report_error(msg); + + return 1; +} + +int +yajlpp_parse_context::handle_unused_or_delete(void* ctx) +{ + yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx; + + if (!ypc->ypc_handler_stack.empty() + && ypc->ypc_handler_stack.back()->jph_obj_deleter) + { + static thread_local auto md = lnav::pcre2pp::match_data::unitialized(); + + auto key_start = ypc->ypc_path_index_stack.back(); + auto path_frag = string_fragment::from_byte_range( + ypc->ypc_path.data(), key_start + 1, ypc->ypc_path.size() - 1); + yajlpp_provider_context provider_ctx{&md, static_cast<size_t>(-1)}; + ypc->ypc_handler_stack.back() + ->jph_regex->capture_from(path_frag) + .into(md) + .matches(); + + ypc->ypc_handler_stack.back()->jph_obj_deleter( + provider_ctx, ypc->ypc_obj_stack.top()); + return 1; + } + + return handle_unused(ctx); +} + +const yajl_callbacks yajlpp_parse_context::DEFAULT_CALLBACKS = { + yajlpp_parse_context::handle_unused_or_delete, + (int (*)(void*, int)) yajlpp_parse_context::handle_unused, + (int (*)(void*, long long)) yajlpp_parse_context::handle_unused, + (int (*)(void*, double)) yajlpp_parse_context::handle_unused, + nullptr, + (int (*)(void*, const unsigned char*, size_t)) + yajlpp_parse_context::handle_unused, + yajlpp_parse_context::map_start, + yajlpp_parse_context::map_key, + yajlpp_parse_context::map_end, + yajlpp_parse_context::array_start, + yajlpp_parse_context::array_end, +}; + +yajl_status +yajlpp_parse_context::parse(const unsigned char* jsonText, size_t jsonTextLen) +{ + this->ypc_json_text = jsonText; + this->ypc_json_text_len = jsonTextLen; + + yajl_status retval = yajl_parse(this->ypc_handle, jsonText, jsonTextLen); + + size_t consumed = yajl_get_bytes_consumed(this->ypc_handle); + + this->ypc_line_number + += std::count(&jsonText[0], &jsonText[consumed], '\n'); + + this->ypc_json_text = nullptr; + + if (retval != yajl_status_ok && this->ypc_error_reporter) { + auto* msg = yajl_get_error(this->ypc_handle, 1, jsonText, jsonTextLen); + + this->report_error( + lnav::console::user_message::error("invalid JSON") + .with_snippet(lnav::console::snippet::from(this->ypc_source, + (const char*) msg) + .with_line(this->get_line_number()))); + yajl_free_error(this->ypc_handle, msg); + } + + return retval; +} + +yajl_status +yajlpp_parse_context::complete_parse() +{ + yajl_status retval = yajl_complete_parse(this->ypc_handle); + + if (retval != yajl_status_ok && this->ypc_error_reporter) { + auto* msg = yajl_get_error(this->ypc_handle, 0, nullptr, 0); + + this->report_error(lnav::console::user_message::error("invalid JSON") + .with_reason((const char*) msg) + .with_snippet(this->get_snippet())); + yajl_free_error(this->ypc_handle, msg); + } + + return retval; +} + +bool +yajlpp_parse_context::parse_doc(const string_fragment& sf) +{ + bool retval = true; + + this->ypc_json_text = (const unsigned char*) sf.data(); + this->ypc_json_text_len = sf.length(); + + auto rc = yajl_parse(this->ypc_handle, this->ypc_json_text, sf.length()); + size_t consumed = yajl_get_bytes_consumed(this->ypc_handle); + this->ypc_total_consumed += consumed; + this->ypc_line_number += std::count( + &this->ypc_json_text[0], &this->ypc_json_text[consumed], '\n'); + + if (rc != yajl_status_ok) { + if (this->ypc_error_reporter) { + auto* msg = yajl_get_error(this->ypc_handle, + 1, + this->ypc_json_text, + this->ypc_json_text_len); + + this->report_error( + lnav::console::user_message::error("invalid JSON") + .with_reason((const char*) msg) + .with_snippet(this->get_snippet())); + yajl_free_error(this->ypc_handle, msg); + } + retval = false; + } else if (this->complete_parse() != yajl_status_ok) { + retval = false; + } + + this->ypc_json_text = nullptr; + + return retval; +} + +const intern_string_t +yajlpp_parse_context::get_path() const +{ + if (this->ypc_path.size() <= 1) { + return intern_string_t(); + } + return intern_string::lookup(&this->ypc_path[1], this->ypc_path.size() - 2); +} + +const intern_string_t +yajlpp_parse_context::get_full_path() const +{ + if (this->ypc_path.size() <= 1) { + static const intern_string_t SLASH = intern_string::lookup("/"); + + return SLASH; + } + return intern_string::lookup(&this->ypc_path[0], this->ypc_path.size() - 1); +} + +void +yajlpp_parse_context::reset(const struct json_path_container* handlers) +{ + this->ypc_handlers = handlers; + this->ypc_path.clear(); + this->ypc_path.push_back('/'); + this->ypc_path.push_back('\0'); + this->ypc_path_index_stack.clear(); + this->ypc_array_index.clear(); + this->ypc_array_handler_count = 0; + this->ypc_callbacks = DEFAULT_CALLBACKS; + memset(&this->ypc_alt_callbacks, 0, sizeof(this->ypc_alt_callbacks)); + this->ypc_sibling_handlers = nullptr; + this->ypc_current_handler = nullptr; + while (!this->ypc_obj_stack.empty()) { + this->ypc_obj_stack.pop(); + } +} + +void +yajlpp_parse_context::set_static_handler(const json_path_handler_base& jph) +{ + this->ypc_path.clear(); + this->ypc_path.push_back('/'); + this->ypc_path.push_back('\0'); + this->ypc_path_index_stack.clear(); + this->ypc_array_index.clear(); + this->ypc_array_handler_count = 0; + if (jph.jph_callbacks.yajl_null != nullptr) { + this->ypc_callbacks.yajl_null = jph.jph_callbacks.yajl_null; + } + if (jph.jph_callbacks.yajl_boolean != nullptr) { + this->ypc_callbacks.yajl_boolean = jph.jph_callbacks.yajl_boolean; + } + if (jph.jph_callbacks.yajl_integer != nullptr) { + this->ypc_callbacks.yajl_integer = jph.jph_callbacks.yajl_integer; + } + if (jph.jph_callbacks.yajl_double != nullptr) { + this->ypc_callbacks.yajl_double = jph.jph_callbacks.yajl_double; + } + if (jph.jph_callbacks.yajl_string != nullptr) { + this->ypc_callbacks.yajl_string = jph.jph_callbacks.yajl_string; + } +} + +yajlpp_parse_context& +yajlpp_parse_context::set_path(const std::string& path) +{ + this->ypc_path.resize(path.size() + 1); + std::copy(path.begin(), path.end(), this->ypc_path.begin()); + this->ypc_path[path.size()] = '\0'; + for (size_t lpc = 0; lpc < path.size(); lpc++) { + switch (path[lpc]) { + case '/': + this->ypc_path_index_stack.push_back( + this->ypc_path_index_stack.empty() ? 1 : 0 + lpc); + break; + } + } + return *this; +} + +const char* +yajlpp_parse_context::get_path_fragment(int offset, + char* frag_in, + size_t& len_out) const +{ + const char* retval; + size_t start, end; + + if (offset < 0) { + offset = ((int) this->ypc_path_index_stack.size()) + offset; + } + start = this->ypc_path_index_stack[offset] + ((offset == 0) ? 0 : 1); + if ((offset + 1) < (int) this->ypc_path_index_stack.size()) { + end = this->ypc_path_index_stack[offset + 1]; + } else { + end = this->ypc_path.size() - 1; + } + if (this->ypc_handlers) { + len_out + = json_ptr::decode(frag_in, &this->ypc_path[start], end - start); + retval = frag_in; + } else { + retval = &this->ypc_path[start]; + len_out = end - start; + } + + return retval; +} + +int +yajlpp_parse_context::get_line_number() const +{ + if (this->ypc_handle != nullptr && this->ypc_json_text) { + size_t consumed = yajl_get_bytes_consumed(this->ypc_handle); + long current_count = std::count( + &this->ypc_json_text[0], &this->ypc_json_text[consumed], '\n'); + + return this->ypc_line_number + current_count; + } + return this->ypc_line_number; +} + +void +yajlpp_gen_context::gen() +{ + yajlpp_map root(this->ygc_handle); + + for (const auto& jph : this->ygc_handlers->jpc_children) { + jph.gen(*this, this->ygc_handle); + } +} + +void +yajlpp_gen_context::gen_schema(const json_path_container* handlers) +{ + if (handlers == nullptr) { + handlers = this->ygc_handlers; + } + + { + yajlpp_map schema(this->ygc_handle); + + if (!handlers->jpc_schema_id.empty()) { + schema.gen("$id"); + schema.gen(handlers->jpc_schema_id); + schema.gen("title"); + schema.gen(handlers->jpc_schema_id); + } + schema.gen("$schema"); + schema.gen("http://json-schema.org/draft-07/schema#"); + if (!handlers->jpc_description.empty()) { + schema.gen("description"); + schema.gen(handlers->jpc_description); + } + handlers->gen_schema(*this); + + if (!this->ygc_schema_definitions.empty()) { + schema.gen("definitions"); + + yajlpp_map defs(this->ygc_handle); + for (auto& container : this->ygc_schema_definitions) { + defs.gen(container.first); + + yajlpp_map def(this->ygc_handle); + + def.gen("title"); + def.gen(container.first); + if (!container.second->jpc_description.empty()) { + def.gen("description"); + def.gen(container.second->jpc_description); + } + def.gen("type"); + def.gen("object"); + def.gen("$$target"); + def.gen(fmt::format(FMT_STRING("#/definitions/{}"), + container.first)); + container.second->gen_properties(*this); + } + } + } +} + +yajlpp_gen_context& +yajlpp_gen_context::with_context(yajlpp_parse_context& ypc) +{ + this->ygc_obj_stack = ypc.ypc_obj_stack; + if (ypc.ypc_current_handler == nullptr && !ypc.ypc_handler_stack.empty() + && ypc.ypc_handler_stack.back() != nullptr) + { + this->ygc_handlers = ypc.ypc_handler_stack.back()->jph_children; + this->ygc_depth += 1; + } + return *this; +} + +json_path_handler& +json_path_handler::with_children(const json_path_container& container) +{ + this->jph_children = &container; + return *this; +} + +lnav::console::snippet +yajlpp_parse_context::get_snippet() const +{ + auto line_number = this->get_line_number(); + attr_line_t content; + + if (this->ypc_json_text != nullptr) { + auto in_text_line = line_number - this->ypc_line_number; + const auto* line_start = this->ypc_json_text; + auto text_len_remaining = this->ypc_json_text_len; + + while (in_text_line > 0) { + const auto* line_end = (const unsigned char*) memchr( + line_start, '\n', text_len_remaining); + if (line_end == nullptr) { + break; + } + + text_len_remaining -= (line_end - line_start) + 1; + line_start = line_end + 1; + in_text_line -= 1; + } + + if (text_len_remaining > 0) { + const auto* line_end = (const unsigned char*) memchr( + line_start, '\n', text_len_remaining); + if (line_end) { + text_len_remaining = (line_end - line_start); + } + content.append( + string_fragment::from_bytes(line_start, text_len_remaining)); + } + } + + content.with_attr_for_all(VC_ROLE.value(role_t::VCR_QUOTED_CODE)); + return lnav::console::snippet::from(this->ypc_source, content) + .with_line(line_number); +} + +void +json_path_handler_base::validate_string(yajlpp_parse_context& ypc, + string_fragment sf) const +{ + if (this->jph_pattern) { + if (!this->jph_pattern->find_in(sf).ignore_error()) { + this->report_pattern_error(&ypc, sf.to_string()); + } + } + if (sf.empty() && this->jph_min_length > 0) { + ypc.report_error(lnav::console::user_message::error( + attr_line_t("invalid value for option ") + .append_quoted(lnav::roles::symbol( + ypc.get_full_path().to_string()))) + .with_reason("empty values are not allowed") + .with_snippet(ypc.get_snippet()) + .with_help(this->get_help_text(&ypc))); + } else if (sf.length() < this->jph_min_length) { + ypc.report_error( + lnav::console::user_message::error( + attr_line_t() + .append_quoted(sf) + .append(" is not a valid value for option ") + .append_quoted( + lnav::roles::symbol(ypc.get_full_path().to_string()))) + .with_reason(attr_line_t("value must be at least ") + .append(lnav::roles::number( + fmt::to_string(this->jph_min_length))) + .append(" characters long")) + .with_snippet(ypc.get_snippet()) + .with_help(this->get_help_text(&ypc))); + } +} + +void +json_path_handler_base::report_pattern_error(yajlpp_parse_context* ypc, + const std::string& value_str) const +{ + ypc->report_error( + lnav::console::user_message::error( + attr_line_t() + .append_quoted(value_str) + .append(" is not a valid value for option ") + .append_quoted( + lnav::roles::symbol(ypc->get_full_path().to_string()))) + .with_snippet(ypc->get_snippet()) + .with_reason(attr_line_t("value does not match pattern: ") + .append(lnav::roles::symbol(this->jph_pattern_re))) + .with_help(this->get_help_text(ypc))); +} + +attr_line_t +json_path_handler_base::get_help_text(const std::string& full_path) const +{ + attr_line_t retval; + + retval.append(lnav::roles::h2("Property Synopsis")) + .append("\n ") + .append(lnav::roles::symbol(full_path)) + .append(" ") + .append(lnav::roles::variable(this->jph_synopsis)) + .append("\n") + .append(lnav::roles::h2("Description")) + .append("\n ") + .append(this->jph_description) + .append("\n"); + + if (this->jph_enum_values != nullptr) { + retval.append(lnav::roles::h2("Allowed Values")).append("\n "); + + for (int lpc = 0; this->jph_enum_values[lpc].first; lpc++) { + const auto& ev = this->jph_enum_values[lpc]; + + retval.append(lpc == 0 ? "" : ", ") + .append(lnav::roles::symbol(ev.first)); + } + } + + if (!this->jph_examples.empty()) { + retval + .append(lnav::roles::h2( + this->jph_examples.size() == 1 ? "Example" : "Examples")) + .append("\n"); + + for (const auto& ex : this->jph_examples) { + retval.appendf(FMT_STRING(" {}\n"), ex); + } + } + + return retval; +} + +attr_line_t +json_path_handler_base::get_help_text(yajlpp_parse_context* ypc) const +{ + return this->get_help_text(ypc->get_full_path().to_string()); +} + +void +json_path_handler_base::report_min_value_error(yajlpp_parse_context* ypc, + long long value) const +{ + ypc->report_error( + lnav::console::user_message::error( + attr_line_t() + .append_quoted(fmt::to_string(value)) + .append(" is not a valid value for option ") + .append_quoted( + lnav::roles::symbol(ypc->get_full_path().to_string()))) + .with_reason(attr_line_t("value must be greater than or equal to ") + .append(lnav::roles::number( + fmt::to_string(this->jph_min_value)))) + .with_snippet(ypc->get_snippet()) + .with_help(this->get_help_text(ypc))); +} + +void +json_path_handler_base::report_duration_error( + yajlpp_parse_context* ypc, + const std::string& value_str, + const relative_time::parse_error& pe) const +{ + ypc->report_error(lnav::console::user_message::error( + attr_line_t() + .append_quoted(value_str) + .append(" is not a valid duration value " + "for option ") + .append_quoted(lnav::roles::symbol( + ypc->get_full_path().to_string()))) + .with_snippet(ypc->get_snippet()) + .with_reason(pe.pe_msg) + .with_help(this->get_help_text(ypc))); +} + +void +json_path_handler_base::report_enum_error(yajlpp_parse_context* ypc, + const std::string& value_str) const +{ + ypc->report_error(lnav::console::user_message::error( + attr_line_t() + .append_quoted(value_str) + .append(" is not a valid value for option ") + .append_quoted(lnav::roles::symbol( + ypc->get_full_path().to_string()))) + .with_snippet(ypc->get_snippet()) + .with_help(this->get_help_text(ypc))); +} + +void +json_path_handler_base::report_error(yajlpp_parse_context* ypc, + const std::string& value, + lnav::console::user_message um) const +{ + ypc->report_error(um.with_snippet(ypc->get_snippet()) + .with_help(this->get_help_text(ypc))); +} + +void +json_path_container::gen_schema(yajlpp_gen_context& ygc) const +{ + if (!this->jpc_definition_id.empty()) { + ygc.ygc_schema_definitions[this->jpc_definition_id] = this; + + yajl_gen_string(ygc.ygc_handle, "$ref"); + yajl_gen_string(ygc.ygc_handle, + fmt::format(FMT_STRING("#/definitions/{}"), + this->jpc_definition_id)); + return; + } + + this->gen_properties(ygc); +} + +void +json_path_container::gen_properties(yajlpp_gen_context& ygc) const +{ + auto pattern_count = count_if( + this->jpc_children.begin(), this->jpc_children.end(), [](auto& jph) { + return jph.jph_is_pattern_property; + }); + auto plain_count = this->jpc_children.size() - pattern_count; + + if (plain_count > 0) { + yajl_gen_string(ygc.ygc_handle, "properties"); + { + yajlpp_map properties(ygc.ygc_handle); + + for (const auto& child_handler : this->jpc_children) { + if (child_handler.jph_is_pattern_property) { + continue; + } + properties.gen(child_handler.jph_property); + child_handler.gen_schema(ygc); + } + } + } + if (pattern_count > 0) { + yajl_gen_string(ygc.ygc_handle, "patternProperties"); + { + yajlpp_map properties(ygc.ygc_handle); + + for (const auto& child_handler : this->jpc_children) { + if (!child_handler.jph_is_pattern_property) { + continue; + } + properties.gen(child_handler.jph_property); + child_handler.gen_schema(ygc); + } + } + } + + yajl_gen_string(ygc.ygc_handle, "additionalProperties"); + yajl_gen_bool(ygc.ygc_handle, false); +} + +static void +schema_printer(FILE* file, const char* str, size_t len) +{ + fwrite(str, len, 1, file); +} + +void +dump_schema_to(const json_path_container& jpc, const char* internals_dir) +{ + yajlpp_gen genner; + yajlpp_gen_context ygc(genner, jpc); + auto internals_dir_path = ghc::filesystem::path(internals_dir); + auto schema_file_name = ghc::filesystem::path(jpc.jpc_schema_id).filename(); + auto schema_path = internals_dir_path / schema_file_name; + auto file = std::unique_ptr<FILE, decltype(&fclose)>( + fopen(schema_path.c_str(), "w+"), fclose); + + if (!file.get()) { + return; + } + + yajl_gen_config(genner, yajl_gen_beautify, true); + yajl_gen_config( + genner, yajl_gen_print_callback, schema_printer, file.get()); + + ygc.gen_schema(); +} + +string_fragment +yajlpp_gen::to_string_fragment() +{ + const unsigned char* buf; + size_t len; + + yajl_gen_get_buf(this->yg_handle.in(), &buf, &len); + + return string_fragment::from_bytes(buf, len); +} diff --git a/src/yajlpp/yajlpp.hh b/src/yajlpp/yajlpp.hh new file mode 100644 index 0000000..7632329 --- /dev/null +++ b/src/yajlpp/yajlpp.hh @@ -0,0 +1,682 @@ +/** + * Copyright (c) 2013-2019, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file yajlpp.hh + */ + +#ifndef yajlpp_hh +#define yajlpp_hh + +#include <functional> +#include <map> +#include <memory> +#include <set> +#include <stack> +#include <string> +#include <utility> +#include <vector> + +#include <limits.h> +#include <stdarg.h> +#include <string.h> + +#include "base/file_range.hh" +#include "base/intern_string.hh" +#include "base/lnav.console.hh" +#include "base/lnav.console.into.hh" +#include "base/lnav_log.hh" +#include "base/opt_util.hh" +#include "json_ptr.hh" +#include "optional.hpp" +#include "pcrepp/pcre2pp.hh" +#include "relative_time.hh" +#include "yajl/api/yajl_gen.h" +#include "yajl/api/yajl_parse.h" + +inline yajl_gen_status +yajl_gen_pstring(yajl_gen hand, const char* str, size_t len) +{ + if (len == (size_t) -1) { + len = strlen(str); + } + return yajl_gen_string(hand, (const unsigned char*) str, len); +} + +inline yajl_gen_status +yajl_gen_string(yajl_gen hand, const std::string& str) +{ + return yajl_gen_string( + hand, (const unsigned char*) str.c_str(), str.length()); +} + +yajl_gen_status yajl_gen_tree(yajl_gen hand, yajl_val val); + +void yajl_cleanup_tree(yajl_val val); + +template<typename T> +struct positioned_property { + intern_string_t pp_path; + source_location pp_location; + T pp_value; + + lnav::console::snippet to_snippet() const + { + return lnav::console::snippet::from(this->pp_location, ""); + } +}; + +template<typename T, typename... Types> +struct factory_container : public positioned_property<std::shared_ptr<T>> { + template<Types... DefaultArgs> + struct with_default_args : public positioned_property<std::shared_ptr<T>> { + template<typename... Args> + static Result<with_default_args, lnav::console::user_message> from( + intern_string_t src, source_location loc, Args... args) + { + auto from_res = T::from(args..., DefaultArgs...); + + if (from_res.isOk()) { + with_default_args retval; + + retval.pp_path = src; + retval.pp_location = loc; + retval.pp_value = from_res.unwrap().to_shared(); + return Ok(retval); + } + + return Err( + lnav::console::to_user_message(src, from_res.unwrapErr())); + } + }; + + template<typename... Args> + static Result<factory_container, lnav::console::user_message> from( + intern_string_t src, source_location loc, Args... args) + { + auto from_res = T::from(args...); + + if (from_res.isOk()) { + factory_container retval; + + retval.pp_path = src; + retval.pp_location = loc; + retval.pp_value = from_res.unwrap().to_shared(); + return Ok(retval); + } + + return Err(lnav::console::to_user_message(src, from_res.unwrapErr())); + } +}; + +class yajlpp_gen_context; +class yajlpp_parse_context; + +struct yajlpp_provider_context { + lnav::pcre2pp::match_data* ypc_extractor; + size_t ypc_index{0}; + yajlpp_parse_context* ypc_parse_context; + + static constexpr size_t nindex = static_cast<size_t>(-1); + + template<typename T> + intern_string_t get_substr_i(T&& name) const + { + auto cap = (*this->ypc_extractor)[std::forward<T>(name)].value(); + char path[cap.length() + 1]; + size_t len = json_ptr::decode(path, cap.data(), cap.length()); + + return intern_string::lookup(path, len); + } + + template<typename T> + std::string get_substr(T&& name) const + { + auto cap = (*this->ypc_extractor)[std::forward<T>(name)].value(); + char path[cap.length() + 1]; + size_t len = json_ptr::decode(path, cap.data(), cap.length()); + + return {path, len}; + } +}; + +class yajlpp_error : public std::exception { +public: + yajlpp_error(yajl_handle handle, const char* json, size_t len) noexcept + { + auto_mem<unsigned char> yajl_msg; + + yajl_msg = yajl_get_error( + handle, 1, reinterpret_cast<const unsigned char*>(json), len); + this->ye_msg = reinterpret_cast<const char*>(yajl_msg.in()); + } + + const char* what() const noexcept override { return this->ye_msg.c_str(); } + +private: + std::string ye_msg; +}; + +struct json_path_container; + +struct json_path_handler_base { + struct enum_value_t { + template<typename T> + enum_value_t(const char* name, T value) + : first(name), second((unsigned int) value) + { + } + + const char* first; + int second; + }; + + static const enum_value_t ENUM_TERMINATOR; + + explicit json_path_handler_base(const std::string& property); + + explicit json_path_handler_base( + const std::shared_ptr<const lnav::pcre2pp::code>& property_re); + + json_path_handler_base( + std::string property, + const std::shared_ptr<const lnav::pcre2pp::code>& property_re); + + bool is_array() const { return this->jph_is_array; } + + nonstd::optional<int> to_enum_value(const string_fragment& sf) const; + const char* to_enum_string(int value) const; + + template<typename T> + std::enable_if_t<!detail::is_optional<T>::value, const char*> + to_enum_string(T value) const + { + return this->to_enum_string((int) value); + } + + template<typename T> + std::enable_if_t<detail::is_optional<T>::value, const char*> to_enum_string( + T value) const + { + return this->to_enum_string((int) value.value()); + } + + yajl_gen_status gen(yajlpp_gen_context& ygc, yajl_gen handle) const; + yajl_gen_status gen_schema(yajlpp_gen_context& ygc) const; + yajl_gen_status gen_schema_type(yajlpp_gen_context& ygc) const; + void walk( + const std::function< + void(const json_path_handler_base&, const std::string&, void*)>& cb, + void* root = nullptr, + const std::string& base = "/") const; + + enum class schema_type_t : std::uint32_t { + ANY, + BOOLEAN, + INTEGER, + NUMBER, + STRING, + ARRAY, + OBJECT, + }; + + std::vector<schema_type_t> get_types() const; + + std::string jph_property; + std::shared_ptr<const lnav::pcre2pp::code> jph_regex; + yajl_callbacks jph_callbacks{}; + std::function<yajl_gen_status( + yajlpp_gen_context&, const json_path_handler_base&, yajl_gen)> + jph_gen_callback; + std::function<void(yajlpp_parse_context& ypc, + const json_path_handler_base& jph)> + jph_validator; + std::function<void*(void* root, nonstd::optional<std::string> name)> + jph_field_getter; + std::function<void*(const yajlpp_provider_context& pe, void* root)> + jph_obj_provider; + std::function<void(void* root, std::vector<std::string>& paths_out)> + jph_path_provider; + std::function<void(const yajlpp_provider_context& pe, void* root)> + jph_obj_deleter; + std::function<size_t(void* root)> jph_size_provider; + const char* jph_synopsis{""}; + const char* jph_description{""}; + const json_path_container* jph_children{nullptr}; + std::shared_ptr<const lnav::pcre2pp::code> jph_pattern; + const char* jph_pattern_re{nullptr}; + std::function<void(const string_fragment&)> jph_string_validator; + size_t jph_min_length{0}; + size_t jph_max_length{INT_MAX}; + const enum_value_t* jph_enum_values{nullptr}; + long long jph_min_value{LLONG_MIN}; + bool jph_optional_wrapper{false}; + bool jph_is_array; + bool jph_is_pattern_property{false}; + std::vector<std::string> jph_examples; + + std::function<int(yajlpp_parse_context*)> jph_null_cb; + std::function<int(yajlpp_parse_context*, int)> jph_bool_cb; + std::function<int(yajlpp_parse_context*, long long)> jph_integer_cb; + std::function<int(yajlpp_parse_context*, double)> jph_double_cb; + std::function<int( + yajlpp_parse_context*, const unsigned char* str, size_t len)> + jph_str_cb; + + void validate_string(yajlpp_parse_context& ypc, string_fragment sf) const; + + void report_pattern_error(yajlpp_parse_context* ypc, + const std::string& value_str) const; + void report_min_value_error(yajlpp_parse_context* ypc, + long long value) const; + void report_duration_error(yajlpp_parse_context* ypc, + const std::string& value_str, + const relative_time::parse_error& pe) const; + void report_enum_error(yajlpp_parse_context* ypc, + const std::string& value_str) const; + void report_error(yajlpp_parse_context* ypc, + const std::string& value_str, + lnav::console::user_message um) const; + + attr_line_t get_help_text(const std::string& full_path) const; + attr_line_t get_help_text(yajlpp_parse_context* ypc) const; +}; + +struct json_path_handler; + +class yajlpp_parse_context { +public: + using error_reporter_t = std::function<void( + const yajlpp_parse_context& ypc, const lnav::console::user_message&)>; + + yajlpp_parse_context(intern_string_t source, + const struct json_path_container* handlers = nullptr); + + const char* get_path_fragment(int offset, + char* frag_in, + size_t& len_out) const; + + intern_string_t get_path_fragment_i(int offset) const + { + char fragbuf[this->ypc_path.size()]; + const char* frag; + size_t len; + + frag = this->get_path_fragment(offset, fragbuf, len); + return intern_string::lookup(frag, len); + } + + std::string get_path_fragment(int offset) const + { + char fragbuf[this->ypc_path.size()]; + const char* frag; + size_t len; + + frag = this->get_path_fragment(offset, fragbuf, len); + return std::string(frag, len); + } + + const intern_string_t get_path() const; + + const intern_string_t get_full_path() const; + + bool is_level(size_t level) const + { + return this->ypc_path_index_stack.size() == level; + } + + yajlpp_parse_context& set_path(const std::string& path); + + void reset(const struct json_path_container* handlers); + + void set_static_handler(const struct json_path_handler_base& jph); + + template<typename T> + yajlpp_parse_context& with_obj(T& obj) + { + this->ypc_obj_stack.push(&obj); + return *this; + } + + yajlpp_parse_context& with_handle(yajl_handle handle) + { + this->ypc_handle = handle; + return *this; + } + + yajlpp_parse_context& with_error_reporter(error_reporter_t err) + { + this->ypc_error_reporter = err; + return *this; + } + + yajlpp_parse_context& with_ignore_unused(bool ignore) + { + this->ypc_ignore_unused = ignore; + return *this; + } + + yajl_status parse(const unsigned char* jsonText, size_t jsonTextLen); + + yajl_status parse(const string_fragment& sf) + { + return this->parse((const unsigned char*) sf.data(), sf.length()); + } + + int get_line_number() const; + + yajl_status complete_parse(); + + bool parse_doc(const string_fragment& sf); + + void report_error(const lnav::console::user_message& msg) const + { + if (this->ypc_error_reporter) { + this->ypc_error_reporter(*this, msg); + } + } + + lnav::console::snippet get_snippet() const; + + template<typename T> + std::vector<T>& get_lvalue(std::map<std::string, std::vector<T>>& value) + { + return value[this->get_path_fragment(-2)]; + } + + template<typename T> + T& get_lvalue(std::map<std::string, T>& value) + { + return value[this->get_path_fragment(-1)]; + } + + template<typename T> + T& get_lvalue(T& lvalue) + { + return lvalue; + } + + template<typename T> + T& get_rvalue(std::map<std::string, std::vector<T>>& value) + { + return value[this->get_path_fragment(-2)].back(); + } + + template<typename T> + T& get_rvalue(std::map<std::string, T>& value) + { + return value[this->get_path_fragment(-1)]; + } + + template<typename T> + T& get_rvalue(std::vector<T>& value) + { + return value.back(); + } + + template<typename T> + T& get_rvalue(T& lvalue) + { + return lvalue; + } + + template<typename T, typename MEM_T, MEM_T T::*MEM> + auto& get_obj_member() + { + auto obj = (T*) this->ypc_obj_stack.top(); + + return obj->*MEM; + } + + const intern_string_t ypc_source; + int ypc_line_number{1}; + const struct json_path_container* ypc_handlers; + std::stack<void*> ypc_obj_stack; + void* ypc_userdata{nullptr}; + yajl_handle ypc_handle{nullptr}; + const unsigned char* ypc_json_text{nullptr}; + size_t ypc_json_text_len{0}; + size_t ypc_total_consumed{0}; + yajl_callbacks ypc_callbacks; + yajl_callbacks ypc_alt_callbacks; + std::vector<char> ypc_path; + std::vector<size_t> ypc_path_index_stack; + std::vector<size_t> ypc_array_index; + std::vector<const json_path_handler_base*> ypc_handler_stack; + size_t ypc_array_handler_count{0}; + bool ypc_ignore_unused{false}; + const struct json_path_container* ypc_sibling_handlers{nullptr}; + const struct json_path_handler_base* ypc_current_handler{nullptr}; + std::set<std::string> ypc_active_paths; + error_reporter_t ypc_error_reporter{nullptr}; + std::map<intern_string_t, source_location>* ypc_locations{nullptr}; + + void update_callbacks(const json_path_container* handlers = nullptr, + int child_start = 0); + +private: + static const yajl_callbacks DEFAULT_CALLBACKS; + + static int map_start(void* ctx); + static int map_key(void* ctx, const unsigned char* key, size_t len); + static int map_end(void* ctx); + static int array_start(void* ctx); + static int array_end(void* ctx); + static int handle_unused(void* ctx); + static int handle_unused_or_delete(void* ctx); +}; + +class yajlpp_generator { +public: + yajlpp_generator(yajl_gen handle) : yg_handle(handle) {} + + yajl_gen_status operator()(const std::string& str) + { + return yajl_gen_string(this->yg_handle, str); + } + + yajl_gen_status operator()(const char* str) + { + return yajl_gen_string( + this->yg_handle, (const unsigned char*) str, strlen(str)); + } + + yajl_gen_status operator()(const char* str, size_t len) + { + return yajl_gen_string( + this->yg_handle, (const unsigned char*) str, len); + } + + yajl_gen_status operator()(const intern_string_t& str) + { + return yajl_gen_string( + this->yg_handle, (const unsigned char*) str.get(), str.size()); + } + + yajl_gen_status operator()(const string_fragment& str) + { + return yajl_gen_string( + this->yg_handle, (const unsigned char*) str.data(), str.length()); + } + + yajl_gen_status operator()(bool value) + { + return yajl_gen_bool(this->yg_handle, value); + } + + yajl_gen_status operator()(double value) + { + return yajl_gen_double(this->yg_handle, value); + } + + template<typename T> + yajl_gen_status operator()( + T value, + typename std::enable_if<std::is_integral<T>::value + && !std::is_same<T, bool>::value>::type* dummy + = 0) + { + return yajl_gen_integer(this->yg_handle, value); + } + + template<typename T> + yajl_gen_status operator()(nonstd::optional<T> value) + { + if (!value.has_value()) { + return yajl_gen_status_ok; + } + + return (*this)(value.value()); + } + + template<typename T> + yajl_gen_status operator()( + const T& container, + typename std::enable_if<!std::is_integral<T>::value>::type* dummy = 0) + { + yajl_gen_array_open(this->yg_handle); + for (const auto& elem : container) { + yajl_gen_status rc = (*this)(elem); + + if (rc != yajl_gen_status_ok) { + return rc; + } + } + + yajl_gen_array_close(this->yg_handle); + + return yajl_gen_status_ok; + } + + yajl_gen_status operator()() { return yajl_gen_null(this->yg_handle); } + +private: + yajl_gen yg_handle; +}; + +class yajlpp_container_base { +public: + yajlpp_container_base(yajl_gen handle) : gen(handle), ycb_handle(handle) {} + + yajlpp_generator gen; + +protected: + yajl_gen ycb_handle; +}; + +class yajlpp_map : public yajlpp_container_base { +public: + yajlpp_map(yajl_gen handle) : yajlpp_container_base(handle) + { + yajl_gen_map_open(handle); + } + + ~yajlpp_map() { yajl_gen_map_close(this->ycb_handle); } +}; + +class yajlpp_array : public yajlpp_container_base { +public: + yajlpp_array(yajl_gen handle) : yajlpp_container_base(handle) + { + yajl_gen_array_open(handle); + } + + ~yajlpp_array() { yajl_gen_array_close(this->ycb_handle); } +}; + +class yajlpp_gen_context { +public: + yajlpp_gen_context(yajl_gen handle, + const struct json_path_container& handlers) + : ygc_handle(handle), ygc_depth(0), ygc_handlers(&handlers) + { + } + + template<typename T> + yajlpp_gen_context& with_default_obj(T& obj) + { + this->ygc_default_stack.push(&obj); + return *this; + } + + template<typename T> + yajlpp_gen_context& with_obj(const T& obj) + { + this->ygc_obj_stack.push((void*) &obj); + return *this; + } + + yajlpp_gen_context& with_context(yajlpp_parse_context& ypc); + + void gen(); + + void gen_schema(const json_path_container* handlers = nullptr); + + yajl_gen ygc_handle; + int ygc_depth; + std::stack<void*> ygc_default_stack; + std::stack<void*> ygc_obj_stack; + std::vector<std::string> ygc_path; + const json_path_container* ygc_handlers; + std::map<std::string, const json_path_container*> ygc_schema_definitions; +}; + +class yajlpp_gen { +public: + yajlpp_gen() : yg_handle(yajl_gen_free) + { + this->yg_handle = yajl_gen_alloc(nullptr); + } + + yajl_gen get_handle() const { return this->yg_handle.in(); } + + operator yajl_gen() { return this->yg_handle.in(); } + + string_fragment to_string_fragment(); + +private: + auto_mem<yajl_gen_t> yg_handle; +}; + +struct json_string { + explicit json_string(yajl_gen_t* gen) + { + const unsigned char* buf; + + yajl_gen_get_buf(gen, &buf, &this->js_len); + + this->js_content = (const unsigned char*) malloc(this->js_len); + memcpy((void*) this->js_content.in(), buf, this->js_len); + } + + auto_mem<const unsigned char> js_content; + size_t js_len{0}; +}; + +void dump_schema_to(const json_path_container& jpc, const char* internals_dir); + +#endif diff --git a/src/yajlpp/yajlpp_def.hh b/src/yajlpp/yajlpp_def.hh new file mode 100644 index 0000000..b511fa9 --- /dev/null +++ b/src/yajlpp/yajlpp_def.hh @@ -0,0 +1,1388 @@ +/** + * Copyright (c) 2018, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file yajlpp_def.hh + */ + +#ifndef yajlpp_def_hh +#define yajlpp_def_hh + +#include <chrono> + +#include "base/date_time_scanner.hh" +#include "base/time_util.hh" +#include "config.h" +#include "mapbox/variant.hpp" +#include "relative_time.hh" +#include "yajlpp.hh" + +struct json_null_t { + bool operator==(const json_null_t& other) const { return true; } +}; + +using json_any_t + = mapbox::util::variant<json_null_t, bool, int64_t, double, std::string>; + +struct json_path_container; + +struct json_path_handler : public json_path_handler_base { + template<typename P> + json_path_handler(P path, int (*null_func)(yajlpp_parse_context*)) + : json_path_handler_base(path) + { + this->jph_callbacks.yajl_null = (int (*)(void*)) null_func; + } + + template<typename P> + json_path_handler(P path, int (*bool_func)(yajlpp_parse_context*, int)) + : json_path_handler_base(path) + { + this->jph_callbacks.yajl_boolean = (int (*)(void*, int)) bool_func; + } + + template<typename P> + json_path_handler(P path, int (*int_func)(yajlpp_parse_context*, long long)) + : json_path_handler_base(path) + { + this->jph_callbacks.yajl_integer = (int (*)(void*, long long)) int_func; + } + + template<typename P> + json_path_handler(P path, int (*double_func)(yajlpp_parse_context*, double)) + : json_path_handler_base(path) + { + this->jph_callbacks.yajl_double = (int (*)(void*, double)) double_func; + } + + template<typename P> + json_path_handler(P path) : json_path_handler_base(path) + { + } + + template<typename P> + json_path_handler(P path, + int (*str_func)(yajlpp_parse_context*, + const unsigned char*, + size_t)) + : json_path_handler_base(path) + { + this->jph_callbacks.yajl_string + = (int (*)(void*, const unsigned char*, size_t)) str_func; + } + + json_path_handler(const std::string& path, + const std::shared_ptr<const lnav::pcre2pp::code>& re) + : json_path_handler_base(path, re) + { + } + + json_path_handler& add_cb(int (*null_func)(yajlpp_parse_context*)) + { + this->jph_callbacks.yajl_null = (int (*)(void*)) null_func; + return *this; + } + + json_path_handler& add_cb(int (*bool_func)(yajlpp_parse_context*, int)) + { + this->jph_callbacks.yajl_boolean = (int (*)(void*, int)) bool_func; + return *this; + } + + json_path_handler& add_cb(int (*int_func)(yajlpp_parse_context*, long long)) + { + this->jph_callbacks.yajl_integer = (int (*)(void*, long long)) int_func; + return *this; + } + + json_path_handler& add_cb(int (*double_func)(yajlpp_parse_context*, double)) + { + this->jph_callbacks.yajl_double = (int (*)(void*, double)) double_func; + return *this; + } + + json_path_handler& add_cb(int (*str_func)(yajlpp_parse_context*, + const unsigned char*, + size_t)) + { + this->jph_callbacks.yajl_string + = (int (*)(void*, const unsigned char*, size_t)) str_func; + return *this; + } + + json_path_handler& with_synopsis(const char* synopsis) + { + this->jph_synopsis = synopsis; + return *this; + } + + json_path_handler& with_description(const char* description) + { + this->jph_description = description; + return *this; + } + + json_path_handler& with_min_length(size_t len) + { + this->jph_min_length = len; + return *this; + } + + json_path_handler& with_max_length(size_t len) + { + this->jph_max_length = len; + return *this; + } + + json_path_handler& with_enum_values(const enum_value_t values[]) + { + this->jph_enum_values = values; + return *this; + } + + template<typename T, std::size_t N> + json_path_handler& with_pattern(const T (&re)[N]) + { + this->jph_pattern_re = re; + this->jph_pattern = lnav::pcre2pp::code::from_const(re).to_shared(); + return *this; + } + + json_path_handler& with_min_value(long long val) + { + this->jph_min_value = val; + return *this; + } + + template<typename R, typename T> + json_path_handler& with_obj_provider( + R* (*provider)(const yajlpp_provider_context& pc, T* root)) + { + this->jph_obj_provider + = [provider](const yajlpp_provider_context& ypc, void* root) { + return (R*) provider(ypc, (T*) root); + }; + return *this; + } + + template<typename R> + json_path_handler& with_size_provider(size_t (*provider)(const R* root)) + { + this->jph_size_provider + = [provider](const void* root) { return provider((R*) root); }; + return *this; + } + + template<typename T> + json_path_handler& with_path_provider( + void (*provider)(T* root, std::vector<std::string>& paths_out)) + { + this->jph_path_provider + = [provider](void* root, std::vector<std::string>& paths_out) { + provider((T*) root, paths_out); + }; + return *this; + } + + template<typename T> + json_path_handler& with_obj_deleter( + void (*provider)(const yajlpp_provider_context& pc, T* root)) + { + this->jph_obj_deleter + = [provider](const yajlpp_provider_context& ypc, void* root) { + provider(ypc, (T*) root); + }; + return *this; + } + + static int null_field_cb(yajlpp_parse_context* ypc) + { + return ypc->ypc_current_handler->jph_null_cb(ypc); + } + + static int bool_field_cb(yajlpp_parse_context* ypc, int val) + { + return ypc->ypc_current_handler->jph_bool_cb(ypc, val); + } + + static int str_field_cb2(yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) + { + return ypc->ypc_current_handler->jph_str_cb(ypc, str, len); + } + + static int int_field_cb(yajlpp_parse_context* ypc, long long val) + { + return ypc->ypc_current_handler->jph_integer_cb(ypc, val); + } + + static int dbl_field_cb(yajlpp_parse_context* ypc, double val) + { + return ypc->ypc_current_handler->jph_double_cb(ypc, val); + } + + template<typename T, typename U> + static inline U& get_field(T& input, U(T::*field)) + { + return input.*field; + } + + template<typename T, typename U, typename... V> + static inline auto get_field(T& input, U(T::*field), V... args) + -> decltype(get_field(input.*field, args...)) + { + return get_field(input.*field, args...); + } + + template<typename T, typename U, typename... V> + static inline auto get_field(void* input, U(T::*field), V... args) + -> decltype(get_field(*((T*) input), field, args...)) + { + return get_field(*((T*) input), field, args...); + } + + template<typename R, typename T, typename... Args> + struct LastIs { + static constexpr bool value = LastIs<R, Args...>::value; + }; + + template<typename R, typename T> + struct LastIs<R, T> { + static constexpr bool value = false; + }; + + template<typename R, typename T> + struct LastIs<R, R T::*> { + static constexpr bool value = true; + }; + + template<typename T, typename... Args> + struct LastIsEnum { + using value_type = typename LastIsEnum<Args...>::value_type; + static constexpr bool value = LastIsEnum<Args...>::value; + }; + + template<typename T, typename U> + struct LastIsEnum<U T::*> { + using value_type = U; + + static constexpr bool value = std::is_enum<U>::value; + }; + + template<typename T, typename U> + struct LastIsEnum<nonstd::optional<U> T::*> { + using value_type = U; + + static constexpr bool value = std::is_enum<U>::value; + }; + + template<typename T, typename... Args> + struct LastIsInteger { + static constexpr bool value = LastIsInteger<Args...>::value; + }; + + template<typename T, typename U> + struct LastIsInteger<U T::*> { + static constexpr bool value + = std::is_integral<U>::value && !std::is_same<U, bool>::value; + }; + + template<typename T, typename U> + struct LastIsInteger<nonstd::optional<U> T::*> { + static constexpr bool value + = std::is_integral<U>::value && !std::is_same<U, bool>::value; + }; + + template<typename T, typename... Args> + struct LastIsFloat { + static constexpr bool value = LastIsFloat<Args...>::value; + }; + + template<typename T, typename U> + struct LastIsFloat<U T::*> { + static constexpr bool value = std::is_same<U, double>::value; + }; + + template<typename T, typename U> + struct LastIsFloat<nonstd::optional<U> T::*> { + static constexpr bool value = std::is_same<U, double>::value; + }; + + template<typename T, typename... Args> + struct LastIsVector { + using value_type = typename LastIsVector<Args...>::value_type; + static constexpr bool value = LastIsVector<Args...>::value; + }; + + template<typename T, typename U> + struct LastIsVector<std::vector<U> T::*> { + using value_type = U; + static constexpr bool value = true; + }; + + template<typename T, typename U> + struct LastIsVector<U T::*> { + using value_type = void; + static constexpr bool value = false; + }; + + template<typename T> + static bool is_field_set(const nonstd::optional<T>& field) + { + return field.has_value(); + } + + template<typename T> + static bool is_field_set(const T&) + { + return true; + } + + template<typename... Args, + std::enable_if_t<LastIs<bool, Args...>::value, bool> = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(bool_field_cb); + this->jph_bool_cb = [args...](yajlpp_parse_context* ypc, int val) { + auto* obj = ypc->ypc_obj_stack.top(); + + json_path_handler::get_field(obj, args...) = static_cast<bool>(val); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(field); + }; + this->jph_field_getter + = [args...](void* root, nonstd::optional<std::string> name) { + return (void*) &json_path_handler::get_field(root, args...); + }; + + return *this; + } + + template< + typename... Args, + std::enable_if_t<LastIs<std::vector<std::string>, Args...>::value, bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto value_str = std::string((const char*) str, len); + auto jph = ypc->ypc_current_handler; + + jph->validate_string(*ypc, value_str); + json_path_handler::get_field(obj, args...) + .emplace_back(std::move(value_str)); + + return 1; + }; + return *this; + } + + template<typename... Args, + std::enable_if_t<LastIsVector<Args...>::value, bool> = true, + std::enable_if_t< + !std::is_same<typename LastIsVector<Args...>::value_type, + std::string>::value, + bool> + = true> + json_path_handler& for_field(Args... args) + { + this->jph_obj_provider + = [args...](const yajlpp_provider_context& ypc, void* root) { + auto& vec = json_path_handler::get_field(root, args...); + + if (ypc.ypc_index >= vec.size()) { + vec.resize(ypc.ypc_index + 1); + } + + return &vec[ypc.ypc_index]; + }; + this->jph_size_provider = [args...](void* root) { + auto& vec = json_path_handler::get_field(root, args...); + + return vec.size(); + }; + + return *this; + } + + template<typename T, typename U> + json_path_handler& for_child(positioned_property<U>(T::*field)) + { + this->jph_obj_provider + = [field](const yajlpp_provider_context& ypc, void* root) -> void* { + auto& child = json_path_handler::get_field(root, field); + + if (ypc.ypc_parse_context != nullptr && child.pp_path.empty()) { + child.pp_path = ypc.ypc_parse_context->get_full_path(); + } + return &child.pp_value; + }; + + return *this; + } + + template<typename... Args> + json_path_handler& for_child(Args... args) + { + this->jph_obj_provider = [args...](const yajlpp_provider_context& ypc, + void* root) -> void* { + auto& child = json_path_handler::get_field(root, args...); + + return &child; + }; + + return *this; + } + + template<typename... Args, + std::enable_if_t< + LastIs<std::map<std::string, std::string>, Args...>::value, + bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto key = ypc->get_path_fragment(-1); + + json_path_handler::get_field(obj, args...)[key] + = std::string((const char*) str, len); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + { + yajlpp_generator gen(handle); + + for (const auto& pair : field) { + gen(pair.first); + gen(pair.second); + } + } + + return yajl_gen_status_ok; + }; + return *this; + } + + template<typename... Args, + std::enable_if_t< + LastIs<std::map<std::string, nonstd::optional<std::string>>, + Args...>::value, + bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto key = ypc->get_path_fragment(-1); + + json_path_handler::get_field(obj, args...)[key] + = std::string((const char*) str, len); + + return 1; + }; + this->add_cb(null_field_cb); + this->jph_null_cb = [args...](yajlpp_parse_context* ypc) { + auto* obj = ypc->ypc_obj_stack.top(); + auto key = ypc->get_path_fragment(-1); + + json_path_handler::get_field(obj, args...)[key] = nonstd::nullopt; + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + { + yajlpp_generator gen(handle); + + for (const auto& pair : field) { + gen(pair.first); + gen(pair.second); + } + } + + return yajl_gen_status_ok; + }; + return *this; + } + + template<typename... Args, + std::enable_if_t< + LastIs<std::map<std::string, json_any_t>, Args...>::value, + bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(bool_field_cb); + this->jph_bool_cb = [args...](yajlpp_parse_context* ypc, int val) { + auto* obj = ypc->ypc_obj_stack.top(); + auto key = ypc->get_path_fragment(-1); + + json_path_handler::get_field(obj, args...)[key] = val ? true + : false; + + return 1; + }; + this->add_cb(int_field_cb); + this->jph_integer_cb + = [args...](yajlpp_parse_context* ypc, long long val) { + auto* obj = ypc->ypc_obj_stack.top(); + auto key = ypc->get_path_fragment(-1); + + json_path_handler::get_field(obj, args...)[key] + = static_cast<int64_t>(val); + + return 1; + }; + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto* obj = ypc->ypc_obj_stack.top(); + auto key = ypc->get_path_fragment(-1); + + json_path_handler::get_field(obj, args...)[key] + = std::string((const char*) str, len); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + { + yajlpp_generator gen(handle); + + for (const auto& pair : field) { + gen(pair.first); + pair.second.match([&gen](json_null_t v) { gen(); }, + [&gen](bool v) { gen(v); }, + [&gen](int64_t v) { gen(v); }, + [&gen](double v) { gen(v); }, + [&gen](const std::string& v) { gen(v); }); + } + } + + return yajl_gen_status_ok; + }; + return *this; + } + + template<typename... Args, + std::enable_if_t<LastIs<std::string, Args...>::value, bool> = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto value_str = std::string((const char*) str, len); + auto jph = ypc->ypc_current_handler; + + jph->validate_string(*ypc, value_str); + json_path_handler::get_field(obj, args...) = std::move(value_str); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(field); + }; + this->jph_field_getter + = [args...](void* root, nonstd::optional<std::string> name) { + return (void*) &json_path_handler::get_field(root, args...); + }; + return *this; + } + + template<typename... Args, + std::enable_if_t<LastIs<timeval, Args...>::value, bool> = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto jph = ypc->ypc_current_handler; + + date_time_scanner dts; + timeval tv{}; + exttm tm; + + if (dts.scan((char*) str, len, nullptr, &tm, tv) == nullptr) { + ypc->report_error( + lnav::console::user_message::error( + attr_line_t("unrecognized timestamp ") + .append_quoted( + string_fragment::from_bytes(str, len))) + .with_snippet(ypc->get_snippet()) + .with_help(jph->get_help_text(ypc))); + } else { + json_path_handler::get_field(obj, args...) = tv; + } + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + char buf[64]; + + auto buf_len = lnav::strftime_rfc3339( + buf, sizeof(buf), field.tv_sec, field.tv_usec, 'T'); + + return gen(string_fragment::from_bytes(buf, buf_len)); + }; + this->jph_field_getter + = [args...](void* root, nonstd::optional<std::string> name) { + return (void*) &json_path_handler::get_field(root, args...); + }; + return *this; + } + + template< + typename... Args, + std::enable_if_t<LastIs<nonstd::optional<std::string>, Args...>::value, + bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto value_str = std::string((const char*) str, len); + auto jph = ypc->ypc_current_handler; + + jph->validate_string(*ypc, value_str); + json_path_handler::get_field(obj, args...) = std::move(value_str); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (!field) { + return yajl_gen_status_ok; + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(field.value()); + }; + this->jph_field_getter + = [args...](void* root, nonstd::optional<std::string> name) { + return (void*) &json_path_handler::get_field(root, args...); + }; + return *this; + } + + template<typename... Args, + std::enable_if_t< + LastIs<positioned_property<std::string>, Args...>::value, + bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto value_str = std::string((const char*) str, len); + auto jph = ypc->ypc_current_handler; + + jph->validate_string(*ypc, value_str); + auto& field = json_path_handler::get_field(obj, args...); + + field.pp_path = ypc->get_full_path(); + field.pp_location.sl_source = ypc->ypc_source; + field.pp_location.sl_line_number = ypc->get_line_number(); + field.pp_value = std::move(value_str); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field.pp_value == field_def.pp_value) { + return yajl_gen_status_ok; + } + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(field.pp_value); + }; + return *this; + } + + template<typename... Args, + std::enable_if_t<LastIs<intern_string_t, Args...>::value, bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto value_str = std::string((const char*) str, len); + auto jph = ypc->ypc_current_handler; + + jph->validate_string(*ypc, value_str); + json_path_handler::get_field(obj, args...) + = intern_string::lookup(value_str); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(field); + }; + return *this; + } + + template<typename... Args, + std::enable_if_t< + LastIs<positioned_property<intern_string_t>, Args...>::value, + bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto value_str = std::string((const char*) str, len); + auto jph = ypc->ypc_current_handler; + + jph->validate_string(*ypc, value_str); + auto& field = json_path_handler::get_field(obj, args...); + field.pp_path = ypc->get_full_path(); + field.pp_location.sl_source = ypc->ypc_source; + field.pp_location.sl_line_number = ypc->get_line_number(); + field.pp_value = intern_string::lookup(value_str); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field.pp_value == field_def.pp_value) { + return yajl_gen_status_ok; + } + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(field.pp_value); + }; + return *this; + } + + template<typename> + struct int_ { + typedef int type; + }; + template< + typename C, + typename T, + typename int_<decltype(T::from( + intern_string_t{}, source_location{}, string_fragment{}))>::type + = 0, + typename... Args> + json_path_handler& for_field(Args... args, T C::*ptr_arg) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args..., ptr_arg](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto* obj = ypc->ypc_obj_stack.top(); + auto value_frag = string_fragment::from_bytes(str, len); + const auto* jph = ypc->ypc_current_handler; + auto loc = source_location{ypc->ypc_source, ypc->get_line_number()}; + + auto from_res = T::from(ypc->get_full_path(), loc, value_frag); + if (from_res.isErr()) { + jph->report_error( + ypc, value_frag.to_string(), from_res.unwrapErr()); + } else { + json_path_handler::get_field(obj, args..., ptr_arg) + = from_res.unwrap(); + } + + return 1; + }; + return *this; + } + + template<typename... Args, + std::enable_if_t<LastIsInteger<Args...>::value, bool> = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(int_field_cb); + this->jph_integer_cb + = [args...](yajlpp_parse_context* ypc, long long val) { + auto jph = ypc->ypc_current_handler; + auto* obj = ypc->ypc_obj_stack.top(); + + if (val < jph->jph_min_value) { + jph->report_min_value_error(ypc, val); + return 1; + } + + json_path_handler::get_field(obj, args...) = val; + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (!is_field_set(field)) { + return yajl_gen_status_ok; + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(field); + }; + this->jph_field_getter + = [args...](void* root, nonstd::optional<std::string> name) { + return (void*) &json_path_handler::get_field(root, args...); + }; + + return *this; + } + + template<typename... Args, + std::enable_if_t<LastIsFloat<Args...>::value, bool> = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(dbl_field_cb); + this->jph_double_cb = [args...](yajlpp_parse_context* ypc, double val) { + auto jph = ypc->ypc_current_handler; + auto* obj = ypc->ypc_obj_stack.top(); + + if (val < jph->jph_min_value) { + jph->report_min_value_error(ypc, val); + return 1; + } + + json_path_handler::get_field(obj, args...) = val; + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (!is_field_set(field)) { + return yajl_gen_status_ok; + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(field); + }; + this->jph_field_getter + = [args...](void* root, nonstd::optional<std::string> name) { + return (void*) &json_path_handler::get_field(root, args...); + }; + + return *this; + } + + template< + typename... Args, + std::enable_if_t<LastIs<std::chrono::seconds, Args...>::value, bool> + = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto handler = ypc->ypc_current_handler; + auto parse_res = relative_time::from_str( + string_fragment::from_bytes(str, len)); + + if (parse_res.isErr()) { + auto parse_error = parse_res.unwrapErr(); + auto value_str = std::string((const char*) str, len); + + handler->report_duration_error(ypc, value_str, parse_error); + return 1; + } + + json_path_handler::get_field(obj, args...) = std::chrono::seconds( + parse_res.template unwrap().to_timeval().tv_sec); + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(relative_time::from_timeval( + {static_cast<time_t>(field.count()), 0}) + .to_string()); + }; + this->jph_field_getter + = [args...](void* root, nonstd::optional<std::string> name) { + return (void*) &json_path_handler::get_field(root, args...); + }; + return *this; + } + + template<typename... Args, + std::enable_if_t<LastIsEnum<Args...>::value, bool> = true> + json_path_handler& for_field(Args... args) + { + this->add_cb(str_field_cb2); + this->jph_str_cb = [args...](yajlpp_parse_context* ypc, + const unsigned char* str, + size_t len) { + auto obj = ypc->ypc_obj_stack.top(); + auto handler = ypc->ypc_current_handler; + auto res = handler->to_enum_value(string_fragment(str, 0, len)); + + if (res) { + json_path_handler::get_field(obj, args...) + = (typename LastIsEnum<Args...>::value_type) res.value(); + } else { + handler->report_enum_error(ypc, + std::string((const char*) str, len)); + } + + return 1; + }; + this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, + const json_path_handler_base& jph, + yajl_gen handle) { + const auto& field = json_path_handler::get_field( + ygc.ygc_obj_stack.top(), args...); + + if (!ygc.ygc_default_stack.empty()) { + const auto& field_def = json_path_handler::get_field( + ygc.ygc_default_stack.top(), args...); + + if (field == field_def) { + return yajl_gen_status_ok; + } + } + + if (!is_field_set(field)) { + return yajl_gen_status_ok; + } + + if (ygc.ygc_depth) { + yajl_gen_string(handle, jph.jph_property); + } + + yajlpp_generator gen(handle); + + return gen(jph.to_enum_string(field)); + }; + this->jph_field_getter + = [args...](void* root, nonstd::optional<std::string> name) { + return (void*) &json_path_handler::get_field(root, args...); + }; + + return *this; + } + + json_path_handler& with_children(const json_path_container& container); + + json_path_handler& with_example(const std::string& example) + { + this->jph_examples.emplace_back(example); + return *this; + } +}; + +struct json_path_container { + json_path_container(std::initializer_list<json_path_handler> children) + : jpc_children(children) + { + } + + json_path_container& with_definition_id(const std::string& id) + { + this->jpc_definition_id = id; + return *this; + } + + json_path_container& with_schema_id(const std::string& id) + { + this->jpc_schema_id = id; + return *this; + } + + json_path_container& with_description(std::string desc) + { + this->jpc_description = std::move(desc); + return *this; + } + + void gen_schema(yajlpp_gen_context& ygc) const; + + void gen_properties(yajlpp_gen_context& ygc) const; + + std::string jpc_schema_id; + std::string jpc_definition_id; + std::string jpc_description; + std::vector<json_path_handler> jpc_children; +}; + +template<typename T> +class yajlpp_parser { +public: + yajlpp_parser(intern_string_t src, const json_path_container* container) + : yp_parse_context(src, container) + { + this->yp_handle = yajl_alloc(&this->yp_parse_context.ypc_callbacks, + nullptr, + &this->yp_parse_context); + this->yp_parse_context.with_handle(this->yp_handle); + this->yp_parse_context.template with_obj(this->yp_obj); + this->yp_parse_context.ypc_userdata = this; + this->yp_parse_context.with_error_reporter( + [](const auto& ypc, const auto& um) { + auto* yp = static_cast<yajlpp_parser<T>*>(ypc.ypc_userdata); + + yp->yp_errors.template emplace_back(um); + }); + } + + yajlpp_parser& with_ignore_unused(bool value) + { + this->yp_parse_context.with_ignore_unused(value); + + return *this; + } + + Result<T, std::vector<lnav::console::user_message>> of( + const string_fragment& json) + { + if (this->yp_parse_context.parse_doc(json)) { + return Ok(std::move(this->yp_obj)); + } + + return Err(std::move(this->yp_errors)); + } + +private: + yajlpp_parse_context yp_parse_context; + auto_mem<yajl_handle_t> yp_handle{yajl_free}; + std::vector<lnav::console::user_message> yp_errors; + T yp_obj; +}; + +template<typename T> +struct typed_json_path_container : public json_path_container { + typed_json_path_container(std::initializer_list<json_path_handler> children) + : json_path_container(children) + { + } + + typed_json_path_container<T>& with_schema_id2(const std::string& id) + { + this->jpc_schema_id = id; + return *this; + } + + typed_json_path_container<T>& with_description2(std::string desc) + { + this->jpc_description = std::move(desc); + return *this; + } + + yajlpp_parser<T> parser_for(intern_string_t src) const + { + return yajlpp_parser<T>{src, this}; + } + + std::string to_string(const T& obj) const + { + yajlpp_gen gen; + yajlpp_gen_context ygc(gen, *this); + ygc.template with_obj(obj); + ygc.ygc_depth = 1; + ygc.gen(); + + return gen.to_string_fragment().to_string(); + } + + json_string to_json_string(const T& obj) const + { + yajlpp_gen gen; + yajlpp_gen_context ygc(gen, *this); + ygc.template with_obj(obj); + ygc.ygc_depth = 1; + ygc.gen(); + + return json_string{gen.get_handle()}; + } +}; + +namespace yajlpp { +inline json_path_handler +property_handler(const std::string& path) +{ + return {path}; +} + +template<typename T, std::size_t N> +inline json_path_handler +pattern_property_handler(const T (&path)[N]) +{ + return {lnav::pcre2pp::code::from_const(path).to_shared()}; +} + +} // namespace yajlpp + +#endif |