diff options
Diffstat (limited to 'storage/mroonga/vendor/groonga/src')
29 files changed, 11952 insertions, 0 deletions
diff --git a/storage/mroonga/vendor/groonga/src/.gitignore b/storage/mroonga/vendor/groonga/src/.gitignore new file mode 100644 index 00000000..a8278d75 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/.gitignore @@ -0,0 +1,2 @@ +/grntest +/*.exe diff --git a/storage/mroonga/vendor/groonga/src/CMakeLists.txt b/storage/mroonga/vendor/groonga/src/CMakeLists.txt new file mode 100644 index 00000000..17ef3a41 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/CMakeLists.txt @@ -0,0 +1,62 @@ +# Copyright(C) 2012-2013 Brazil +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +include_directories( + ${MRUBY_INCLUDE_DIRS} + ${MESSAGE_PACK_INCLUDE_DIRS} + ) + +add_subdirectory(suggest) + +read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/groonga_sources.am GROONGA_SOURCES) +add_executable(groonga ${GROONGA_SOURCES}) +set_source_files_properties(${GROONGA_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") +target_link_libraries(groonga libgroonga) +install(TARGETS groonga DESTINATION ${BIN_DIR}) + +if(GRN_WITH_MRUBY) + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/grndb_sources.am GRNDB_SOURCES) + add_executable(grndb ${GRNDB_SOURCES}) + set_source_files_properties(${GRNDB_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") + set_source_files_properties(${GRNDB_SOURCES} + PROPERTIES + COMPILE_DEFINITIONS "${MRUBY_DEFINITIONS}") + target_link_libraries(grndb libgroonga) + install(TARGETS grndb DESTINATION ${BIN_DIR}) +endif() + +if(NOT WIN32) + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/grnslap_sources.am GRNSLAP_SOURCES) + add_executable(grnslap ${GRNSLAP_SOURCES}) + set_source_files_properties(${GRNSLAP_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") + target_link_libraries(grnslap libgroonga) + install(TARGETS grnslap DESTINATION ${BIN_DIR}) +endif() + +read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/groonga_benchmark_sources.am + GROONGA_BENCHMARK_SOURCES) +add_executable(groonga-benchmark ${GROONGA_BENCHMARK_SOURCES}) +set_source_files_properties(${GROONGA_BENCHMARK_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") +target_link_libraries(groonga-benchmark libgroonga) +install(TARGETS groonga-benchmark DESTINATION ${BIN_DIR}) + diff --git a/storage/mroonga/vendor/groonga/src/Makefile.am b/storage/mroonga/vendor/groonga/src/Makefile.am new file mode 100644 index 00000000..2ee68723 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/Makefile.am @@ -0,0 +1,67 @@ +SUBDIRS = \ + suggest \ + httpd + +NONEXISTENT_CXX_SOURCE = nonexistent.cpp + +bin_PROGRAMS = groonga groonga-benchmark +noinst_PROGRAMS = grnslap +if WITH_MRUBY +bin_PROGRAMS += grndb +noinst_PROGRAMS += groonga-mruby +endif + +EXTRA_DIST = \ + CMakeLists.txt + +AM_CFLAGS = \ + $(NO_STRICT_ALIASING_CFLAGS) \ + $(COVERAGE_CFLAGS) \ + $(GRN_CFLAGS) \ + $(MESSAGE_PACK_CFLAGS) \ + $(MRUBY_CFLAGS) + +AM_CPPFLAGS = \ + $(MRUBY_CPPFLAGS) + +DEFS += $(GRN_DEFS) + +AM_LDFLAGS = -no-undefined + +DEFAULT_INCLUDES = \ + -I$(top_builddir) \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/lib \ + $(GROONGA_INCLUDEDIR) + +include groonga_sources.am +nodist_EXTRA_groonga_SOURCES = $(NONEXISTENT_CXX_SOURCE) +groonga_CFLAGS = $(AM_CFLAGS) $(LIBEDIT_CFLAGS) +groonga_LDADD = \ + $(top_builddir)/lib/libgroonga.la \ + $(LIBEDIT_LIBS) \ + $(MESSAGE_PACK_LIBS) + +include grnslap_sources.am +nodist_EXTRA_grnslap_SOURCES = $(NONEXISTENT_CXX_SOURCE) +grnslap_LDADD = \ + $(top_builddir)/lib/libgroonga.la \ + $(MESSAGE_PACK_LIBS) + +include groonga_benchmark_sources.am +nodist_EXTRA_groonga_benchmark_SOURCES = $(NONEXISTENT_CXX_SOURCE) +groonga_benchmark_LDADD = \ + $(top_builddir)/lib/libgroonga.la \ + $(MESSAGE_PACK_LIBS) + +include grndb_sources.am +nodist_EXTRA_grndb_SOURCES = $(NONEXISTENT_CXX_SOURCE) +grndb_LDADD = \ + $(top_builddir)/lib/libgroonga.la \ + $(MESSAGE_PACK_LIBS) + +include groonga_mruby_sources.am +nodist_EXTRA_groonga_mruby_SOURCES = $(NONEXISTENT_CXX_SOURCE) +groonga_mruby_LDADD = \ + $(top_builddir)/lib/libgroonga.la \ + $(MESSAGE_PACK_LIBS) diff --git a/storage/mroonga/vendor/groonga/src/grndb.c b/storage/mroonga/vendor/groonga/src/grndb.c new file mode 100644 index 00000000..11f3ff5a --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/grndb.c @@ -0,0 +1,197 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2014-2016 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#ifdef WIN32 +# define GROONGA_MAIN +#endif /* WIN32 */ + +#include <stdio.h> + +#include <grn_mrb.h> +#include <grn_ctx_impl.h> +#include <grn_ctx_impl_mrb.h> + +#include <mruby/variable.h> +#include <mruby/array.h> + +static int +run_command(grn_ctx *ctx, int argc, char **argv) +{ + int exit_code = EXIT_SUCCESS; + grn_mrb_data *data = &(ctx->impl->mrb); + mrb_state *mrb = data->state; + mrb_value mrb_command_line_module; + mrb_value mrb_grndb_class; + + mrb_command_line_module = mrb_const_get(mrb, + mrb_obj_value(data->module), + mrb_intern_cstr(mrb, "CommandLine")); + if (mrb->exc) { + goto exit; + } + + mrb_grndb_class = mrb_const_get(mrb, + mrb_command_line_module, + mrb_intern_cstr(mrb, "Grndb")); + if (mrb->exc) { + goto exit; + } + + { + int i; + mrb_value mrb_argv; + mrb_value mrb_grndb; + mrb_value mrb_result; + + mrb_argv = mrb_ary_new_capa(mrb, argc); + for (i = 0; i < argc; i++) { + mrb_ary_push(mrb, mrb_argv, mrb_str_new_cstr(mrb, argv[i])); + } + mrb_grndb = mrb_funcall(mrb, mrb_grndb_class, "new", 1, mrb_argv); + if (mrb->exc) { + goto exit; + } + + mrb_result = mrb_funcall(mrb, mrb_grndb, "run", 0); + if (mrb->exc) { + goto exit; + } + + if (!mrb_bool(mrb_result)) { + exit_code = EXIT_FAILURE; + } + } + +exit : + if (mrb->exc) { + mrb_print_error(mrb); + exit_code = EXIT_FAILURE; + } + + return exit_code; +} + +static int +run(grn_ctx *ctx, int argc, char **argv) +{ + int exit_code = EXIT_SUCCESS; + const char *grndb_rb = "command_line/grndb.rb"; + grn_mrb_data *data = &(ctx->impl->mrb); + mrb_state *mrb = data->state; + + mrb_gv_set(mrb, mrb_intern_lit(mrb, "$0"), mrb_str_new_cstr(mrb, argv[0])); + + grn_mrb_load(ctx, grndb_rb); + if (ctx->rc != GRN_SUCCESS) { + fprintf(stderr, "Failed to load Ruby script: <%s>: %s", + grndb_rb, ctx->errbuf); + goto exit; + } + + { + int arena_index; + + arena_index = mrb_gc_arena_save(mrb); + exit_code = run_command(ctx, argc, argv); + mrb_gc_arena_restore(mrb, arena_index); + } + +exit : + if (ctx->rc != GRN_SUCCESS) { + exit_code = EXIT_FAILURE; + } + return exit_code; +} + +int +main(int argc, char **argv) +{ + int exit_code = EXIT_SUCCESS; + const char *log_path = GRN_LOG_PATH; + const char *log_level_name = NULL; + + { + int i; + for (i = 1; i < argc; i++) { + const char *arg = argv[i]; + + if (arg[0] != '-') { + continue; + } + + if (arg[1] == '-' && arg[2] == '\0') { + break; + } + +#define log_path_prefix "--log-path" +#define log_level_prefix "--log-level" + if (strcmp(arg, log_path_prefix) == 0) { + if (i + 1 < argc) { + log_path = argv[i + 1]; + i++; + } + } else if (strncmp(arg, + log_path_prefix "=", + strlen(log_path_prefix "=")) == 0) { + log_path = arg + strlen(log_path_prefix "="); + } else if (strcmp(arg, log_level_prefix) == 0) { + if (i + 1 < argc) { + log_level_name = argv[i + 1]; + i++; + } + } else if (strncmp(arg, + log_level_prefix "=", + strlen(log_level_prefix "=")) == 0) { + log_level_name = arg + strlen(log_level_prefix "="); + } +#undef log_path_equal_prefix +#undef log_level_equal_prefix + } + } + + grn_default_logger_set_path(log_path); + if (log_level_name) { + grn_log_level log_level = GRN_LOG_DEFAULT_LEVEL; + if (!grn_log_level_parse(log_level_name, &log_level)) { + fprintf(stderr, "%s: failed to parse log level: <%s>\n", + argv[0], log_level_name); + return EXIT_FAILURE; + } + grn_default_logger_set_max_level(log_level); + } + + if (grn_init() != GRN_SUCCESS) { + return EXIT_FAILURE; + } + + { + grn_ctx ctx; + grn_ctx_init(&ctx, 0); + grn_ctx_impl_mrb_ensure_init(&ctx); + if (ctx.rc == GRN_SUCCESS) { + exit_code = run(&ctx, argc, argv); + } else { + exit_code = EXIT_FAILURE; + } + grn_ctx_fin(&ctx); + } + + grn_fin(); + + return exit_code; +} diff --git a/storage/mroonga/vendor/groonga/src/grndb_sources.am b/storage/mroonga/vendor/groonga/src/grndb_sources.am new file mode 100644 index 00000000..ce2e2bb3 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/grndb_sources.am @@ -0,0 +1,2 @@ +grndb_SOURCES = \ + grndb.c diff --git a/storage/mroonga/vendor/groonga/src/grnslap.c b/storage/mroonga/vendor/groonga/src/grnslap.c new file mode 100644 index 00000000..cdd51853 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/grnslap.c @@ -0,0 +1,375 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2009-2012 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#include <grn_com.h> +#include <grn_ctx_impl.h> +#include <string.h> +#include <stdio.h> +#ifdef HAVE_SYS_WAIT_H +# include <sys/wait.h> +#endif /* HAVE_SYS_WAIT_H */ +#ifndef WIN32 +# include <netinet/in.h> +#endif /* WIN32 */ + +#define DEFAULT_PORT 10041 +#define DEFAULT_HOST "localhost" +#define DEFAULT_MAX_CONCURRENCY 10 +#define DEFAULT_MAX_THROUGHPUT 10000 +#define MAX_DEST 256 + +typedef struct { + const char *host; + uint16_t port; +} grn_slap_dest; + +static int proto = 'g'; +static int verbose = 0; +static int dest_cnt = 0; +static grn_slap_dest dests[MAX_DEST]; +static int max_con = DEFAULT_MAX_CONCURRENCY; +static int max_tp = DEFAULT_MAX_THROUGHPUT; + +#include <stdarg.h> +static void +lprint(grn_ctx *ctx, const char *fmt, ...) +{ + char buf[1024]; + grn_timeval tv; + int len; + va_list argp; + grn_timeval_now(ctx, &tv); + grn_timeval2str(ctx, &tv, buf, 1024); + len = strlen(buf); + buf[len++] = '|'; + va_start(argp, fmt); + vsnprintf(buf + len, 1023 - len, fmt, argp); + va_end(argp); + buf[1023] = '\0'; + puts(buf); +} + +static void +parse_dest(char *deststr, grn_slap_dest *dest) +{ + int p; + char *d; + if ((d = strchr(deststr, ':'))) { + if ((p = atoi(d + 1))) { + *d = '\0'; + dest->host = deststr; + dest->port = p; + return; + } + } + dest->host = NULL; + dest->port = 0; +} + +static void +usage(void) +{ + fprintf(stderr, + "Usage: grnslap [options...] [dest...]\n" + "options:\n" + " -P <protocol>: http or gqtp (default: gqtp)\n" + " -m <max concurrency>: number of max concurrency (default: %d)\n" + "dest: hostname:port number (default: \"%s:%d\")\n", + DEFAULT_MAX_CONCURRENCY, DEFAULT_HOST, DEFAULT_PORT); +} + +#define BUFSIZE 0x1000000 + +typedef struct _session session; + +struct _session { + grn_com_queue_entry eq; + grn_com *com; + struct timeval tv; + grn_id id; + int stat; + int query_id; + int n_query; + int n_sessions; +}; + +static grn_com_event ev; +static grn_com_queue fsessions; +static grn_hash *sessions; +static int done = 0; +static int nsent = 0; +static int nrecv = 0; +static int etime_min = INT32_MAX; +static int etime_max = 0; +static int64_t etime_amount = 0; + +static session * +session_open(grn_ctx *ctx, grn_slap_dest *dest) +{ + grn_id id; + session *s; + grn_com *com; + if (!(com = grn_com_copen(ctx, &ev, dest->host, dest->port))) { return NULL; } + id = grn_hash_add(ctx, sessions, &com->fd, sizeof(grn_sock), (void **)&s, NULL); + com->opaque = s; + s->com = com; + s->id = id; + s->stat = 1; + return s; +} + +static void +session_close(grn_ctx *ctx, session *s) +{ + if (!s->stat) { return; } + grn_com_close(ctx, s->com); + s->stat = 0; + grn_hash_delete_by_id(ctx, sessions, s->id, NULL); +} + +static session * +session_alloc(grn_ctx *ctx, grn_slap_dest *dest) +{ + session *s; + while ((s = (session *)grn_com_queue_deque(ctx, &fsessions))) { + if (s->n_query < 1000000 && !s->com->closed) { return s; } + //session_close(ctx, s); + } + return session_open(ctx, dest); +} + +static void +msg_handler(grn_ctx *ctx, grn_obj *msg) +{ + uint32_t etime; + struct timeval tv; + grn_msg *m = (grn_msg *)msg; + grn_com *com = ((grn_msg *)msg)->u.peer; + session *s = com->opaque; + s->stat = 3; + gettimeofday(&tv, NULL); + etime = (tv.tv_sec - s->tv.tv_sec) * 1000000 + (tv.tv_usec - s->tv.tv_usec); + if (etime > etime_max) { etime_max = etime; } + if (etime < etime_min) { etime_min = etime; } + if (ctx->rc) { m->header.proto = 0; } + switch (m->header.proto) { + case GRN_COM_PROTO_GQTP : + if (GRN_BULK_VSIZE(msg) == 2) { + etime_amount += etime; + } else { + if (verbose) { + GRN_TEXT_PUTC(ctx, msg, '\0'); + lprint(ctx, "%8d(%4d) %8d : %s", s->query_id, s->n_sessions, etime, GRN_BULK_HEAD(msg)); + } + } + if ((m->header.flags & GRN_CTX_TAIL)) { + grn_com_queue_enque(ctx, &fsessions, (grn_com_queue_entry *)s); + nrecv++; + } + break; + case GRN_COM_PROTO_HTTP : + nrecv++; + /* lprint(ctx, "recv: %d, %d", (int)GRN_BULK_VSIZE(msg), nrecv); */ + grn_com_close_(ctx, com); + grn_com_queue_enque(ctx, &fsessions, (grn_com_queue_entry *)s); + break; + default : + grn_com_close_(ctx, com); + grn_com_queue_enque(ctx, &fsessions, (grn_com_queue_entry *)s); + break; + } + grn_msg_close(ctx, msg); +} + +static grn_thread_func_result CALLBACK +receiver(void *arg) +{ + grn_ctx ctx_, *ctx = &ctx_; + grn_ctx_init(ctx, 0); + while (!grn_com_event_poll(ctx, &ev, 100)) { + if (nsent == nrecv && done) { break; } + /* + { + session *s; + GRN_HASH_EACH(ctx, sessions, id, NULL, NULL, &s, { + printf("id=%d: fd=%d stat=%d q=%d n=%d\n", s->id, s->com->fd, s->stat, s->query_id, s->n_query); + }); + } + */ + } + grn_ctx_fin(ctx); + return GRN_THREAD_FUNC_RETURN_VALUE; +} + +static int +do_client() +{ + int rc = -1; + grn_obj text; + grn_thread thread; + struct timeval tvb, tve; + grn_com_header sheader; + grn_ctx ctx_, *ctx = &ctx_; + grn_ctx_init(ctx, 0); + GRN_COM_QUEUE_INIT(&fsessions); + sessions = grn_hash_create(ctx, NULL, sizeof(grn_sock), sizeof(session), 0); + sheader.proto = GRN_COM_PROTO_GQTP; + sheader.qtype = 0; + sheader.keylen = 0; + sheader.level = 0; + sheader.flags = 0; + sheader.status = 0; + sheader.opaque = 0; + sheader.cas = 0; + GRN_TEXT_INIT(&text, 0); + rc = grn_bulk_reserve(ctx, &text, BUFSIZE); + if (!rc) { + char *buf = GRN_TEXT_VALUE(&text); + if (!grn_com_event_init(ctx, &ev, 1000, sizeof(grn_com))) { + ev.msg_handler = msg_handler; + if (!THREAD_CREATE(thread, receiver, NULL)) { + int cnt = 0; + gettimeofday(&tvb, NULL); + lprint(ctx, "begin: procotol=%c max_concurrency=%d max_tp=%d", proto, max_con, max_tp); + while (fgets(buf, BUFSIZE, stdin)) { + uint32_t size = strlen(buf) - 1; + session *s = session_alloc(ctx, dests + (cnt++ % dest_cnt)); + if (s) { + gettimeofday(&s->tv, NULL); + s->n_query++; + s->query_id = ++nsent; + s->n_sessions = (nsent - nrecv); + switch (proto) { + case 'H' : + case 'h' : + if (grn_com_send_http(ctx, s->com, buf, size, 0)) { + fprintf(stderr, "grn_com_send_http failed\n"); + } + s->stat = 2; + /* + lprint(ctx, "sent %04d %04d %d", + s->n_query, s->query_id, s->com->fd); + */ + break; + default : + if (grn_com_send(ctx, s->com, &sheader, buf, size, 0)) { + fprintf(stderr, "grn_com_send failed\n"); + } + break; + } + } else { + fprintf(stderr, "grn_com_copen failed\n"); + } + for (;;) { + gettimeofday(&tve, NULL); + if ((nrecv < max_tp * (tve.tv_sec - tvb.tv_sec)) && + (nsent - nrecv) < max_con) { break; } + /* lprint(ctx, "s:%d r:%d", nsent, nrecv); */ + grn_nanosleep(1000000); + } + if (!(nsent % 1000)) { lprint(ctx, " : %d", nsent); } + } + done = 1; + if (THREAD_JOIN(thread)) { + fprintf(stderr, "THREAD_JOIN failed\n"); + } + gettimeofday(&tve, NULL); + { + double qps; + uint64_t etime = (tve.tv_sec - tvb.tv_sec); + etime *= 1000000; + etime += (tve.tv_usec - tvb.tv_usec); + qps = (double)nsent * 1000000 / etime; + lprint(ctx, "end : n=%d min=%d max=%d avg=%d qps=%f etime=%d.%06d", nsent, etime_min, etime_max, (int)(etime_amount / nsent), qps, etime / 1000000, etime % 1000000); + } + { + session *s; + GRN_HASH_EACH(ctx, sessions, id, NULL, NULL, &s, { + session_close(ctx, s); + }); + } + rc = 0; + } else { + fprintf(stderr, "THREAD_CREATE failed\n"); + } + grn_com_event_fin(ctx, &ev); + } else { + fprintf(stderr, "grn_com_event_init failed\n"); + } + } + grn_obj_unlink(ctx, &text); + grn_hash_close(ctx, sessions); + grn_ctx_fin(ctx); + return rc; +} + +enum { + flag_usage = 1, + flag_verbose = 2 +}; + +int +main(int argc, char **argv) +{ + const char *protostr = NULL, *maxconstr = NULL, *maxtpstr = NULL; + int r, i, flags = 0; + static grn_str_getopt_opt opts[] = { + {'P', NULL, NULL, 0, GETOPT_OP_NONE}, + {'m', NULL, NULL, 0, GETOPT_OP_NONE}, + {'t', NULL, NULL, 0, GETOPT_OP_NONE}, + {'h', NULL, NULL, flag_usage, GETOPT_OP_ON}, + {'v', NULL, NULL, flag_verbose, GETOPT_OP_ON}, + {'\0', NULL, NULL, 0, 0} + }; + opts[0].arg = &protostr; + opts[1].arg = &maxconstr; + opts[2].arg = &maxtpstr; + i = grn_str_getopt(argc, argv, opts, &flags); + if (protostr) { proto = *protostr; } + if (maxconstr) { max_con = atoi(maxconstr); } + if (maxtpstr) { max_tp = atoi(maxtpstr); } + if (flags & flag_verbose) { verbose = 1; } + + if (argc <= i) { + dests[0].host = DEFAULT_HOST; + dests[0].port = DEFAULT_PORT; + dest_cnt = 1; + } else if (i > 0 && argc <= (i + MAX_DEST)){ + for (dest_cnt = 0; i < argc; i++) { + parse_dest(argv[i], &dests[dest_cnt]); + if (dests[dest_cnt].host) { + dest_cnt++; + } + } + if (!dest_cnt) { flags |= flag_usage; } + } else { + /* too much dests */ + flags |= flag_usage; + } + + grn_default_logger_set_path(GRN_LOG_PATH); + + if (grn_init()) { return -1; } + if (flags & flag_usage) { + usage(); r = -1; + } else { + r = do_client(); + } + grn_fin(); + return r; +} diff --git a/storage/mroonga/vendor/groonga/src/grnslap_sources.am b/storage/mroonga/vendor/groonga/src/grnslap_sources.am new file mode 100644 index 00000000..244f5bfc --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/grnslap_sources.am @@ -0,0 +1,2 @@ +grnslap_SOURCES = \ + grnslap.c diff --git a/storage/mroonga/vendor/groonga/src/groonga.c b/storage/mroonga/vendor/groonga/src/groonga.c new file mode 100644 index 00000000..a7f15d13 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/groonga.c @@ -0,0 +1,3763 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2009-2017 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#include <string.h> +#include <stdio.h> +#include <ctype.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <errno.h> + +#ifdef WIN32 +# define GROONGA_MAIN +#endif /* WIN32 */ +#include <grn.h> + +#include <grn_com.h> +#include <grn_ctx_impl.h> +#include <grn_proc.h> +#include <grn_db.h> +#include <grn_util.h> +#include <grn_error.h> + +#ifdef HAVE_SYS_WAIT_H +# include <sys/wait.h> +#endif /* HAVE_SYS_WAIT_H */ +#ifdef HAVE_SYS_SOCKET_H +# include <sys/socket.h> +#endif /* HAVE_SYS_SOCKET_H */ +#ifndef WIN32 +# include <netinet/in.h> +#endif /* WIN32 */ + +#ifdef HAVE_SYS_RESOURCE_H +# include <sys/resource.h> +#endif /* HAVE_SYS_RESOURCE_H */ + +#ifdef HAVE_SYS_SYSCTL_H +# include <sys/sysctl.h> +#endif /* HAVE_SYS_SYSCTL_H */ + +#ifdef WIN32 +# include <io.h> +# include <direct.h> +#else /* WIN32 */ +# include <sys/uio.h> +#endif /* WIN32 */ + +#ifndef USE_MSG_NOSIGNAL +# ifdef MSG_NOSIGNAL +# undef MSG_NOSIGNAL +# endif +# define MSG_NOSIGNAL 0 +#endif /* USE_MSG_NOSIGNAL */ + +#ifndef STDIN_FILENO +# define STDIN_FILENO 0 +#endif /* STDIN_FILENO */ +#ifndef STDOUT_FILENO +# define STDOUT_FILENO 1 +#endif /* STDOUT_FILENO */ +#ifndef STDERR_FILENO +# define STDERR_FILENO 2 +#endif /* STDERR_FILENO */ + +#define DEFAULT_HTTP_PORT 10041 +#define DEFAULT_GQTP_PORT 10043 +#define DEFAULT_DEST "localhost" +#define DEFAULT_MAX_N_FLOATING_THREADS 8 +#define MAX_CON 0x10000 + +#define RLIMIT_NOFILE_MINIMUM 4096 + +static char bind_address[HOST_NAME_MAX + 1]; +static char hostname[HOST_NAME_MAX + 1]; +static int port = DEFAULT_GQTP_PORT; +static int batchmode; +static int number_of_lines = 0; +static int newdb; +static grn_bool is_daemon_mode = GRN_FALSE; +static int (*do_client)(int argc, char **argv); +static int (*do_server)(char *path); +static const char *pid_file_path = NULL; +static const char *input_path = NULL; +static grn_file_reader *input_reader = NULL; +static FILE *output = NULL; +static grn_bool is_memcached_mode = GRN_FALSE; +static const char *memcached_column_name = NULL; + +static int ready_notify_pipe[2]; +#define PIPE_READ 0 +#define PIPE_WRITE 1 + +static grn_encoding encoding; +static const char *windows_event_source_name = "Groonga"; +static grn_bool use_windows_event_log = GRN_FALSE; +static grn_obj http_response_server_line; + +static int +grn_rc_to_exit_code(grn_rc rc) +{ + if (rc == GRN_SUCCESS) { + return EXIT_SUCCESS; + } else { + return EXIT_FAILURE; + } +} + +static void +break_accept_event_loop(grn_ctx *ctx) +{ + grn_com *client; + const char *address; + + if (strcmp(bind_address, "0.0.0.0") == 0) { + address = "127.0.0.1"; + } else if (strcmp(bind_address, "::") == 0) { + address = "::1"; + } else { + address = bind_address; + } + client = grn_com_copen(ctx, NULL, address, port); + if (client) { + grn_com_close(ctx, client); + } +} + +#ifdef GRN_WITH_LIBEDIT +#include <locale.h> +#include <histedit.h> +static EditLine *line_editor = NULL; +static HistoryW *line_editor_history = NULL; +static HistEventW line_editor_history_event; +static char line_editor_history_path[PATH_MAX] = ""; + +static const wchar_t * +line_editor_prompt(EditLine *e __attribute__((unused))) +{ + return L"> "; +} +static const wchar_t * const line_editor_editor = L"emacs"; + +static void +line_editor_init(int argc __attribute__((unused)), char *argv[]) +{ + const char * const HOME_PATH = getenv("HOME"); + const char * const HISTORY_PATH = "/.groonga-history"; + + setlocale(LC_ALL, ""); + + if (strlen(HOME_PATH) + strlen(HISTORY_PATH) < PATH_MAX) { + grn_strcpy(line_editor_history_path, PATH_MAX, HOME_PATH); + grn_strcat(line_editor_history_path, PATH_MAX, HISTORY_PATH); + } else { + line_editor_history_path[0] = '\0'; + } + + line_editor_history = history_winit(); + history_w(line_editor_history, &line_editor_history_event, H_SETSIZE, 200); + if (line_editor_history_path[0]) { + history_w(line_editor_history, &line_editor_history_event, + H_LOAD, line_editor_history_path); + } + + line_editor = el_init(argv[0], stdin, stdout, stderr); + el_wset(line_editor, EL_PROMPT, &line_editor_prompt); + el_wset(line_editor, EL_EDITOR, line_editor_editor); + el_wset(line_editor, EL_HIST, history_w, line_editor_history); + el_source(line_editor, NULL); +} + +static void +line_editor_fin(void) +{ + if (line_editor) { + el_end(line_editor); + if (line_editor_history) { + if (line_editor_history_path[0]) { + history_w(line_editor_history, &line_editor_history_event, + H_SAVE, line_editor_history_path); + } + history_wend(line_editor_history); + } + } +} + +static grn_rc +line_editor_fgets(grn_ctx *ctx, grn_obj *buf) +{ + grn_rc rc = GRN_SUCCESS; + const wchar_t *line; + int nchar; + line = el_wgets(line_editor, &nchar); + if (nchar > 0) { + int i; + char multibyte_buf[MB_CUR_MAX]; + size_t multibyte_len; + mbstate_t ps; + history_w(line_editor_history, &line_editor_history_event, H_ENTER, line); + memset(&ps, 0, sizeof(ps)); + wcrtomb(NULL, L'\0', &ps); + for (i = 0; i < nchar; i++) { + multibyte_len = wcrtomb(multibyte_buf, line[i], &ps); + if (multibyte_len == (size_t)-1) { + GRN_LOG(ctx, GRN_LOG_WARNING, + "[prompt][libedit] failed to read input: %s", strerror(errno)); + rc = GRN_INVALID_ARGUMENT; + } else { + GRN_TEXT_PUT(ctx, buf, multibyte_buf, multibyte_len); + } + } + } else { + rc = GRN_END_OF_DATA; + } + return rc; +} +#endif /* GRN_WITH_LIBEDIT */ + +inline static grn_rc +read_next_line(grn_ctx *ctx, grn_obj *buf) +{ + static int the_first_read = GRN_TRUE; + grn_rc rc = GRN_SUCCESS; + if (!batchmode) { +#ifdef GRN_WITH_LIBEDIT + rc = line_editor_fgets(ctx, buf); +#else + fprintf(stderr, "> "); + fflush(stderr); + rc = grn_file_reader_read_line(ctx, input_reader, buf); +#endif + } else { + rc = grn_file_reader_read_line(ctx, input_reader, buf); + if (rc != GRN_END_OF_DATA) { + number_of_lines++; + } + } + if (the_first_read && GRN_TEXT_LEN(buf) > 0) { + const char bom[] = {0xef, 0xbb, 0xbf}; + if (GRN_CTX_GET_ENCODING(ctx) == GRN_ENC_UTF8 && + GRN_TEXT_LEN(buf) > 3 && !memcmp(GRN_TEXT_VALUE(buf), bom, 3)) { + grn_obj buf_without_bom; + GRN_TEXT_INIT(&buf_without_bom, 0); + GRN_TEXT_PUT(ctx, &buf_without_bom, + GRN_TEXT_VALUE(buf) + 3, GRN_TEXT_LEN(buf) - 3); + GRN_TEXT_SET(ctx, buf, + GRN_TEXT_VALUE(&buf_without_bom), + GRN_TEXT_LEN(&buf_without_bom)); + grn_obj_unlink(ctx, &buf_without_bom); + } + the_first_read = GRN_FALSE; + } + if (GRN_TEXT_LEN(buf) > 0 && + GRN_TEXT_VALUE(buf)[GRN_TEXT_LEN(buf) - 1] == '\n') { + grn_bulk_truncate(ctx, buf, GRN_TEXT_LEN(buf) - 1); + } + if (GRN_TEXT_LEN(buf) > 0 && + GRN_TEXT_VALUE(buf)[GRN_TEXT_LEN(buf) - 1] == '\r') { + grn_bulk_truncate(ctx, buf, GRN_TEXT_LEN(buf) - 1); + } + return rc; +} + +inline static grn_rc +prompt(grn_ctx *ctx, grn_obj *buf) +{ + grn_rc rc = GRN_SUCCESS; + grn_bool need_next_line = GRN_TRUE; + GRN_BULK_REWIND(buf); + while (need_next_line) { + rc = read_next_line(ctx, buf); + if (rc == GRN_SUCCESS && + GRN_TEXT_LEN(buf) > 0 && + GRN_TEXT_VALUE(buf)[GRN_TEXT_LEN(buf) - 1] == '\\') { + grn_bulk_truncate(ctx, buf, GRN_TEXT_LEN(buf) - 1); + need_next_line = GRN_TRUE; + } else { + need_next_line = GRN_FALSE; + } + } + return rc; +} + +static void +output_envelope(grn_ctx *ctx, grn_rc rc, grn_obj *head, grn_obj *body, grn_obj *foot) +{ + grn_output_envelope(ctx, rc, head, body, foot, input_path, number_of_lines); +} + +static void +s_output_raw(grn_ctx *ctx, int flags, FILE *stream) +{ + char *chunk = NULL; + unsigned int chunk_size = 0; + int recv_flags; + + grn_ctx_recv(ctx, &chunk, &chunk_size, &recv_flags); + if (chunk_size > 0) { + fwrite(chunk, 1, chunk_size, stream); + } + + if (flags & GRN_CTX_TAIL) { + grn_obj *command; + + if (grn_ctx_get_output_type(ctx) == GRN_CONTENT_GROONGA_COMMAND_LIST && + chunk_size > 0 && + chunk[chunk_size - 1] != '\n') { + fwrite("\n", 1, 1, stream); + } + fflush(stream); + + command = GRN_CTX_USER_DATA(ctx)->ptr; + GRN_BULK_REWIND(command); + } +} + +static void +s_output_typed(grn_ctx *ctx, int flags, FILE *stream) +{ + if (ctx && ctx->impl && (flags & GRN_CTX_TAIL)) { + char *chunk = NULL; + unsigned int chunk_size = 0; + int recv_flags; + grn_obj body; + grn_obj *command; + + GRN_TEXT_INIT(&body, 0); + grn_ctx_recv(ctx, &chunk, &chunk_size, &recv_flags); + GRN_TEXT_SET(ctx, &body, chunk, chunk_size); + + if (GRN_TEXT_LEN(&body) || ctx->rc) { + grn_obj head, foot; + GRN_TEXT_INIT(&head, 0); + GRN_TEXT_INIT(&foot, 0); + output_envelope(ctx, ctx->rc, &head, &body, &foot); + fwrite(GRN_TEXT_VALUE(&head), 1, GRN_TEXT_LEN(&head), stream); + fwrite(GRN_TEXT_VALUE(&body), 1, GRN_TEXT_LEN(&body), stream); + fwrite(GRN_TEXT_VALUE(&foot), 1, GRN_TEXT_LEN(&foot), stream); + fputc('\n', stream); + fflush(stream); + GRN_OBJ_FIN(ctx, &head); + GRN_OBJ_FIN(ctx, &foot); + } + GRN_OBJ_FIN(ctx, &body); + + command = GRN_CTX_USER_DATA(ctx)->ptr; + GRN_BULK_REWIND(command); + } +} + +static void +s_output(grn_ctx *ctx, int flags, void *arg) +{ + FILE *stream = (FILE *)arg; + + switch (grn_ctx_get_output_type(ctx)) { + case GRN_CONTENT_GROONGA_COMMAND_LIST : + case GRN_CONTENT_NONE : + s_output_raw(ctx, flags, stream); + break; + default : + s_output_typed(ctx, flags, stream); + break; + } +} + +static int +do_alone(int argc, char **argv) +{ + int exit_code = EXIT_FAILURE; + char *path = NULL; + grn_obj *db; + grn_ctx ctx_, *ctx = &ctx_; + grn_ctx_init(ctx, 0); + if (argc > 0 && argv) { path = *argv++; argc--; } + db = (newdb || !path) ? grn_db_create(ctx, path, NULL) : grn_db_open(ctx, path); + if (db) { + grn_obj command; + GRN_TEXT_INIT(&command, 0); + GRN_CTX_USER_DATA(ctx)->ptr = &command; + grn_ctx_recv_handler_set(ctx, s_output, output); + if (!argc) { + grn_obj text; + GRN_TEXT_INIT(&text, 0); + while (prompt(ctx, &text) != GRN_END_OF_DATA) { + GRN_TEXT_PUT(ctx, &command, GRN_TEXT_VALUE(&text), GRN_TEXT_LEN(&text)); + grn_ctx_send(ctx, GRN_TEXT_VALUE(&text), GRN_TEXT_LEN(&text), 0); + if (ctx->stat == GRN_CTX_QUIT) { break; } + } + exit_code = grn_rc_to_exit_code(ctx->rc); + grn_obj_unlink(ctx, &text); + } else { + grn_rc rc; + rc = grn_ctx_sendv(ctx, argc, argv, 0); + exit_code = grn_rc_to_exit_code(rc); + } + grn_obj_unlink(ctx, &command); + grn_obj_close(ctx, db); + } else { + fprintf(stderr, "db open failed (%s): %s\n", path, ctx->errbuf); + } + grn_ctx_fin(ctx); + return exit_code; +} + +static int +c_output(grn_ctx *ctx) +{ + int flags; + char *str; + unsigned int str_len; + do { + grn_ctx_recv(ctx, &str, &str_len, &flags); + /* + if (ctx->rc) { + fprintf(stderr, "grn_ctx_recv failed\n"); + return -1; + } + */ + if (str_len || ctx->rc) { + grn_obj head, body, foot; + GRN_TEXT_INIT(&head, 0); + GRN_TEXT_INIT(&body, GRN_OBJ_DO_SHALLOW_COPY); + GRN_TEXT_INIT(&foot, 0); + if (ctx->rc == GRN_SUCCESS) { + GRN_TEXT_SET(ctx, &body, str, str_len); + } else { + ERR(ctx->rc, "%.*s", str_len, str); + } + output_envelope(ctx, ctx->rc, &head, &body, &foot); + fwrite(GRN_TEXT_VALUE(&head), 1, GRN_TEXT_LEN(&head), output); + fwrite(GRN_TEXT_VALUE(&body), 1, GRN_TEXT_LEN(&body), output); + fwrite(GRN_TEXT_VALUE(&foot), 1, GRN_TEXT_LEN(&foot), output); + fputc('\n', output); + fflush(output); + GRN_OBJ_FIN(ctx, &head); + GRN_OBJ_FIN(ctx, &body); + GRN_OBJ_FIN(ctx, &foot); + } + } while ((flags & GRN_CTX_MORE)); + return 0; +} + +static int +g_client(int argc, char **argv) +{ + int exit_code = EXIT_FAILURE; + grn_ctx ctx_, *ctx = &ctx_; + const char *hostname = DEFAULT_DEST; + if (argc > 0 && argv) { hostname = *argv++; argc--; } + grn_ctx_init(ctx, 0); + if (!grn_ctx_connect(ctx, hostname, port, 0)) { + if (!argc) { + grn_obj text; + GRN_TEXT_INIT(&text, 0); + while (prompt(ctx, &text) != GRN_END_OF_DATA) { + grn_ctx_send(ctx, GRN_TEXT_VALUE(&text), GRN_TEXT_LEN(&text), 0); + exit_code = grn_rc_to_exit_code(ctx->rc); + if (ctx->rc != GRN_SUCCESS) { break; } + if (c_output(ctx)) { goto exit; } + if (ctx->stat == GRN_CTX_QUIT) { break; } + } + grn_obj_unlink(ctx, &text); + } else { + grn_rc rc; + rc = grn_ctx_sendv(ctx, argc, argv, 0); + exit_code = grn_rc_to_exit_code(rc); + if (c_output(ctx)) { goto exit; } + } + } else { + fprintf(stderr, "grn_ctx_connect failed (%s:%d)\n", hostname, port); + } +exit : + grn_ctx_fin(ctx); + return exit_code; +} + +/* server */ + +typedef void (*grn_edge_dispatcher_func)(grn_ctx *ctx, grn_edge *edge); +typedef void (*grn_handler_func)(grn_ctx *ctx, grn_obj *msg); + +static grn_com_queue ctx_new; +static grn_com_queue ctx_old; +static grn_mutex q_mutex; +static grn_cond q_cond; +static uint32_t n_running_threads = 0; +static uint32_t n_floating_threads = 0; +static uint32_t max_n_floating_threads; + +static uint32_t +groonga_get_thread_limit(void *data) +{ + return max_n_floating_threads; +} + +static void +groonga_set_thread_limit(uint32_t new_limit, void *data) +{ + uint32_t i; + uint32_t current_n_floating_threads; + static uint32_t n_changing_threads = 0; + uint32_t prev_n_changing_threads; + + GRN_ATOMIC_ADD_EX(&n_changing_threads, 1, prev_n_changing_threads); + + MUTEX_LOCK_ENSURE(&grn_gctx, q_mutex); + current_n_floating_threads = n_floating_threads; + max_n_floating_threads = new_limit; + MUTEX_UNLOCK(q_mutex); + + if (prev_n_changing_threads > 0) { + GRN_ATOMIC_ADD_EX(&n_changing_threads, -1, prev_n_changing_threads); + return; + } + + if (current_n_floating_threads > new_limit) { + for (i = 0; i < current_n_floating_threads; i++) { + MUTEX_LOCK_ENSURE(&grn_gctx, q_mutex); + COND_SIGNAL(q_cond); + MUTEX_UNLOCK(q_mutex); + } + } + + while (GRN_TRUE) { + grn_bool is_reduced; + MUTEX_LOCK_ENSURE(&grn_gctx, q_mutex); + is_reduced = (n_running_threads <= max_n_floating_threads); + if (!is_reduced && n_floating_threads > 0) { + COND_SIGNAL(q_cond); + } + MUTEX_UNLOCK(q_mutex); + if (is_reduced) { + break; + } + grn_nanosleep(1000000); + } + + GRN_ATOMIC_ADD_EX(&n_changing_threads, -1, prev_n_changing_threads); +} + +typedef struct { + grn_mutex mutex; + grn_ctx ctx; + grn_pat *entries; + uint64_t earliest_unix_time_msec; +} request_timer_data; +static request_timer_data the_request_timer_data; + +static void * +request_timer_register(const char *request_id, + unsigned int request_id_size, + double timeout, + void *user_data) +{ + request_timer_data *data = user_data; + grn_id id = GRN_ID_NIL; + + { + grn_ctx *ctx = &(data->ctx); + grn_bool is_first_timer; + grn_timeval tv; + uint64_t timeout_unix_time_msec; + void *value; + + MUTEX_LOCK(data->mutex); + is_first_timer = (grn_pat_size(ctx, data->entries) == 0); + grn_timeval_now(ctx, &tv); + timeout_unix_time_msec = GRN_TIMEVAL_TO_MSEC(&tv) + (timeout * 1000); + while (GRN_TRUE) { + int added; + id = grn_pat_add(ctx, data->entries, + &timeout_unix_time_msec, sizeof(uint64_t), + &value, &added); + if (added != 0) { + break; + } + timeout_unix_time_msec++; + } + grn_memcpy(value, &request_id_size, sizeof(unsigned int)); + grn_memcpy(((uint8_t *)value) + sizeof(unsigned int), + request_id, request_id_size); + if (data->earliest_unix_time_msec == 0 || + data->earliest_unix_time_msec > timeout_unix_time_msec) { + data->earliest_unix_time_msec = timeout_unix_time_msec; + } + if (is_first_timer) { + break_accept_event_loop(ctx); + } + MUTEX_UNLOCK(data->mutex); + } + + return (void *)(uint64_t)id; +} + +static void +request_timer_unregister(void *timer_id, + void *user_data) +{ + request_timer_data *data = user_data; + grn_id id = (grn_id)(uint64_t)timer_id; + + { + grn_ctx *ctx = &(data->ctx); + uint64_t timeout_unix_time_msec; + int key_size; + + MUTEX_LOCK(data->mutex); + key_size = grn_pat_get_key(ctx, + data->entries, + id, + &timeout_unix_time_msec, + sizeof(uint64_t)); + if (key_size > 0) { + grn_pat_delete_by_id(ctx, data->entries, id, NULL); + if (data->earliest_unix_time_msec >= timeout_unix_time_msec) { + data->earliest_unix_time_msec = 0; + } + } + MUTEX_UNLOCK(data->mutex); + } +} + +static void +request_timer_fin(void *user_data) +{ + request_timer_data *data = user_data; + + { + grn_ctx *ctx = &(data->ctx); + grn_pat_close(ctx, data->entries); + grn_ctx_fin(ctx); + MUTEX_FIN(data->mutex); + } +} + +static void +request_timer_init(void) +{ + static grn_request_timer timer; + request_timer_data *data = &the_request_timer_data; + grn_ctx *ctx; + + MUTEX_INIT(data->mutex); + ctx = &(data->ctx); + grn_ctx_init(ctx, 0); + data->entries = grn_pat_create(ctx, + NULL, + sizeof(uint64_t), + GRN_TABLE_MAX_KEY_SIZE, + GRN_OBJ_KEY_UINT); + data->earliest_unix_time_msec = 0; + + timer.user_data = data; + timer.register_func = request_timer_register; + timer.unregister_func = request_timer_unregister; + timer.fin_func = request_timer_fin; + + grn_request_timer_set(&timer); +} + +static grn_bool +request_timer_ensure_earliest_unix_time_msec(void) +{ + request_timer_data *data = &the_request_timer_data; + grn_ctx *ctx; + grn_pat_cursor *cursor; + + if (data->earliest_unix_time_msec > 0) { + return GRN_TRUE; + } + + ctx = &(data->ctx); + cursor = grn_pat_cursor_open(ctx, data->entries, + NULL, 0, + NULL, 0, + 0, 1, GRN_CURSOR_ASCENDING); + if (!cursor) { + return GRN_FALSE; + } + while (grn_pat_cursor_next(ctx, cursor) != GRN_ID_NIL) { + void *key; + uint64_t timeout_unix_time_msec; + + grn_pat_cursor_get_key(ctx, cursor, &key); + timeout_unix_time_msec = *(uint64_t *)key; + data->earliest_unix_time_msec = timeout_unix_time_msec; + break; + } + grn_pat_cursor_close(ctx, cursor); + + return data->earliest_unix_time_msec > 0; +} + +static int +request_timer_get_poll_timeout(void) +{ + request_timer_data *data = &the_request_timer_data; + int timeout = 1000; + grn_ctx *ctx; + grn_timeval tv; + + MUTEX_LOCK(data->mutex); + ctx = &(data->ctx); + if (grn_pat_size(ctx, data->entries) == 0) { + goto exit; + } + + if (!request_timer_ensure_earliest_unix_time_msec()) { + goto exit; + } + + grn_timeval_now(ctx, &tv); + timeout = data->earliest_unix_time_msec - GRN_TIMEVAL_TO_MSEC(&tv); + if (timeout < 0) { + timeout = 0; + } else if (timeout > 1000) { + timeout = 1000; + } + +exit : + MUTEX_UNLOCK(data->mutex); + + return timeout; +} + +static void +request_timer_process_timeout(void) +{ + request_timer_data *data = &the_request_timer_data; + grn_ctx *ctx; + grn_timeval tv; + uint64_t max; + grn_pat_cursor *cursor; + + ctx = &(data->ctx); + if (grn_pat_size(ctx, data->entries) == 0) { + return; + } + + grn_timeval_now(ctx, &tv); + max = GRN_TIMEVAL_TO_MSEC(&tv); + cursor = grn_pat_cursor_open(ctx, data->entries, + NULL, 0, + &max, sizeof(uint64_t), + 0, -1, GRN_CURSOR_ASCENDING); + if (!cursor) { + return; + } + + grn_id id; + while ((id = grn_pat_cursor_next(ctx, cursor)) != GRN_ID_NIL) { + void *value; + const char *request_id; + unsigned int request_id_size; + + grn_pat_cursor_get_value(ctx, cursor, &value); + request_id_size = *((unsigned int *)value); + request_id = (const char *)(((uint8_t *)value) + sizeof(unsigned int)); + grn_request_canceler_cancel(request_id, request_id_size); + } + grn_pat_cursor_close(ctx, cursor); +} + +static void +reset_ready_notify_pipe(void) +{ + ready_notify_pipe[PIPE_READ] = 0; + ready_notify_pipe[PIPE_WRITE] = 0; +} + +static void +close_ready_notify_pipe(void) +{ + if (ready_notify_pipe[PIPE_READ] > 0) { + close(ready_notify_pipe[PIPE_READ]); + } + if (ready_notify_pipe[PIPE_WRITE] > 0) { + close(ready_notify_pipe[PIPE_WRITE]); + } + reset_ready_notify_pipe(); +} + +static void +send_ready_notify(void) +{ + if (ready_notify_pipe[PIPE_WRITE] > 0) { + const char *ready_notify_message = "ready"; + write(ready_notify_pipe[PIPE_WRITE], + ready_notify_message, + strlen(ready_notify_message)); + } + close_ready_notify_pipe(); +} + +static void +create_pid_file(void) +{ + FILE *pid_file = NULL; + + if (!pid_file_path) { + return; + } + + pid_file = fopen(pid_file_path, "w"); + if (!pid_file) { + fprintf(stderr, + "Failed to open PID file: <%s>: <%s>\n", + pid_file_path, grn_strerror(errno)); + return; + } + + { +#ifdef WIN32 + DWORD pid; + pid = GetCurrentProcessId(); + fprintf(pid_file, "%" GRN_FMT_DWORD "\n", pid); +#else /* WIN32 */ + pid_t pid; + pid = grn_getpid(); + fprintf(pid_file, "%d\n", pid); +#endif /* WIN32 */ + } + fclose(pid_file); +} + +static void +clean_pid_file(void) +{ + if (pid_file_path) { + grn_unlink(pid_file_path); + } +} + +static int +daemonize(void) +{ + int exit_code = EXIT_SUCCESS; +#ifndef WIN32 + + if (pipe(ready_notify_pipe) == -1) { + reset_ready_notify_pipe(); + } + + switch (fork()) { + case 0: + break; + case -1: + perror("fork"); + return EXIT_FAILURE; + default: + wait(NULL); + if (ready_notify_pipe[PIPE_READ] > 0) { + int max_fd; + fd_set read_fds; + FD_ZERO(&read_fds); + FD_SET(ready_notify_pipe[PIPE_READ], &read_fds); + max_fd = ready_notify_pipe[PIPE_READ] + 1; + select(max_fd, &read_fds, NULL, NULL, NULL); + } + close_ready_notify_pipe(); + _exit(EXIT_SUCCESS); + } + switch (fork()) { + case 0: + if (pid_file_path) { + create_pid_file(); + } else { + pid_t pid; + pid = grn_getpid(); + fprintf(stderr, "%d\n", pid); + } + break; + case -1: + perror("fork"); + return EXIT_FAILURE; + default: + close_ready_notify_pipe(); + _exit(EXIT_SUCCESS); + } + { + int null_fd; + grn_open(null_fd, "/dev/null", O_RDWR); + if (null_fd != -1) { + dup2(null_fd, STDIN_FILENO); + dup2(null_fd, STDOUT_FILENO); + dup2(null_fd, STDERR_FILENO); + if (null_fd > STDERR_FILENO) { grn_close(null_fd); } + } + } +#endif /* WIN32 */ + return exit_code; +} + +static void +run_server_loop(grn_ctx *ctx, grn_com_event *ev) +{ + request_timer_init(); + while (!grn_com_event_poll(ctx, ev, request_timer_get_poll_timeout()) && + grn_gctx.stat != GRN_CTX_QUIT) { + grn_edge *edge; + while ((edge = (grn_edge *)grn_com_queue_deque(ctx, &ctx_old))) { + grn_obj *msg; + while ((msg = (grn_obj *)grn_com_queue_deque(ctx, &edge->send_old))) { + grn_msg_close(&edge->ctx, msg); + } + while ((msg = (grn_obj *)grn_com_queue_deque(ctx, &edge->recv_new))) { + grn_msg_close(ctx, msg); + } + grn_ctx_fin(&edge->ctx); + if (edge->com->has_sid && edge->com->opaque == edge) { + grn_com_close(ctx, edge->com); + } + grn_edges_delete(ctx, edge); + } + request_timer_process_timeout(); + /* todo : log stat */ + } + for (;;) { + MUTEX_LOCK_ENSURE(ctx, q_mutex); + if (n_running_threads == n_floating_threads) { break; } + MUTEX_UNLOCK(q_mutex); + grn_nanosleep(1000000); + } + { + grn_edge *edge; + GRN_HASH_EACH(ctx, grn_edges, id, NULL, NULL, &edge, { + grn_obj *obj; + while ((obj = (grn_obj *)grn_com_queue_deque(ctx, &edge->send_old))) { + grn_msg_close(&edge->ctx, obj); + } + while ((obj = (grn_obj *)grn_com_queue_deque(ctx, &edge->recv_new))) { + grn_msg_close(ctx, obj); + } + grn_ctx_fin(&edge->ctx); + if (edge->com->has_sid) { + grn_com_close(ctx, edge->com); + } + grn_edges_delete(ctx, edge); + }); + } + { + grn_com *com; + GRN_HASH_EACH(ctx, ev->hash, id, NULL, NULL, &com, { grn_com_close(ctx, com); }); + } +} + +static int +run_server(grn_ctx *ctx, grn_obj *db, grn_com_event *ev, + grn_edge_dispatcher_func dispatcher, grn_handler_func handler) +{ + int exit_code = EXIT_SUCCESS; + struct hostent *he; + if (!(he = gethostbyname(hostname))) { + send_ready_notify(); + SOERR("gethostbyname"); + } else { + ev->opaque = db; + grn_edges_init(ctx, dispatcher); + if (!grn_com_sopen(ctx, ev, bind_address, port, handler, he)) { + send_ready_notify(); + run_server_loop(ctx, ev); + exit_code = EXIT_SUCCESS; + } else { + send_ready_notify(); + fprintf(stderr, "grn_com_sopen failed (%s:%d): %s\n", + bind_address, port, ctx->errbuf); + } + grn_edges_fin(ctx); + } + return exit_code; +} + +static grn_bool memcached_init(grn_ctx *ctx); + +static int +start_service(grn_ctx *ctx, const char *db_path, + grn_edge_dispatcher_func dispatcher, grn_handler_func handler) +{ + int exit_code = EXIT_SUCCESS; + grn_com_event ev; + + if (is_daemon_mode) { + exit_code = daemonize(); + if (exit_code != EXIT_SUCCESS) { + return exit_code; + } + } else { + create_pid_file(); + } + + if (!grn_com_event_init(ctx, &ev, MAX_CON, sizeof(grn_com))) { + grn_obj *db; + db = (newdb || !db_path) ? grn_db_create(ctx, db_path, NULL) : grn_db_open(ctx, db_path); + if (db) { + if (is_memcached_mode) { + if (!memcached_init(ctx)) { + fprintf(stderr, "failed to initialize memcached mode: %s\n", + ctx->errbuf); + exit_code = EXIT_FAILURE; + send_ready_notify(); + } + } + if (exit_code == EXIT_SUCCESS) { + exit_code = run_server(ctx, db, &ev, dispatcher, handler); + } + grn_obj_close(ctx, db); + } else { + fprintf(stderr, "db open failed (%s): %s\n", db_path, ctx->errbuf); + exit_code = EXIT_FAILURE; + send_ready_notify(); + } + grn_com_event_fin(ctx, &ev); + } else { + fprintf(stderr, "grn_com_event_init failed\n"); + exit_code = EXIT_FAILURE; + send_ready_notify(); + } + + clean_pid_file(); + + return exit_code; +} + +typedef struct { + grn_msg *msg; + grn_bool in_body; + grn_bool is_chunked; +} ht_context; + +static void +h_output_set_header(grn_ctx *ctx, + grn_obj *header, + grn_rc rc, + long long int content_length, + grn_obj *foot) +{ + switch (rc) { + case GRN_SUCCESS : + GRN_TEXT_SETS(ctx, header, "HTTP/1.1 200 OK\r\n"); + break; + case GRN_INVALID_ARGUMENT : + case GRN_FUNCTION_NOT_IMPLEMENTED : + case GRN_SYNTAX_ERROR : + GRN_TEXT_SETS(ctx, header, "HTTP/1.1 400 Bad Request\r\n"); + break; + case GRN_NO_SUCH_FILE_OR_DIRECTORY : + GRN_TEXT_SETS(ctx, header, "HTTP/1.1 404 Not Found\r\n"); + break; + case GRN_CANCEL : + GRN_TEXT_SETS(ctx, header, "HTTP/1.1 408 Request Timeout\r\n"); + break; + default : + GRN_TEXT_SETS(ctx, header, "HTTP/1.1 500 Internal Server Error\r\n"); + break; + } + GRN_TEXT_PUT(ctx, header, + GRN_TEXT_VALUE(&http_response_server_line), + GRN_TEXT_LEN(&http_response_server_line)); + GRN_TEXT_PUTS(ctx, header, "Content-Type: "); + if (grn_ctx_get_output_type(ctx) == GRN_CONTENT_JSON && + foot && + GRN_TEXT_LEN(foot) > 0 && + GRN_TEXT_VALUE(foot)[GRN_TEXT_LEN(foot) - 1] == ';') { + GRN_TEXT_PUTS(ctx, header, "application/javascript"); + } else { + GRN_TEXT_PUTS(ctx, header, grn_ctx_get_mime_type(ctx)); + } + GRN_TEXT_PUTS(ctx, header, "\r\n"); + if (content_length >= 0) { + GRN_TEXT_PUTS(ctx, header, "Connection: close\r\n"); + GRN_TEXT_PUTS(ctx, header, "Content-Length: "); + grn_text_lltoa(ctx, header, content_length); + GRN_TEXT_PUTS(ctx, header, "\r\n"); + } else { + GRN_TEXT_PUTS(ctx, header, "Transfer-Encoding: chunked\r\n"); + } + GRN_TEXT_PUTS(ctx, header, "\r\n"); +} + +static void +h_output_send(grn_ctx *ctx, grn_sock fd, + grn_obj *header, grn_obj *head, grn_obj *body, grn_obj *foot) +{ + ssize_t ret; + ssize_t len = 0; +#ifdef WIN32 + int n_buffers = 0; + WSABUF wsabufs[4]; + if (header) { + wsabufs[n_buffers].buf = GRN_TEXT_VALUE(header); + wsabufs[n_buffers].len = GRN_TEXT_LEN(header); + len += GRN_TEXT_LEN(header); + n_buffers++; + } + if (head) { + wsabufs[n_buffers].buf = GRN_TEXT_VALUE(head); + wsabufs[n_buffers].len = GRN_TEXT_LEN(head); + len += GRN_TEXT_LEN(head); + n_buffers++; + } + if (body) { + wsabufs[n_buffers].buf = GRN_TEXT_VALUE(body); + wsabufs[n_buffers].len = GRN_TEXT_LEN(body); + len += GRN_TEXT_LEN(body); + n_buffers++; + } + if (foot) { + wsabufs[n_buffers].buf = GRN_TEXT_VALUE(foot); + wsabufs[n_buffers].len = GRN_TEXT_LEN(foot); + len += GRN_TEXT_LEN(foot); + n_buffers++; + } + { + DWORD sent; + if (WSASend(fd, wsabufs, n_buffers, &sent, 0, NULL, NULL) == SOCKET_ERROR) { + SOERR("WSASend"); + } + ret = sent; + } +#else /* WIN32 */ + struct iovec msg_iov[4]; + struct msghdr msg; + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = msg_iov; + msg.msg_iovlen = 0; + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_flags = 0; + + if (header) { + msg_iov[msg.msg_iovlen].iov_base = GRN_TEXT_VALUE(header); + msg_iov[msg.msg_iovlen].iov_len = GRN_TEXT_LEN(header); + len += GRN_TEXT_LEN(header); + msg.msg_iovlen++; + } + if (head) { + msg_iov[msg.msg_iovlen].iov_base = GRN_TEXT_VALUE(head); + msg_iov[msg.msg_iovlen].iov_len = GRN_TEXT_LEN(head); + len += GRN_TEXT_LEN(head); + msg.msg_iovlen++; + } + if (body) { + msg_iov[msg.msg_iovlen].iov_base = GRN_TEXT_VALUE(body); + msg_iov[msg.msg_iovlen].iov_len = GRN_TEXT_LEN(body); + len += GRN_TEXT_LEN(body); + msg.msg_iovlen++; + } + if (foot) { + msg_iov[msg.msg_iovlen].iov_base = GRN_TEXT_VALUE(foot); + msg_iov[msg.msg_iovlen].iov_len = GRN_TEXT_LEN(foot); + len += GRN_TEXT_LEN(foot); + msg.msg_iovlen++; + } + if ((ret = sendmsg(fd, &msg, MSG_NOSIGNAL)) == -1) { + SOERR("sendmsg"); + } +#endif /* WIN32 */ + if (ret != len) { + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, + "couldn't send all data (%" GRN_FMT_LLD "/%" GRN_FMT_LLD ")", + (long long int)ret, (long long int)len); + } +} + +static void +h_output_raw(grn_ctx *ctx, int flags, ht_context *hc) +{ + grn_rc expr_rc = ctx->rc; + grn_sock fd = hc->msg->u.fd; + grn_obj header_; + grn_obj head_; + grn_obj body_; + grn_obj foot_; + grn_obj *header = NULL; + grn_obj *head = NULL; + grn_obj *body = NULL; + grn_obj *foot = NULL; + char *chunk = NULL; + unsigned int chunk_size = 0; + int recv_flags; + grn_bool is_last_message = (flags & GRN_CTX_TAIL); + + GRN_TEXT_INIT(&header_, 0); + GRN_TEXT_INIT(&head_, 0); + GRN_TEXT_INIT(&body_, GRN_OBJ_DO_SHALLOW_COPY); + GRN_TEXT_INIT(&foot_, 0); + + grn_ctx_recv(ctx, &chunk, &chunk_size, &recv_flags); + GRN_TEXT_SET(ctx, &body_, chunk, chunk_size); + + if (!hc->in_body) { + if (is_last_message) { + h_output_set_header(ctx, &header_, expr_rc, GRN_TEXT_LEN(&body_), NULL); + hc->is_chunked = GRN_FALSE; + } else { + h_output_set_header(ctx, &header_, expr_rc, -1, NULL); + hc->is_chunked = GRN_TRUE; + } + header = &header_; + hc->in_body = GRN_TRUE; + } + + if (GRN_TEXT_LEN(&body_) > 0) { + if (hc->is_chunked) { + grn_text_printf(ctx, &head_, + "%x\r\n", (unsigned int)GRN_TEXT_LEN(&body_)); + head = &head_; + GRN_TEXT_PUTS(ctx, &foot_, "\r\n"); + foot = &foot_; + } + body = &body_; + } + + if (is_last_message) { + if (hc->is_chunked) { + GRN_TEXT_PUTS(ctx, &foot_, "0\r\n"); + GRN_TEXT_PUTS(ctx, &foot_, "Connection: close\r\n"); + GRN_TEXT_PUTS(ctx, &foot_, "\r\n"); + foot = &foot_; + } + } + + h_output_send(ctx, fd, header, head, body, foot); + + GRN_OBJ_FIN(ctx, &foot_); + GRN_OBJ_FIN(ctx, &body_); + GRN_OBJ_FIN(ctx, &head_); + GRN_OBJ_FIN(ctx, &header_); +} + +static void +h_output_typed(grn_ctx *ctx, int flags, ht_context *hc) +{ + grn_rc expr_rc = ctx->rc; + grn_sock fd = hc->msg->u.fd; + grn_obj header, head, body, foot; + char *chunk = NULL; + unsigned int chunk_size = 0; + int recv_flags; + grn_bool should_return_body; + + if (!(flags & GRN_CTX_TAIL)) { return; } + + switch (hc->msg->header.qtype) { + case 'G' : + case 'P' : + should_return_body = GRN_TRUE; + break; + default : + should_return_body = GRN_FALSE; + break; + } + + GRN_TEXT_INIT(&header, 0); + GRN_TEXT_INIT(&head, 0); + GRN_TEXT_INIT(&body, 0); + GRN_TEXT_INIT(&foot, 0); + + grn_ctx_recv(ctx, &chunk, &chunk_size, &recv_flags); + GRN_TEXT_SET(ctx, &body, chunk, chunk_size); + + output_envelope(ctx, expr_rc, &head, &body, &foot); + h_output_set_header(ctx, &header, expr_rc, + GRN_TEXT_LEN(&head) + + GRN_TEXT_LEN(&body) + + GRN_TEXT_LEN(&foot), + &foot); + if (should_return_body) { + h_output_send(ctx, fd, &header, &head, &body, &foot); + } else { + h_output_send(ctx, fd, &header, NULL, NULL, NULL); + } + GRN_OBJ_FIN(ctx, &foot); + GRN_OBJ_FIN(ctx, &body); + GRN_OBJ_FIN(ctx, &head); + GRN_OBJ_FIN(ctx, &header); +} + +static void +h_output(grn_ctx *ctx, int flags, void *arg) +{ + ht_context *hc = (ht_context *)arg; + + switch (grn_ctx_get_output_type(ctx)) { + case GRN_CONTENT_GROONGA_COMMAND_LIST : + case GRN_CONTENT_NONE : + h_output_raw(ctx, flags, hc); + break; + default : + h_output_typed(ctx, flags, hc); + break; + } +} + +static void +do_htreq_get(grn_ctx *ctx, ht_context *hc) +{ + grn_msg *msg = hc->msg; + char *path = NULL; + char *pathe = GRN_BULK_HEAD((grn_obj *)msg); + char *e = GRN_BULK_CURR((grn_obj *)msg); + for (;; pathe++) { + if (e <= pathe + 6) { + /* invalid request */ + return; + } + if (*pathe == ' ') { + if (!path) { + path = pathe + 1; + } else { + if (!memcmp(pathe + 1, "HTTP/1", 6)) { + break; + } + } + } + } + grn_ctx_send(ctx, path, pathe - path, GRN_CTX_TAIL); +} + +typedef struct { + const char *path_start; + int path_length; + long long int content_length; + grn_bool have_100_continue; + const char *body_start; +} h_post_header; + +#define STRING_EQUAL(string, string_length, constant_string)\ + (string_length == strlen(constant_string) &&\ + strncmp(string, constant_string, string_length) == 0) + +#define STRING_EQUAL_CI(string, string_length, constant_string)\ + (string_length == strlen(constant_string) &&\ + grn_strncasecmp(string, constant_string, string_length) == 0) + +static const char * +do_htreq_post_parse_header_request_line(grn_ctx *ctx, + const char *start, + const char *end, + h_post_header *header) +{ + const char *current; + + { + const char *method = start; + int method_length = -1; + + for (current = method; current < end; current++) { + if (current[0] == '\n') { + return NULL; + } + if (current[0] == ' ') { + method_length = current - method; + current++; + break; + } + } + if (method_length == -1) { + return NULL; + } + if (!STRING_EQUAL_CI(method, method_length, "POST")) { + return NULL; + } + } + + { + header->path_start = current; + header->path_length = -1; + for (; current < end; current++) { + if (current[0] == '\n') { + return NULL; + } + if (current[0] == ' ') { + header->path_length = current - header->path_start; + current++; + break; + } + } + if (header->path_length == -1) { + return NULL; + } + } + + { + const char *http_version_start = current; + int http_version_length = -1; + for (; current < end; current++) { + if (current[0] == '\n') { + http_version_length = current - http_version_start; + if (http_version_length > 0 && + http_version_start[http_version_length - 1] == '\r') { + http_version_length--; + } + current++; + break; + } + } + if (http_version_length == -1) { + return NULL; + } + if (!(STRING_EQUAL_CI(http_version_start, http_version_length, "HTTP/1.0") || + STRING_EQUAL_CI(http_version_start, http_version_length, "HTTP/1.1"))) { + return NULL; + } + } + + return current; +} + +static const char * +do_htreq_post_parse_header_values(grn_ctx *ctx, + const char *start, + const char *end, + h_post_header *header) +{ + const char *current; + const char *name = start; + int name_length = -1; + const char *value = NULL; + int value_length = -1; + + for (current = start; current < end; current++) { + switch (current[0]) { + case '\n' : + if (name_length == -1) { + if (current - name == 1 && current[-1] == '\r') { + return current + 1; + } else { + /* No ":" header line. TODO: report error. */ + return NULL; + } + } else { + while (value < current && value[0] == ' ') { + value++; + } + value_length = current - value; + if (value_length > 0 && value[value_length - 1] == '\r') { + value_length--; + } + if (STRING_EQUAL_CI(name, name_length, "Content-Length")) { + const char *rest; + header->content_length = grn_atoll(value, value + value_length, &rest); + if (rest != value + value_length) { + /* Invalid Content-Length value. TODO: report error. */ + header->content_length = -1; + } + } else if (STRING_EQUAL_CI(name, name_length, "Expect")) { + if (STRING_EQUAL(value, value_length, "100-continue")) { + header->have_100_continue = GRN_TRUE; + } + } + } + name = current + 1; + name_length = -1; + value = NULL; + value_length = -1; + break; + case ':' : + if (name_length == -1) { + name_length = current - name; + value = current + 1; + } + break; + default : + break; + } + } + + return NULL; +} + +static grn_bool +do_htreq_post_parse_header(grn_ctx *ctx, + const char *start, + const char *end, + h_post_header *header) +{ + const char *current; + + current = do_htreq_post_parse_header_request_line(ctx, start, end, header); + if (!current) { + return GRN_FALSE; + } + current = do_htreq_post_parse_header_values(ctx, current, end, header); + if (!current) { + return GRN_FALSE; + } + + if (current == end) { + header->body_start = NULL; + } else { + header->body_start = current; + } + + return GRN_TRUE; +} + +static void +do_htreq_post(grn_ctx *ctx, ht_context *hc) +{ + grn_msg *msg = hc->msg; + grn_sock fd = msg->u.fd; + const char *end; + h_post_header header; + + header.path_start = NULL; + header.path_length = -1; + header.content_length = -1; + header.body_start = NULL; + header.have_100_continue = GRN_FALSE; + + end = GRN_BULK_CURR((grn_obj *)msg); + if (!do_htreq_post_parse_header(ctx, + GRN_BULK_HEAD((grn_obj *)msg), + end, + &header)) { + return; + } + + grn_ctx_send(ctx, header.path_start, header.path_length, GRN_CTX_MORE); + if (ctx->rc != GRN_SUCCESS) { + ht_context context; + context.msg = msg; + context.in_body = GRN_FALSE; + context.is_chunked = GRN_FALSE; + h_output(ctx, GRN_CTX_TAIL, &context); + return; + } + + if (header.have_100_continue) { + const char *continue_message = "HTTP/1.1 100 Continue\r\n"; + ssize_t send_size; + int send_flags = MSG_NOSIGNAL; + send_size = send(fd, continue_message, strlen(continue_message), send_flags); + if (send_size == -1) { + SOERR("send"); + return; + } + } + + { + grn_obj chunk_buffer; + long long int read_content_length = 0; + + GRN_TEXT_INIT(&chunk_buffer, 0); + while (read_content_length < header.content_length) { +#define POST_BUFFER_SIZE 8192 + char buffer[POST_BUFFER_SIZE]; + const char *buffer_start, *buffer_current, *buffer_end; + + if (header.body_start) { + buffer_start = header.body_start; + buffer_end = end; + header.body_start = NULL; + } else { + ssize_t recv_length; + int recv_flags = 0; + recv_length = recv(fd, buffer, POST_BUFFER_SIZE, recv_flags); + if (recv_length == 0) { + break; + } + if (recv_length == -1) { + SOERR("recv"); + break; + } + buffer_start = buffer; + buffer_end = buffer_start + recv_length; + } + read_content_length += buffer_end - buffer_start; + + buffer_current = buffer_end - 1; + for (; buffer_current > buffer_start; buffer_current--) { + grn_bool is_separator; + switch (buffer_current[0]) { + case '\n' : + case ',' : + is_separator = GRN_TRUE; + break; + default : + is_separator = GRN_FALSE; + break; + } + if (!is_separator) { + continue; + } + + GRN_TEXT_PUT(ctx, + &chunk_buffer, + buffer_start, + buffer_current + 1 - buffer_start); + { + int flags = 0; + if (!(read_content_length == header.content_length && + buffer_current + 1 == buffer_end)) { + flags |= GRN_CTX_MORE; + } else { + flags |= GRN_CTX_TAIL; + } + grn_ctx_send(ctx, + GRN_TEXT_VALUE(&chunk_buffer), + GRN_TEXT_LEN(&chunk_buffer), + flags); + } + buffer_start = buffer_current + 1; + GRN_BULK_REWIND(&chunk_buffer); + break; + } + if (buffer_end > buffer_start) { + GRN_TEXT_PUT(ctx, &chunk_buffer, + buffer_start, buffer_end - buffer_start); + } +#undef POST_BUFFER_SIZE + + if (ctx->rc != GRN_SUCCESS) { + break; + } + } + + if (ctx->rc == GRN_CANCEL) { + h_output(ctx, GRN_CTX_TAIL, hc); + } else if (ctx->rc == GRN_SUCCESS && GRN_TEXT_LEN(&chunk_buffer) > 0) { + grn_ctx_send(ctx, + GRN_TEXT_VALUE(&chunk_buffer), + GRN_TEXT_LEN(&chunk_buffer), + GRN_CTX_TAIL); + } + + GRN_OBJ_FIN(ctx, &chunk_buffer); + } +} + +static void +do_htreq(grn_ctx *ctx, ht_context *hc) +{ + grn_msg *msg = hc->msg; + grn_com_header *header = &msg->header; + switch (header->qtype) { + case 'G' : /* GET */ + case 'H' : /* HEAD */ + do_htreq_get(ctx, hc); + break; + case 'P' : /* POST */ + do_htreq_post(ctx, hc); + break; + } + /* if (ctx->rc != GRN_OPERATION_WOULD_BLOCK) {...} */ + grn_msg_close(ctx, (grn_obj *)msg); + /* if not keep alive connection */ + grn_sock_close(msg->u.fd); + grn_com_event_start_accept(ctx, msg->acceptor->ev); +} + +enum { + MBRES_SUCCESS = 0x00, + MBRES_KEY_ENOENT = 0x01, + MBRES_KEY_EEXISTS = 0x02, + MBRES_E2BIG = 0x03, + MBRES_EINVAL = 0x04, + MBRES_NOT_STORED = 0x05, + MBRES_UNKNOWN_COMMAND = 0x81, + MBRES_ENOMEM = 0x82, +}; + +enum { + MBCMD_GET = 0x00, + MBCMD_SET = 0x01, + MBCMD_ADD = 0x02, + MBCMD_REPLACE = 0x03, + MBCMD_DELETE = 0x04, + MBCMD_INCREMENT = 0x05, + MBCMD_DECREMENT = 0x06, + MBCMD_QUIT = 0x07, + MBCMD_FLUSH = 0x08, + MBCMD_GETQ = 0x09, + MBCMD_NOOP = 0x0a, + MBCMD_VERSION = 0x0b, + MBCMD_GETK = 0x0c, + MBCMD_GETKQ = 0x0d, + MBCMD_APPEND = 0x0e, + MBCMD_PREPEND = 0x0f, + MBCMD_STAT = 0x10, + MBCMD_SETQ = 0x11, + MBCMD_ADDQ = 0x12, + MBCMD_REPLACEQ = 0x13, + MBCMD_DELETEQ = 0x14, + MBCMD_INCREMENTQ = 0x15, + MBCMD_DECREMENTQ = 0x16, + MBCMD_QUITQ = 0x17, + MBCMD_FLUSHQ = 0x18, + MBCMD_APPENDQ = 0x19, + MBCMD_PREPENDQ = 0x1a +}; + +static grn_obj *cache_table = NULL; +static grn_obj *cache_value = NULL; +static grn_obj *cache_flags = NULL; +static grn_obj *cache_expire = NULL; +static grn_obj *cache_cas = NULL; + +#define CTX_GET(name) (grn_ctx_get(ctx, (name), strlen(name))) + +static grn_bool +memcached_setup_flags_column(grn_ctx *ctx, const char *name) +{ + cache_flags = grn_obj_column(ctx, cache_table, name, strlen(name)); + if (cache_flags) { + return GRN_TRUE; + } + + cache_flags = grn_column_create(ctx, cache_table, name, strlen(name), NULL, + GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_UINT32)); + if (!cache_flags) { + return GRN_FALSE; + } + + return GRN_TRUE; +} + +static grn_bool +memcached_setup_expire_column(grn_ctx *ctx, const char *name) +{ + cache_expire = grn_obj_column(ctx, cache_table, name, strlen(name)); + if (cache_expire) { + return GRN_TRUE; + } + + cache_expire = grn_column_create(ctx, cache_table, name, strlen(name), NULL, + GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_UINT32)); + if (!cache_expire) { + return GRN_FALSE; + } + + return GRN_TRUE; +} + +static grn_bool +memcached_setup_cas_column(grn_ctx *ctx, const char *name) +{ + cache_cas = grn_obj_column(ctx, cache_table, name, strlen(name)); + if (cache_cas) { + return GRN_TRUE; + } + + cache_cas = grn_column_create(ctx, cache_table, name, strlen(name), NULL, + GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_UINT64)); + if (!cache_cas) { + return GRN_FALSE; + } + + return GRN_TRUE; +} + +static grn_bool +memcached_init(grn_ctx *ctx) +{ + if (memcached_column_name) { + cache_value = CTX_GET(memcached_column_name); + if (!cache_value) { + ERR(GRN_INVALID_ARGUMENT, + "memcached column doesn't exist: <%s>", + memcached_column_name); + return GRN_FALSE; + } + if (!(grn_obj_is_column(ctx, cache_value) && + ((cache_value->header.flags & GRN_OBJ_COLUMN_TYPE_MASK) == + GRN_OBJ_COLUMN_SCALAR))) { + grn_obj inspected; + GRN_TEXT_INIT(&inspected, 0); + grn_inspect(ctx, &inspected, cache_value); + ERR(GRN_INVALID_ARGUMENT, + "memcached column must be scalar column: <%.*s>", + (int)GRN_TEXT_LEN(&inspected), + GRN_TEXT_VALUE(&inspected)); + GRN_OBJ_FIN(ctx, &inspected); + return GRN_FALSE; + } + if (!(GRN_DB_SHORT_TEXT <= grn_obj_get_range(ctx, cache_value) && + grn_obj_get_range(ctx, cache_value) <= GRN_DB_LONG_TEXT)) { + grn_obj inspected; + GRN_TEXT_INIT(&inspected, 0); + grn_inspect(ctx, &inspected, cache_value); + ERR(GRN_INVALID_ARGUMENT, + "memcached column must be text column: <%.*s>", + (int)GRN_TEXT_LEN(&inspected), + GRN_TEXT_VALUE(&inspected)); + GRN_OBJ_FIN(ctx, &inspected); + return GRN_FALSE; + } + + cache_table = grn_ctx_at(ctx, cache_value->header.domain); + if (cache_table->header.type == GRN_TABLE_NO_KEY) { + grn_obj inspected; + GRN_TEXT_INIT(&inspected, 0); + grn_inspect(ctx, &inspected, cache_table); + ERR(GRN_INVALID_ARGUMENT, + "memcached column's table must be HASH_KEY, PAT_KEY or DAT_KEY table: " + "<%.*s>", + (int)GRN_TEXT_LEN(&inspected), + GRN_TEXT_VALUE(&inspected)); + GRN_OBJ_FIN(ctx, &inspected); + return GRN_FALSE; + } + + { + char column_name[GRN_TABLE_MAX_KEY_SIZE]; + char value_column_name[GRN_TABLE_MAX_KEY_SIZE]; + int value_column_name_size; + + value_column_name_size = grn_column_name(ctx, cache_value, + value_column_name, + GRN_TABLE_MAX_KEY_SIZE); + grn_snprintf(column_name, + GRN_TABLE_MAX_KEY_SIZE, + GRN_TABLE_MAX_KEY_SIZE, + "%.*s_memcached_flags", + value_column_name_size, + value_column_name); + if (!memcached_setup_flags_column(ctx, column_name)) { + return GRN_FALSE; + } + grn_snprintf(column_name, + GRN_TABLE_MAX_KEY_SIZE, + GRN_TABLE_MAX_KEY_SIZE, + "%.*s_memcached_expire", + value_column_name_size, + value_column_name); + if (!memcached_setup_expire_column(ctx, column_name)) { + return GRN_FALSE; + } + grn_snprintf(column_name, + GRN_TABLE_MAX_KEY_SIZE, + GRN_TABLE_MAX_KEY_SIZE, + "%.*s_memcached_cas", + value_column_name_size, + value_column_name); + if (!memcached_setup_cas_column(ctx, column_name)) { + return GRN_FALSE; + } + } + } else { + const char *table_name = "Memcache"; + const char *value_column_name = "value"; + + cache_table = CTX_GET(table_name); + if (!cache_table) { + cache_table = grn_table_create(ctx, table_name, strlen(table_name), NULL, + GRN_OBJ_TABLE_PAT_KEY|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_SHORT_TEXT), + NULL); + if (!cache_table) { + return GRN_FALSE; + } + } + + cache_value = grn_obj_column(ctx, cache_table, + value_column_name, + strlen(value_column_name)); + if (!cache_value) { + cache_value = grn_column_create(ctx, cache_table, + value_column_name, + strlen(value_column_name), + NULL, + GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_SHORT_TEXT)); + if (!cache_value) { + return GRN_FALSE; + } + } + + if (!memcached_setup_flags_column(ctx, "flags")) { + return GRN_FALSE; + } + if (!memcached_setup_expire_column(ctx, "expire")) { + return GRN_FALSE; + } + if (!memcached_setup_cas_column(ctx, "cas")) { + return GRN_FALSE; + } + } + + return GRN_TRUE; +} + +#define RELATIVE_TIME_THRESH 1000000000 + +#define MBRES(ctx,re,status,key_len,extra_len,flags) do {\ + grn_msg_set_property((ctx), (re), (status), (key_len), (extra_len));\ + grn_msg_send((ctx), (re), (flags));\ +} while (0) + +#define GRN_MSG_MBRES(block) do {\ + if (!quiet) {\ + grn_obj *re = grn_msg_open_for_reply(ctx, (grn_obj *)msg, &edge->send_old);\ + ((grn_msg *)re)->header.qtype = header->qtype;\ + block\ + }\ +} while (0) + +static uint64_t +get_mbreq_cas_id() +{ + static uint64_t cas_id = 0; + /* FIXME: use GRN_ATOMIC_ADD_EX_64, but it is not implemented */ + return ++cas_id; +} + +static void +do_mbreq(grn_ctx *ctx, grn_edge *edge) +{ + int quiet = 0; + int flags = 0; + grn_msg *msg = edge->msg; + grn_com_header *header = &msg->header; + + switch (header->qtype) { + case MBCMD_GETQ : + flags = GRN_CTX_MORE; + /* fallthru */ + case MBCMD_GET : + { + grn_id rid; + uint16_t keylen = ntohs(header->keylen); + char *key = GRN_BULK_HEAD((grn_obj *)msg); + rid = grn_table_get(ctx, cache_table, key, keylen); + if (!rid) { + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_KEY_ENOENT, 0, 0, 0); + }); + } else { + grn_timeval tv; + uint32_t expire; + { + grn_obj expire_buf; + GRN_UINT32_INIT(&expire_buf, 0); + grn_obj_get_value(ctx, cache_expire, rid, &expire_buf); + expire = GRN_UINT32_VALUE(&expire_buf); + grn_obj_close(ctx, &expire_buf); + } + grn_timeval_now(ctx, &tv); + if (expire && expire < tv.tv_sec) { + grn_table_delete_by_id(ctx, cache_table, rid); + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_KEY_ENOENT, 0, 0, 0); + }); + } else { + grn_obj cas_buf; + GRN_UINT64_INIT(&cas_buf, 0); + grn_obj_get_value(ctx, cache_cas, rid, &cas_buf); + GRN_MSG_MBRES({ + grn_obj_get_value(ctx, cache_flags, rid, re); + grn_obj_get_value(ctx, cache_value, rid, re); + ((grn_msg *)re)->header.cas = GRN_UINT64_VALUE(&cas_buf); + MBRES(ctx, re, MBRES_SUCCESS, 0, 4, flags); + }); + grn_obj_close(ctx, &cas_buf); + } + } + } + break; + case MBCMD_SETQ : + case MBCMD_ADDQ : + case MBCMD_REPLACEQ : + quiet = 1; + /* fallthru */ + case MBCMD_SET : + case MBCMD_ADD : + case MBCMD_REPLACE : + { + grn_id rid; + uint32_t size = ntohl(header->size); + uint16_t keylen = ntohs(header->keylen); + uint8_t extralen = header->level; + char *body = GRN_BULK_HEAD((grn_obj *)msg); + uint32_t flags = *((uint32_t *)body); + uint32_t expire = ntohl(*((uint32_t *)(body + 4))); + uint32_t valuelen = size - keylen - extralen; + char *key = body + 8; + char *value = key + keylen; + int added = 0; + int f = (header->qtype == MBCMD_REPLACE || + header->qtype == MBCMD_REPLACEQ) ? 0 : GRN_TABLE_ADD; + GRN_ASSERT(extralen == 8); + if (header->qtype == MBCMD_REPLACE || header->qtype == MBCMD_REPLACEQ) { + rid = grn_table_get(ctx, cache_table, key, keylen); + } else { + rid = grn_table_add(ctx, cache_table, key, keylen, &added); + } + if (!rid) { + GRN_MSG_MBRES({ + MBRES(ctx, re, (f & GRN_TABLE_ADD) ? MBRES_ENOMEM : MBRES_NOT_STORED, 0, 0, 0); + }); + } else { + if (added) { + if (header->cas) { + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_EINVAL, 0, 0, 0); + }); + } else { + grn_obj text_buf, uint32_buf; + GRN_TEXT_INIT(&text_buf, GRN_OBJ_DO_SHALLOW_COPY); + GRN_TEXT_SET_REF(&text_buf, value, valuelen); + grn_obj_set_value(ctx, cache_value, rid, &text_buf, GRN_OBJ_SET); + GRN_UINT32_INIT(&uint32_buf, 0); + GRN_UINT32_SET(ctx, &uint32_buf, flags); + grn_obj_set_value(ctx, cache_flags, rid, &uint32_buf, GRN_OBJ_SET); + if (expire && expire < RELATIVE_TIME_THRESH) { + grn_timeval tv; + grn_timeval_now(ctx, &tv); + expire += tv.tv_sec; + } + GRN_UINT32_SET(ctx, &uint32_buf, expire); + grn_obj_set_value(ctx, cache_expire, rid, &uint32_buf, GRN_OBJ_SET); + grn_obj_close(ctx, &uint32_buf); + { + grn_obj cas_buf; + uint64_t cas_id = get_mbreq_cas_id(); + GRN_UINT64_INIT(&cas_buf, 0); + GRN_UINT64_SET(ctx, &cas_buf, cas_id); + grn_obj_set_value(ctx, cache_cas, rid, &cas_buf, GRN_OBJ_SET); + grn_obj_close(ctx, &cas_buf); + GRN_MSG_MBRES({ + ((grn_msg *)re)->header.cas = cas_id; + MBRES(ctx, re, MBRES_SUCCESS, 0, 0, 0); + }); + } + } + } else { + if (header->qtype != MBCMD_SET && header->qtype != MBCMD_SETQ) { + grn_obj uint32_buf; + grn_timeval tv; + uint32_t oexpire; + + GRN_UINT32_INIT(&uint32_buf, 0); + grn_obj_get_value(ctx, cache_expire, rid, &uint32_buf); + oexpire = GRN_UINT32_VALUE(&uint32_buf); + grn_timeval_now(ctx, &tv); + + if (oexpire && oexpire < tv.tv_sec) { + if (header->qtype == MBCMD_REPLACE || + header->qtype == MBCMD_REPLACEQ) { + grn_table_delete_by_id(ctx, cache_table, rid); + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_NOT_STORED, 0, 0, 0); + }); + break; + } + } else if (header->qtype == MBCMD_ADD || + header->qtype == MBCMD_ADDQ) { + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_NOT_STORED, 0, 0, 0); + }); + break; + } + } + { + if (header->cas) { + grn_obj cas_buf; + GRN_UINT64_INIT(&cas_buf, 0); + grn_obj_get_value(ctx, cache_cas, rid, &cas_buf); + if (header->cas != GRN_UINT64_VALUE(&cas_buf)) { + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_NOT_STORED, 0, 0, 0); + }); + } + } + { + grn_obj text_buf, uint32_buf; + GRN_TEXT_INIT(&text_buf, GRN_OBJ_DO_SHALLOW_COPY); + GRN_TEXT_SET_REF(&text_buf, value, valuelen); + grn_obj_set_value(ctx, cache_value, rid, &text_buf, GRN_OBJ_SET); + GRN_UINT32_INIT(&uint32_buf, 0); + GRN_UINT32_SET(ctx, &uint32_buf, flags); + grn_obj_set_value(ctx, cache_flags, rid, &uint32_buf, GRN_OBJ_SET); + if (expire && expire < RELATIVE_TIME_THRESH) { + grn_timeval tv; + grn_timeval_now(ctx, &tv); + expire += tv.tv_sec; + } + GRN_UINT32_SET(ctx, &uint32_buf, expire); + grn_obj_set_value(ctx, cache_expire, rid, &uint32_buf, GRN_OBJ_SET); + { + grn_obj cas_buf; + uint64_t cas_id = get_mbreq_cas_id(); + GRN_UINT64_INIT(&cas_buf, 0); + GRN_UINT64_SET(ctx, &cas_buf, cas_id); + grn_obj_set_value(ctx, cache_cas, rid, &cas_buf, GRN_OBJ_SET); + GRN_MSG_MBRES({ + ((grn_msg *)re)->header.cas = cas_id; + MBRES(ctx, re, MBRES_SUCCESS, 0, 0, 0); + }); + } + } + } + } + } + } + break; + case MBCMD_DELETEQ : + quiet = 1; + /* fallthru */ + case MBCMD_DELETE : + { + grn_id rid; + uint16_t keylen = ntohs(header->keylen); + char *key = GRN_BULK_HEAD((grn_obj *)msg); + rid = grn_table_get(ctx, cache_table, key, keylen); + if (!rid) { + /* GRN_LOG(ctx, GRN_LOG_NOTICE, "GET k=%d not found", keylen); */ + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_KEY_ENOENT, 0, 0, 0); + }); + } else { + grn_table_delete_by_id(ctx, cache_table, rid); + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_SUCCESS, 0, 4, 0); + }); + } + } + break; + case MBCMD_INCREMENTQ : + case MBCMD_DECREMENTQ : + quiet = 1; + /* fallthru */ + case MBCMD_INCREMENT : + case MBCMD_DECREMENT : + { + grn_id rid; + int added = 0; + uint64_t delta, init; + uint16_t keylen = ntohs(header->keylen); + char *body = GRN_BULK_HEAD((grn_obj *)msg); + char *key = body + 20; + uint32_t expire = ntohl(*((uint32_t *)(body + 16))); + grn_ntoh(&delta, body, 8); + grn_ntoh(&init, body + 8, 8); + GRN_ASSERT(header->level == 20); /* extralen */ + if (expire == 0xffffffff) { + rid = grn_table_get(ctx, cache_table, key, keylen); + } else { + rid = grn_table_add(ctx, cache_table, key, keylen, &added); + } + if (!rid) { + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_KEY_ENOENT, 0, 0, 0); + }); + } else { + grn_obj uint32_buf, text_buf; + GRN_UINT32_INIT(&uint32_buf, 0); + GRN_TEXT_INIT(&text_buf, GRN_OBJ_DO_SHALLOW_COPY); + if (added) { + GRN_TEXT_SET_REF(&text_buf, &init, 8); + grn_obj_set_value(ctx, cache_value, rid, &text_buf, GRN_OBJ_SET); + GRN_UINT32_SET(ctx, &uint32_buf, 0); + grn_obj_set_value(ctx, cache_flags, rid, &uint32_buf, GRN_OBJ_SET); + } else { + grn_timeval tv; + uint32_t oexpire; + + grn_obj_get_value(ctx, cache_expire, rid, &uint32_buf); + oexpire = GRN_UINT32_VALUE(&uint32_buf); + grn_timeval_now(ctx, &tv); + + if (oexpire && oexpire < tv.tv_sec) { + if (expire == 0xffffffffU) { + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_KEY_ENOENT, 0, 0, 0); + }); + break; + } else { + GRN_TEXT_SET_REF(&text_buf, &init, 8); + grn_obj_set_value(ctx, cache_value, rid, &text_buf, GRN_OBJ_SET); + GRN_UINT32_SET(ctx, &uint32_buf, 0); + grn_obj_set_value(ctx, cache_flags, rid, &uint32_buf, GRN_OBJ_SET); + } + } else { + grn_obj uint64_buf; + GRN_UINT64_INIT(&uint64_buf, 0); + GRN_UINT64_SET(ctx, &uint64_buf, delta); + grn_obj_set_value(ctx, cache_value, rid, &uint64_buf, + header->qtype == MBCMD_INCREMENT || + header->qtype == MBCMD_INCREMENTQ + ? GRN_OBJ_INCR + : GRN_OBJ_DECR); + } + } + if (expire && expire < RELATIVE_TIME_THRESH) { + grn_timeval tv; + grn_timeval_now(ctx, &tv); + expire += tv.tv_sec; + } + GRN_UINT32_SET(ctx, &uint32_buf, expire); + grn_obj_set_value(ctx, cache_expire, rid, &uint32_buf, GRN_OBJ_SET); + GRN_MSG_MBRES({ + /* TODO: get_mbreq_cas_id() */ + grn_obj_get_value(ctx, cache_value, rid, re); + grn_hton(&delta, (uint64_t *)GRN_BULK_HEAD(re), 8); + GRN_TEXT_SET(ctx, re, &delta, sizeof(uint64_t)); + MBRES(ctx, re, MBRES_SUCCESS, 0, sizeof(uint64_t), 0); + }); + } + } + break; + case MBCMD_FLUSHQ : + quiet = 1; + /* fallthru */ + case MBCMD_FLUSH : + { + uint32_t expire; + uint8_t extralen = header->level; + if (extralen) { + char *body = GRN_BULK_HEAD((grn_obj *)msg); + GRN_ASSERT(extralen == 4); + expire = ntohl(*((uint32_t *)(body))); + if (expire < RELATIVE_TIME_THRESH) { + grn_timeval tv; + grn_timeval_now(ctx, &tv); + if (expire) { + expire += tv.tv_sec; + } else { + expire = tv.tv_sec - 1; + } + } + } else { + grn_timeval tv; + grn_timeval_now(ctx, &tv); + expire = tv.tv_sec - 1; + } + { + grn_obj exp_buf; + GRN_UINT32_INIT(&exp_buf, 0); + GRN_UINT32_SET(ctx, &exp_buf, expire); + GRN_TABLE_EACH(ctx, cache_table, 0, 0, rid, NULL, NULL, NULL, { + grn_obj_set_value(ctx, cache_expire, rid, &exp_buf, GRN_OBJ_SET); + }); + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_SUCCESS, 0, 4, 0); + }); + grn_obj_close(ctx, &exp_buf); + } + } + break; + case MBCMD_NOOP : + break; + case MBCMD_VERSION : + GRN_MSG_MBRES({ + grn_bulk_write(ctx, re, PACKAGE_VERSION, strlen(PACKAGE_VERSION)); + MBRES(ctx, re, MBRES_SUCCESS, 0, 0, 0); + }); + break; + case MBCMD_GETKQ : + flags = GRN_CTX_MORE; + /* fallthru */ + case MBCMD_GETK : + { + grn_id rid; + uint16_t keylen = ntohs(header->keylen); + char *key = GRN_BULK_HEAD((grn_obj *)msg); + rid = grn_table_get(ctx, cache_table, key, keylen); + if (!rid) { + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_KEY_ENOENT, 0, 0, 0); + }); + } else { + grn_obj uint32_buf; + grn_timeval tv; + uint32_t expire; + GRN_UINT32_INIT(&uint32_buf, 0); + grn_obj_get_value(ctx, cache_expire, rid, &uint32_buf); + expire = GRN_UINT32_VALUE(&uint32_buf); + grn_timeval_now(ctx, &tv); + if (expire && expire < tv.tv_sec) { + grn_table_delete_by_id(ctx, cache_table, rid); + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_KEY_ENOENT, 0, 0, 0); + }); + } else { + grn_obj uint64_buf; + GRN_UINT64_INIT(&uint64_buf, 0); + grn_obj_get_value(ctx, cache_cas, rid, &uint64_buf); + GRN_MSG_MBRES({ + grn_obj_get_value(ctx, cache_flags, rid, re); + grn_bulk_write(ctx, re, key, keylen); + grn_obj_get_value(ctx, cache_value, rid, re); + ((grn_msg *)re)->header.cas = GRN_UINT64_VALUE(&uint64_buf); + MBRES(ctx, re, MBRES_SUCCESS, keylen, 4, flags); + }); + } + } + } + break; + case MBCMD_APPENDQ : + case MBCMD_PREPENDQ : + quiet = 1; + /* fallthru */ + case MBCMD_APPEND : + case MBCMD_PREPEND : + { + grn_id rid; + uint32_t size = ntohl(header->size); + uint16_t keylen = ntohs(header->keylen); + char *key = GRN_BULK_HEAD((grn_obj *)msg); + char *value = key + keylen; + uint32_t valuelen = size - keylen; + rid = grn_table_add(ctx, cache_table, key, keylen, NULL); + if (!rid) { + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_ENOMEM, 0, 0, 0); + }); + } else { + /* FIXME: check expire */ + grn_obj buf; + int flags = header->qtype == MBCMD_APPEND ? GRN_OBJ_APPEND : GRN_OBJ_PREPEND; + GRN_TEXT_INIT(&buf, GRN_OBJ_DO_SHALLOW_COPY); + GRN_TEXT_SET_REF(&buf, value, valuelen); + grn_obj_set_value(ctx, cache_value, rid, &buf, flags); + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_SUCCESS, 0, 0, 0); + }); + } + } + break; + case MBCMD_STAT : + { + pid_t pid = grn_getpid(); + GRN_MSG_MBRES({ + grn_bulk_write(ctx, re, "pid", 3); + grn_text_itoa(ctx, re, pid); + MBRES(ctx, re, MBRES_SUCCESS, 3, 0, 0); + }); + } + break; + case MBCMD_QUITQ : + quiet = 1; + /* fallthru */ + case MBCMD_QUIT : + GRN_MSG_MBRES({ + MBRES(ctx, re, MBRES_SUCCESS, 0, 0, 0); + }); + /* fallthru */ + default : + ctx->stat = GRN_CTX_QUIT; + break; + } +} + +/* worker thread */ + +enum { + EDGE_IDLE = 0x00, + EDGE_WAIT = 0x01, + EDGE_DOING = 0x02, + EDGE_ABORT = 0x03, +}; + +static void +check_rlimit_nofile(grn_ctx *ctx) +{ +#ifndef WIN32 + struct rlimit limit; + limit.rlim_cur = 0; + limit.rlim_max = 0; + getrlimit(RLIMIT_NOFILE, &limit); + if (limit.rlim_cur < RLIMIT_NOFILE_MINIMUM) { + limit.rlim_cur = RLIMIT_NOFILE_MINIMUM; + limit.rlim_max = RLIMIT_NOFILE_MINIMUM; + setrlimit(RLIMIT_NOFILE, &limit); + limit.rlim_cur = 0; + limit.rlim_max = 0; + getrlimit(RLIMIT_NOFILE, &limit); + } + GRN_LOG(ctx, GRN_LOG_NOTICE, + "RLIMIT_NOFILE(%" GRN_FMT_LLD ",%" GRN_FMT_LLD ")", + (long long int)limit.rlim_cur, (long long int)limit.rlim_max); +#endif /* WIN32 */ +} + +static grn_thread_func_result CALLBACK +h_worker(void *arg) +{ + ht_context hc; + grn_ctx ctx_, *ctx = &ctx_; + grn_ctx_init(ctx, 0); + grn_ctx_use(ctx, (grn_obj *)arg); + grn_ctx_recv_handler_set(ctx, h_output, &hc); + MUTEX_LOCK_ENSURE(ctx, q_mutex); + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread start (%d/%d)", + n_floating_threads, n_running_threads); + while (n_running_threads <= max_n_floating_threads && + grn_gctx.stat != GRN_CTX_QUIT) { + grn_obj *msg; + if (ctx->rc == GRN_CANCEL) { + ctx->rc = GRN_SUCCESS; + } + n_floating_threads++; + while (!(msg = (grn_obj *)grn_com_queue_deque(&grn_gctx, &ctx_new))) { + COND_WAIT(q_cond, q_mutex); + if (grn_gctx.stat == GRN_CTX_QUIT) { + n_floating_threads--; + goto exit; + } + if (n_running_threads > max_n_floating_threads) { + n_floating_threads--; + goto exit; + } + } + n_floating_threads--; + MUTEX_UNLOCK(q_mutex); + hc.msg = (grn_msg *)msg; + hc.in_body = GRN_FALSE; + hc.is_chunked = GRN_FALSE; + do_htreq(ctx, &hc); + MUTEX_LOCK_ENSURE(ctx, q_mutex); + } +exit : + n_running_threads--; + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread end (%d/%d)", + n_floating_threads, n_running_threads); + if (grn_gctx.stat == GRN_CTX_QUIT) { + break_accept_event_loop(ctx); + } + grn_ctx_fin(ctx); + MUTEX_UNLOCK(q_mutex); + return GRN_THREAD_FUNC_RETURN_VALUE; +} + +static void +h_handler(grn_ctx *ctx, grn_obj *msg) +{ + grn_com *com = ((grn_msg *)msg)->u.peer; + if (ctx->rc) { + grn_com_close(ctx, com); + grn_msg_close(ctx, msg); + } else { + grn_sock fd = com->fd; + void *arg = com->ev->opaque; + /* if not keep alive connection */ + grn_com_event_del(ctx, com->ev, fd); + ((grn_msg *)msg)->u.fd = fd; + MUTEX_LOCK_ENSURE(ctx, q_mutex); + grn_com_queue_enque(ctx, &ctx_new, (grn_com_queue_entry *)msg); + if (n_floating_threads == 0 && n_running_threads < max_n_floating_threads) { + grn_thread thread; + n_running_threads++; + if (THREAD_CREATE(thread, h_worker, arg)) { + n_running_threads--; + SERR("pthread_create"); + } + } + COND_SIGNAL(q_cond); + MUTEX_UNLOCK(q_mutex); + } +} + +static int +h_server(char *path) +{ + int exit_code = EXIT_FAILURE; + grn_ctx ctx_, *ctx = &ctx_; + grn_ctx_init(ctx, 0); + GRN_COM_QUEUE_INIT(&ctx_new); + GRN_COM_QUEUE_INIT(&ctx_old); + check_rlimit_nofile(ctx); + GRN_TEXT_INIT(&http_response_server_line, 0); + grn_text_printf(ctx, + &http_response_server_line, + "Server: %s/%s\r\n", + grn_get_package_label(), + grn_get_version()); + exit_code = start_service(ctx, path, NULL, h_handler); + GRN_OBJ_FIN(ctx, &http_response_server_line); + grn_ctx_fin(ctx); + return exit_code; +} + +static grn_thread_func_result CALLBACK +g_worker(void *arg) +{ + MUTEX_LOCK_ENSURE(NULL, q_mutex); + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread start (%d/%d)", + n_floating_threads, n_running_threads); + while (n_running_threads <= max_n_floating_threads && + grn_gctx.stat != GRN_CTX_QUIT) { + grn_ctx *ctx; + grn_edge *edge; + n_floating_threads++; + while (!(edge = (grn_edge *)grn_com_queue_deque(&grn_gctx, &ctx_new))) { + COND_WAIT(q_cond, q_mutex); + if (grn_gctx.stat == GRN_CTX_QUIT) { + n_floating_threads--; + goto exit; + } + if (n_running_threads > max_n_floating_threads) { + n_floating_threads--; + goto exit; + } + } + ctx = &edge->ctx; + n_floating_threads--; + if (edge->stat == EDGE_DOING) { continue; } + if (edge->stat == EDGE_WAIT) { + edge->stat = EDGE_DOING; + while (!GRN_COM_QUEUE_EMPTYP(&edge->recv_new)) { + grn_obj *msg; + MUTEX_UNLOCK(q_mutex); + /* if (edge->flags == GRN_EDGE_WORKER) */ + while (ctx->stat != GRN_CTX_QUIT && + (edge->msg = (grn_msg *)grn_com_queue_deque(ctx, &edge->recv_new))) { + grn_com_header *header = &edge->msg->header; + msg = (grn_obj *)edge->msg; + switch (header->proto) { + case GRN_COM_PROTO_MBREQ : + do_mbreq(ctx, edge); + break; + case GRN_COM_PROTO_GQTP : + grn_ctx_send(ctx, GRN_BULK_HEAD(msg), GRN_BULK_VSIZE(msg), header->flags); + ERRCLR(ctx); + if (ctx->rc == GRN_CANCEL) { + ctx->rc = GRN_SUCCESS; + } + break; + default : + ctx->stat = GRN_CTX_QUIT; + break; + } + grn_msg_close(ctx, msg); + } + while ((msg = (grn_obj *)grn_com_queue_deque(ctx, &edge->send_old))) { + grn_msg_close(ctx, msg); + } + MUTEX_LOCK_ENSURE(ctx, q_mutex); + if (ctx->stat == GRN_CTX_QUIT || edge->stat == EDGE_ABORT) { break; } + } + } + if (ctx->stat == GRN_CTX_QUIT || edge->stat == EDGE_ABORT) { + grn_com_queue_enque(&grn_gctx, &ctx_old, (grn_com_queue_entry *)edge); + edge->stat = EDGE_ABORT; + } else { + edge->stat = EDGE_IDLE; + } + }; +exit : + n_running_threads--; + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread end (%d/%d)", + n_floating_threads, n_running_threads); + MUTEX_UNLOCK(q_mutex); + return GRN_THREAD_FUNC_RETURN_VALUE; +} + +static void +g_dispatcher(grn_ctx *ctx, grn_edge *edge) +{ + MUTEX_LOCK_ENSURE(ctx, q_mutex); + if (edge->stat == EDGE_IDLE) { + grn_com_queue_enque(ctx, &ctx_new, (grn_com_queue_entry *)edge); + edge->stat = EDGE_WAIT; + if (n_floating_threads == 0 && n_running_threads < max_n_floating_threads) { + grn_thread thread; + n_running_threads++; + if (THREAD_CREATE(thread, g_worker, NULL)) { + n_running_threads--; + SERR("pthread_create"); + } + } + COND_SIGNAL(q_cond); + } + MUTEX_UNLOCK(q_mutex); +} + +static void +g_output(grn_ctx *ctx, int flags, void *arg) +{ + grn_edge *edge = arg; + grn_com *com = edge->com; + grn_msg *req = edge->msg, *msg = (grn_msg *)ctx->impl->output.buf; + msg->edge_id = req->edge_id; + msg->header.proto = req->header.proto == GRN_COM_PROTO_MBREQ + ? GRN_COM_PROTO_MBRES : req->header.proto; + if (ctx->rc != GRN_SUCCESS && GRN_BULK_VSIZE(ctx->impl->output.buf) == 0) { + GRN_TEXT_PUTS(ctx, ctx->impl->output.buf, ctx->errbuf); + } + if (grn_msg_send(ctx, (grn_obj *)msg, + (flags & GRN_CTX_MORE) ? GRN_CTX_MORE : GRN_CTX_TAIL)) { + edge->stat = EDGE_ABORT; + } + ctx->impl->output.buf = grn_msg_open(ctx, com, &edge->send_old); +} + +static void +g_handler(grn_ctx *ctx, grn_obj *msg) +{ + grn_edge *edge; + grn_com *com = ((grn_msg *)msg)->u.peer; + if (ctx->rc) { + if (com->has_sid) { + if ((edge = com->opaque)) { + MUTEX_LOCK_ENSURE(ctx, q_mutex); + if (edge->stat == EDGE_IDLE) { + grn_com_queue_enque(ctx, &ctx_old, (grn_com_queue_entry *)edge); + } + edge->stat = EDGE_ABORT; + MUTEX_UNLOCK(q_mutex); + } else { + grn_com_close(ctx, com); + } + } + grn_msg_close(ctx, msg); + } else { + int added; + edge = grn_edges_add(ctx, &((grn_msg *)msg)->edge_id, &added); + if (added) { + grn_ctx_init(&edge->ctx, 0); + GRN_COM_QUEUE_INIT(&edge->recv_new); + GRN_COM_QUEUE_INIT(&edge->send_old); + grn_ctx_use(&edge->ctx, (grn_obj *)com->ev->opaque); + grn_ctx_recv_handler_set(&edge->ctx, g_output, edge); + com->opaque = edge; + grn_obj_close(&edge->ctx, edge->ctx.impl->output.buf); + edge->ctx.impl->output.buf = + grn_msg_open(&edge->ctx, com, &edge->send_old); + edge->com = com; + edge->stat = EDGE_IDLE; + edge->flags = GRN_EDGE_WORKER; + } + if (edge->ctx.stat == GRN_CTX_QUIT || edge->stat == EDGE_ABORT) { + grn_msg_close(ctx, msg); + } else { + grn_com_queue_enque(ctx, &edge->recv_new, (grn_com_queue_entry *)msg); + g_dispatcher(ctx, edge); + } + } +} + +static int +g_server(char *path) +{ + int exit_code = EXIT_FAILURE; + grn_ctx ctx_, *ctx = &ctx_; + grn_ctx_init(ctx, 0); + GRN_COM_QUEUE_INIT(&ctx_new); + GRN_COM_QUEUE_INIT(&ctx_old); + check_rlimit_nofile(ctx); + exit_code = start_service(ctx, path, g_dispatcher, g_handler); + grn_ctx_fin(ctx); + return exit_code; +} + +enum { + ACTION_USAGE = 1, + ACTION_VERSION, + ACTION_SHOW_CONFIG, + ACTION_ERROR +}; + +#define ACTION_MASK (0x0f) +#define MODE_MASK (0xf0) +#define FLAG_MODE_ALONE (1 << 4) +#define FLAG_MODE_CLIENT (1 << 5) +#define FLAG_MODE_DAEMON (1 << 6) +#define FLAG_MODE_SERVER (1 << 7) +#define FLAG_NEW_DB (1 << 8) +#define FLAG_USE_WINDOWS_EVENT_LOG (1 << 9) + +static uint32_t +get_core_number(void) +{ +#ifdef WIN32 + SYSTEM_INFO sinfo; + GetSystemInfo(&sinfo); + return sinfo.dwNumberOfProcessors; +#else /* WIN32 */ +# ifdef _SC_NPROCESSORS_CONF + return sysconf(_SC_NPROCESSORS_CONF); +# else + int n_processors; + size_t length = sizeof(n_processors); + int mib[] = {CTL_HW, HW_NCPU}; + if (sysctl(mib, sizeof(mib) / sizeof(mib[0]), + &n_processors, &length, NULL, 0) == 0 && + length == sizeof(n_processors) && + 0 < n_processors) { + return n_processors; + } else { + return 1; + } +# endif /* _SC_NPROCESSORS_CONF */ +#endif /* WIN32 */ +} + +/* + * The length of each line, including an end-of-line, in config file should be + * shorter than (CONFIG_FILE_BUF_SIZE - 1) bytes. Too long lines are ignored. + * Note that both '\r' and '\n' are handled as end-of-lines. + * + * '#' and ';' are special symbols to start comments. A comment ends with an + * end-of-line. + * + * Format: name[=value] + * - Preceding/trailing white-spaces of each line are removed. + * - White-spaces aroung '=' are removed. + * - name does not allow white-spaces. + */ +#define CONFIG_FILE_BUF_SIZE 4096 +#define CONFIG_FILE_MAX_NAME_LENGTH 128 +#define CONFIG_FILE_MAX_VALUE_LENGTH 2048 + +typedef enum { + CONFIG_FILE_SUCCESS, + CONFIG_FILE_FORMAT_ERROR, + CONFIG_FILE_FOPEN_ERROR, + CONFIG_FILE_MALLOC_ERROR, + CONFIG_FILE_ATEXIT_ERROR +} config_file_status; + +/* + * The node type of a linked list for storing values. Note that a value is + * stored in the extra space of an object. + */ +typedef struct _config_file_entry { + struct _config_file_entry *next; +} config_file_entry; + +static config_file_entry *config_file_entry_head = NULL; + +static void +config_file_clear(void) { + while (config_file_entry_head) { + config_file_entry *next = config_file_entry_head->next; + free(config_file_entry_head); + config_file_entry_head = next; + } +} + +static config_file_status +config_file_register(const char *path, const grn_str_getopt_opt *opts, + int *flags, const char *name, size_t name_length, + const char *value, size_t value_length) +{ + char name_buf[CONFIG_FILE_MAX_NAME_LENGTH + 3]; + config_file_entry *entry = NULL; + char *args[4]; + + name_buf[0] = name_buf[1] = '-'; + grn_strcpy(name_buf + 2, CONFIG_FILE_MAX_NAME_LENGTH + 1, name); + + if (value) { + const size_t entry_size = sizeof(config_file_entry) + value_length + 1; + entry = (config_file_entry *)malloc(entry_size); + if (!entry) { + fprintf(stderr, "memory allocation failed: %u bytes\n", + (unsigned int)entry_size); + return CONFIG_FILE_MALLOC_ERROR; + } + grn_strcpy((char *)(entry + 1), value_length + 1, value); + entry->next = config_file_entry_head; + if (!config_file_entry_head) { + if (atexit(config_file_clear)) { + free(entry); + return CONFIG_FILE_ATEXIT_ERROR; + } + } + config_file_entry_head = entry; + } + + args[0] = (char *)path; + args[1] = name_buf; + args[2] = entry ? (char *)(entry + 1) : NULL; + args[3] = NULL; + grn_str_getopt(entry ? 3 : 2, args, opts, flags); + return CONFIG_FILE_SUCCESS; +} + +static config_file_status +config_file_parse(const char *path, const grn_str_getopt_opt *opts, + int *flags, char *buf) { + char *ptr, *name, *value; + size_t name_length, value_length; + + while (isspace((unsigned char)*buf)) { + buf++; + } + + ptr = buf; + while (*ptr && *ptr != '#' && *ptr != ';') { + ptr++; + } + + do { + *ptr-- = '\0'; + } while (ptr >= buf && isspace((unsigned char)*ptr)); + + if (!*buf) { + return CONFIG_FILE_SUCCESS; + } + + name = ptr = buf; + while (*ptr && !isspace((unsigned char)*ptr) && *ptr != '=') { + ptr++; + } + while (isspace((unsigned char)*ptr)) { + *ptr++ = '\0'; + } + + name_length = strlen(name); + if (name_length == 0) { + return CONFIG_FILE_SUCCESS; + } else if (name_length > CONFIG_FILE_MAX_NAME_LENGTH) { + fprintf(stderr, "too long name in config file: %u bytes\n", + (unsigned int)name_length); + return CONFIG_FILE_FORMAT_ERROR; + } + + if (*ptr == '=') { + *ptr++ = '\0'; + while (isspace((unsigned char)*ptr)) { + ptr++; + } + value = ptr; + } else if (*ptr) { + fprintf(stderr, "invalid name in config file\n"); + return CONFIG_FILE_FORMAT_ERROR; + } else { + value = NULL; + } + + value_length = value ? strlen(value) : 0; + if (value_length > CONFIG_FILE_MAX_VALUE_LENGTH) { + fprintf(stderr, "too long value in config file: %u bytes\n", + (unsigned int)value_length); + return CONFIG_FILE_FORMAT_ERROR; + } + + return config_file_register(path, opts, flags, + name, name_length, value, value_length); +} + +static config_file_status +config_file_load(const char *path, const grn_str_getopt_opt *opts, int *flags) +{ + config_file_status status = CONFIG_FILE_SUCCESS; + char buf[CONFIG_FILE_BUF_SIZE]; + size_t length = 0; + FILE * const file = fopen(path, "rb"); + if (!file) { + return CONFIG_FILE_FOPEN_ERROR; + } + + for ( ; ; ) { + int c = fgetc(file); + if (c == '\r' || c == '\n' || c == EOF) { + if (length < sizeof(buf) - 1) { + buf[length] = '\0'; + status = config_file_parse(path, opts, flags, buf); + if (status != CONFIG_FILE_SUCCESS) { + break; + } + } + length = 0; + } else if (c == '\0') { + fprintf(stderr, "prohibited '\\0' in config file: %s\n", path); + status = CONFIG_FILE_FORMAT_ERROR; + break; + } else { + if (length < sizeof(buf) - 1) { + buf[length] = (char)c; + } + length++; + } + + if (c == EOF) { + break; + } + } + + fclose(file); + return status; +} + +static const int default_http_port = DEFAULT_HTTP_PORT; +static const int default_gqtp_port = DEFAULT_GQTP_PORT; +static grn_encoding default_encoding = GRN_ENC_DEFAULT; +static uint32_t default_max_n_threads = DEFAULT_MAX_N_FLOATING_THREADS; +static const grn_log_level default_log_level = GRN_LOG_DEFAULT_LEVEL; +static const char * const default_protocol = "gqtp"; +static const char *default_hostname = "localhost"; +static const char * const default_dest = "localhost"; +static const char *default_log_path = ""; +static const char *default_query_log_path = ""; +static const char *default_config_path = ""; +static const char *default_document_root = ""; +static grn_command_version default_default_command_version = + GRN_COMMAND_VERSION_DEFAULT; +static int64_t default_default_match_escalation_threshold = 0; +static const char * const default_bind_address = "0.0.0.0"; +static double default_default_request_timeout = 0.0; + +static void +init_default_hostname(void) +{ + static char hostname[HOST_NAME_MAX + 1]; + struct addrinfo hints, *result; + + hostname[HOST_NAME_MAX] = '\0'; + if (gethostname(hostname, HOST_NAME_MAX) == -1) + return; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_addr = NULL; + hints.ai_canonname = NULL; + hints.ai_next = NULL; + if (getaddrinfo(hostname, NULL, &hints, &result) != 0) + return; + freeaddrinfo(result); + + default_hostname = hostname; +} + +static void +init_default_settings(void) +{ + output = stdout; + + default_encoding = grn_encoding_parse(GRN_DEFAULT_ENCODING); + + { + const uint32_t n_cores = get_core_number(); + if (n_cores != 0) { + default_max_n_threads = n_cores; + } + } + + init_default_hostname(); + + default_log_path = grn_default_logger_get_path(); + default_query_log_path = grn_default_query_logger_get_path(); + + default_config_path = getenv("GRN_CONFIG_PATH"); + if (!default_config_path) { + default_config_path = GRN_CONFIG_PATH; + if (!default_config_path) { + default_config_path = ""; + } + } + +#ifdef WIN32 + { + static char windows_default_document_root[PATH_MAX]; + size_t document_root_length = strlen(grn_windows_base_dir()) + 1 + + strlen(GRN_DEFAULT_RELATIVE_DOCUMENT_ROOT) + 1; + if (document_root_length >= PATH_MAX) { + fprintf(stderr, "can't use default root: too long path\n"); + } else { + grn_strcpy(windows_default_document_root, PATH_MAX, + grn_windows_base_dir()); + grn_strcat(windows_default_document_root, PATH_MAX, + "/"); + grn_strcat(windows_default_document_root, PATH_MAX, + GRN_DEFAULT_RELATIVE_DOCUMENT_ROOT); + default_document_root = windows_default_document_root; + } + } +#else + default_document_root = GRN_DEFAULT_DOCUMENT_ROOT; +#endif + + default_default_command_version = grn_get_default_command_version(); + default_default_match_escalation_threshold = + grn_get_default_match_escalation_threshold(); + default_default_request_timeout = grn_get_default_request_timeout(); +} + +static void +show_config(FILE *out, const grn_str_getopt_opt *opts, int flags) +{ + const grn_str_getopt_opt *o; + + for (o = opts; o->opt || o->longopt; o++) { + switch (o->op) { + case GETOPT_OP_NONE: + if (o->arg && *o->arg) { + if (o->longopt && strcmp(o->longopt, "config-path")) { + fprintf(out, "%s=%s\n", o->longopt, *o->arg); + } + } + break; + case GETOPT_OP_ON: + if (flags & o->flag) { + goto no_arg; + } + break; + case GETOPT_OP_OFF: + if (!(flags & o->flag)) { + goto no_arg; + } + break; + case GETOPT_OP_UPDATE: + if (flags == o->flag) { + no_arg: + if (o->longopt) { + fprintf(out, "%s\n", o->longopt); + } + } + break; + } + } +} + +static void +show_version(void) +{ + printf("%s %s [", + grn_get_package_label(), + grn_get_version()); + + /* FIXME: Should we detect host information dynamically on Windows? */ +#ifdef HOST_OS + printf("%s,", HOST_OS); +#endif +#ifdef HOST_CPU + printf("%s,", HOST_CPU); +#endif + printf("%s", GRN_DEFAULT_ENCODING); + + printf(",match-escalation-threshold=%" GRN_FMT_LLD, + grn_get_default_match_escalation_threshold()); + +#ifndef NO_NFKC + printf(",nfkc"); +#endif +#ifdef GRN_WITH_MECAB + printf(",mecab"); +#endif +#ifdef GRN_WITH_MESSAGE_PACK + printf(",msgpack"); +#endif +#ifdef GRN_WITH_MRUBY + printf(",mruby"); +#endif +#ifdef GRN_WITH_ONIGMO + printf(",onigmo"); +#endif +#ifdef GRN_WITH_ZLIB + printf(",zlib"); +#endif +#ifdef GRN_WITH_LZ4 + printf(",lz4"); +#endif +#ifdef GRN_WITH_ZSTD + printf(",zstd"); +#endif +#ifdef USE_KQUEUE + printf(",kqueue"); +#endif +#ifdef USE_EPOLL + printf(",epoll"); +#endif +#ifdef USE_POLL + printf(",poll"); +#endif + printf("]\n"); + +#ifdef CONFIGURE_OPTIONS + printf("\n"); + printf("configure options: <%s>\n", CONFIGURE_OPTIONS); +#endif +} + +static void +show_usage(FILE *output) +{ + uint32_t default_cache_limit = GRN_CACHE_DEFAULT_MAX_N_ENTRIES; + + fprintf(output, + "Usage: groonga [options...] [dest]\n" + "\n" + "Mode options: (default: standalone)\n" + " By default, groonga runs in standalone mode.\n" + " -c: run in client mode\n" + " -s: run in server mode\n" + " -d: run in daemon mode\n" + "\n" + "Database creation options:\n" + " -n: create new database (except client mode)\n" + " -e, --encoding <encoding>:\n" + " specify encoding for new database\n" + " [none|euc|utf8|sjis|latin1|koi8r] (default: %s)\n" + "\n" + "Standalone/client options:\n" + " --file <path>: read commands from specified file\n" + " --input-fd <FD>: read commands from specified file descriptor\n" + " --file has a prioriry over --input-fd\n" + " --output-fd <FD>: output response to specified file descriptor\n" + " -p, --port <port number>: specify server port number (client mode only)\n" + " (default: %d)\n" + "\n" + "Server/daemon options:\n" + " --bind-address <ip/hostname>:\n" + " specify server address to bind\n" + " (default: %s)\n" + " -p, --port <port number>: specify server port number\n" + " (HTTP default: %d, GQTP default: %d)\n" + " -i, --server-id <ip/hostname>:\n" + " specify server ID address (default: %s)\n" + " --protocol <protocol>: specify server protocol to listen\n" + " [gqtp|http|memcached] (default: %s)\n" + " --document-root <path>: specify document root path (http only)\n" + " (default: %s)\n" + " --cache-limit <limit>: specify max number of cache data (default: %u)\n" + " -t, --max-threads <max threads>:\n" + " specify max number of threads (default: %u)\n" + " --pid-path <path>: specify file to write process ID to\n" + " (daemon mode only)\n" + " --default-request-timeout <timeout>:\n" + " specify the default request timeout in seconds\n" + " (default: %f)\n" + " --cache-base-path <path>: specify the cache base path\n" + " You can make cache persistent by this option\n" + " You must specify path on memory file system\n" + " (default: none; disabled)\n" + "\n" + "Memcached options:\n" + " --memcached-column <column>:\n" + " specify column to access by memcached protocol\n" + " The column must be text type column and\n" + " its table must be not NO_KEY table\n" + "\n" + "Logging options:\n" + " -l, --log-level <log level>:\n" + " specify log level\n" + " [none|emergency|alert|critical|\n" + " error|warning|notice|info|debug|dump]\n" + " (default: %s)\n" + " --log-path <path>: specify log path\n" + " (default: %s)\n" + " --log-rotate-threshold-size <threshold>:\n" + " specify threshold for log rotate\n" + " Log file is rotated when\n" + " log file size is larger than or\n" + " equals to the threshold\n" + " (default: 0; disabled)\n" +#ifdef WIN32 + " --use-windows-event-log:\n" + " report logs as Windows events\n" +#endif /* WIN32 */ + " --query-log-path <path>:\n" + " specify query log path\n" + " (default: %s)\n" + " --query-log-rotate-threshold-size <threshold>:\n" + " specify threshold for query log rotate\n" + " Query log file is rotated when\n" + " query log file size is larger than or\n" + " equals to the threshold\n" + " (default: 0; disabled)\n" + "\n" + "Common options:\n" + " --working-directory <path>:\n" + " specify working directory path\n" + " (none)\n" + " --config-path <path>:\n" + " specify config file path\n" + " (default: %s)\n" + " --default-command-version <version>:\n" + " specify default command version (default: %d)\n" + " --default-match-escalation-threshold <threshold>:\n" + " specify default match escalation threshold" + " (default: %" GRN_FMT_LLD ")\n" + "\n" + " --show-config: show config\n" + " -h, --help: show usage\n" + " --version: show groonga version\n" + "\n" + "dest:\n" + " <db pathname> [<commands>]: in standalone mode\n" + " <db pathname>: in server/daemon mode\n" + " <dest hostname> [<commands>]: in client mode (default: %s)\n", + grn_encoding_to_string(default_encoding), + default_gqtp_port, default_bind_address, + default_http_port, default_gqtp_port, default_hostname, default_protocol, + default_document_root, default_cache_limit, default_max_n_threads, + default_default_request_timeout, + grn_log_level_to_string(default_log_level), + default_log_path, default_query_log_path, + default_config_path, default_default_command_version, + (long long int)default_default_match_escalation_threshold, + default_dest); +} + +int +main(int argc, char **argv) +{ + const char *port_arg = NULL; + const char *encoding_arg = NULL; + const char *max_n_threads_arg = NULL; + const char *log_level_arg = NULL; + const char *bind_address_arg = NULL; + const char *hostname_arg = NULL; + const char *protocol_arg = NULL; + const char *log_path_arg = GRN_LOG_PATH; + const char *log_rotate_threshold_size_arg = NULL; + const char *query_log_path_arg = NULL; + const char *query_log_rotate_threshold_size_arg = NULL; + const char *cache_limit_arg = NULL; + const char *document_root_arg = NULL; + const char *default_command_version_arg = NULL; + const char *default_match_escalation_threshold_arg = NULL; + const char *input_fd_arg = NULL; + const char *output_fd_arg = NULL; + const char *working_directory_arg = NULL; + const char *config_path = NULL; + const char *default_request_timeout_arg = NULL; + const char *cache_base_path = NULL; + int exit_code = EXIT_SUCCESS; + int i; + int flags = 0; + uint32_t cache_limit = 0; + grn_command_version default_command_version; + int64_t default_match_escalation_threshold = 0; + double default_request_timeout = 0.0; + grn_bool need_line_editor = GRN_FALSE; + static grn_str_getopt_opt opts[] = { + {'p', "port", NULL, 0, GETOPT_OP_NONE}, + {'e', "encoding", NULL, 0, GETOPT_OP_NONE}, + {'t', "max-threads", NULL, 0, GETOPT_OP_NONE}, + {'h', "help", NULL, ACTION_USAGE, GETOPT_OP_UPDATE}, + {'c', NULL, NULL, FLAG_MODE_CLIENT, GETOPT_OP_ON}, + {'d', NULL, NULL, FLAG_MODE_DAEMON, GETOPT_OP_ON}, + {'s', NULL, NULL, FLAG_MODE_SERVER, GETOPT_OP_ON}, + {'l', "log-level", NULL, 0, GETOPT_OP_NONE}, + {'i', "server-id", NULL, 0, GETOPT_OP_NONE}, + {'n', NULL, NULL, FLAG_NEW_DB, GETOPT_OP_ON}, + {'\0', "protocol", NULL, 0, GETOPT_OP_NONE}, + {'\0', "version", NULL, ACTION_VERSION, GETOPT_OP_UPDATE}, + {'\0', "log-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "log-rotate-threshold-size", NULL, 0, GETOPT_OP_NONE}, + {'\0', "query-log-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "query-log-rotate-threshold-size", NULL, 0, GETOPT_OP_NONE}, + {'\0', "pid-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "config-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "show-config", NULL, ACTION_SHOW_CONFIG, GETOPT_OP_UPDATE}, + {'\0', "cache-limit", NULL, 0, GETOPT_OP_NONE}, + {'\0', "file", NULL, 0, GETOPT_OP_NONE}, + {'\0', "document-root", NULL, 0, GETOPT_OP_NONE}, + {'\0', "default-command-version", NULL, 0, GETOPT_OP_NONE}, + {'\0', "default-match-escalation-threshold", NULL, 0, GETOPT_OP_NONE}, + {'\0', "bind-address", NULL, 0, GETOPT_OP_NONE}, + {'\0', "input-fd", NULL, 0, GETOPT_OP_NONE}, + {'\0', "output-fd", NULL, 0, GETOPT_OP_NONE}, + {'\0', "working-directory", NULL, 0, GETOPT_OP_NONE}, + {'\0', "use-windows-event-log", NULL, + FLAG_USE_WINDOWS_EVENT_LOG, GETOPT_OP_ON}, + {'\0', "memcached-column", NULL, 0, GETOPT_OP_NONE}, + {'\0', "default-request-timeout", NULL, 0, GETOPT_OP_NONE}, + {'\0', "cache-base-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', NULL, NULL, 0, 0} + }; + opts[0].arg = &port_arg; + opts[1].arg = &encoding_arg; + opts[2].arg = &max_n_threads_arg; + opts[7].arg = &log_level_arg; + opts[8].arg = &hostname_arg; + opts[10].arg = &protocol_arg; + opts[12].arg = &log_path_arg; + opts[13].arg = &log_rotate_threshold_size_arg; + opts[14].arg = &query_log_path_arg; + opts[15].arg = &query_log_rotate_threshold_size_arg; + opts[16].arg = &pid_file_path; + opts[17].arg = &config_path; + opts[19].arg = &cache_limit_arg; + opts[20].arg = &input_path; + opts[21].arg = &document_root_arg; + opts[22].arg = &default_command_version_arg; + opts[23].arg = &default_match_escalation_threshold_arg; + opts[24].arg = &bind_address_arg; + opts[25].arg = &input_fd_arg; + opts[26].arg = &output_fd_arg; + opts[27].arg = &working_directory_arg; + opts[29].arg = &memcached_column_name; + opts[30].arg = &default_request_timeout_arg; + opts[31].arg = &cache_base_path; + + reset_ready_notify_pipe(); + + init_default_settings(); + + /* only for parsing --config-path. */ + i = grn_str_getopt(argc, argv, opts, &flags); + if (i < 0) { + show_usage(stderr); + return EXIT_FAILURE; + } + + if (config_path) { + const config_file_status status = config_file_load(config_path, opts, &flags); + if (status == CONFIG_FILE_FOPEN_ERROR) { + fprintf(stderr, "%s: can't open config file: %s (%s)\n", + argv[0], config_path, strerror(errno)); + return EXIT_FAILURE; + } else if (status != CONFIG_FILE_SUCCESS) { + fprintf(stderr, "%s: failed to parse config file: %s (%s)\n", + argv[0], config_path, + (status == CONFIG_FILE_FORMAT_ERROR) ? "Invalid format" : strerror(errno)); + return EXIT_FAILURE; + } + } else if (*default_config_path) { + const config_file_status status = + config_file_load(default_config_path, opts, &flags); + if (status != CONFIG_FILE_SUCCESS && status != CONFIG_FILE_FOPEN_ERROR) { + fprintf(stderr, "%s: failed to parse config file: %s (%s)\n", + argv[0], default_config_path, + (status == CONFIG_FILE_FORMAT_ERROR) ? "Invalid format" : strerror(errno)); + return EXIT_FAILURE; + } + } + + if (working_directory_arg) { + if (chdir(working_directory_arg) == -1) { + fprintf(stderr, "%s: failed to change directory: %s: %s\n", + argv[0], working_directory_arg, strerror(errno)); + return EXIT_FAILURE; + } + } + + if (cache_base_path) { + grn_set_default_cache_base_path(cache_base_path); + } + + /* ignore mode option in config file */ + flags = (flags == ACTION_ERROR) ? 0 : (flags & ~ACTION_MASK); + + i = grn_str_getopt(argc, argv, opts, &flags); + if (i < 0) { flags = ACTION_ERROR; } + switch (flags & ACTION_MASK) { + case ACTION_VERSION : + show_version(); + return EXIT_SUCCESS; + case ACTION_USAGE : + show_usage(output); + return EXIT_SUCCESS; + case ACTION_SHOW_CONFIG : + show_config(output, opts, flags & ~ACTION_MASK); + return EXIT_SUCCESS; + case ACTION_ERROR : + show_usage(stderr); + return EXIT_FAILURE; + } + + if ((flags & MODE_MASK) == 0) { + flags |= FLAG_MODE_ALONE; + } + + if (port_arg) { + const char * const end = port_arg + strlen(port_arg); + const char *rest = NULL; + const int value = grn_atoi(port_arg, end, &rest); + if (rest != end || value <= 0 || value > 65535) { + fprintf(stderr, "invalid port number: <%s>\n", port_arg); + return EXIT_FAILURE; + } + port = value; + } else { + if (protocol_arg) { + if (*protocol_arg == 'h' || *protocol_arg == 'H') { + port = default_http_port; + } + } + } + + if (encoding_arg) { + switch (*encoding_arg) { + case 'n' : + case 'N' : + encoding = GRN_ENC_NONE; + break; + case 'e' : + case 'E' : + encoding = GRN_ENC_EUC_JP; + break; + case 'u' : + case 'U' : + encoding = GRN_ENC_UTF8; + break; + case 's' : + case 'S' : + encoding = GRN_ENC_SJIS; + break; + case 'l' : + case 'L' : + encoding = GRN_ENC_LATIN1; + break; + case 'k' : + case 'K' : + encoding = GRN_ENC_KOI8R; + break; + default: + encoding = GRN_ENC_DEFAULT; + break; + } + } else { + encoding = GRN_ENC_DEFAULT; + } + + if (!grn_document_root) { + grn_document_root = default_document_root; + } + + if (protocol_arg) { + switch (*protocol_arg) { + case 'g' : + case 'G' : + do_client = g_client; + do_server = g_server; + break; + case 'h' : + case 'H' : + do_client = g_client; + do_server = h_server; + break; + case 'm' : + case 'M' : + is_memcached_mode = GRN_TRUE; + do_client = g_client; + do_server = g_server; + break; + default : + do_client = g_client; + do_server = g_server; + break; + } + } else { + do_client = g_client; + do_server = g_server; + } + +#ifdef WIN32 + if (flags & FLAG_USE_WINDOWS_EVENT_LOG) { + use_windows_event_log = GRN_TRUE; + } +#endif /* WIN32 */ + + if (use_windows_event_log) { + grn_windows_event_logger_set(NULL, windows_event_source_name); + } + + if (log_path_arg) { + grn_default_logger_set_path(log_path_arg); + } + + if (log_rotate_threshold_size_arg) { + const char * const end = + log_rotate_threshold_size_arg + + strlen(log_rotate_threshold_size_arg); + const char *rest = NULL; + const uint64_t value = grn_atoull(log_rotate_threshold_size_arg, end, &rest); + if (end != rest) { + fprintf(stderr, "invalid log rotate threshold size: <%s>\n", + log_rotate_threshold_size_arg); + return EXIT_FAILURE; + } + grn_default_logger_set_rotate_threshold_size(value); + } + + if (query_log_path_arg) { + grn_default_query_logger_set_path(query_log_path_arg); + } + + if (query_log_rotate_threshold_size_arg) { + const char * const end = + query_log_rotate_threshold_size_arg + + strlen(query_log_rotate_threshold_size_arg); + const char *rest = NULL; + const uint64_t value = + grn_atoull(query_log_rotate_threshold_size_arg, end, &rest); + if (end != rest) { + fprintf(stderr, "invalid query log rotate threshold size: <%s>\n", + query_log_rotate_threshold_size_arg); + return EXIT_FAILURE; + } + grn_default_query_logger_set_rotate_threshold_size(value); + } + + { + grn_log_level log_level; + + if (log_level_arg) { + grn_bool parsed; + + parsed = grn_log_level_parse(log_level_arg, &log_level); + if (!parsed) { + const char * const end = log_level_arg + strlen(log_level_arg); + const char *rest = NULL; + const int value = grn_atoi(log_level_arg, end, &rest); + if (end != rest || value < GRN_LOG_NONE || value > GRN_LOG_DUMP) { + fprintf(stderr, "invalid log level: <%s>\n", log_level_arg); + return EXIT_FAILURE; + } + log_level = value; + } + } else { + log_level = default_log_level; + } + + grn_default_logger_set_max_level(log_level); + } + + if (max_n_threads_arg) { + const char * const end = max_n_threads_arg + strlen(max_n_threads_arg); + const char *rest = NULL; + const uint32_t value = grn_atoui(max_n_threads_arg, end, &rest); + if (end != rest || value < 1 || value > 100) { + fprintf(stderr, "invalid max number of threads: <%s>\n", + max_n_threads_arg); + return EXIT_FAILURE; + } + max_n_floating_threads = value; + } else { + if (flags & FLAG_MODE_ALONE) { + max_n_floating_threads = 1; + } else { + max_n_floating_threads = default_max_n_threads; + } + } + + grn_thread_set_get_limit_func(groonga_get_thread_limit, NULL); + grn_thread_set_set_limit_func(groonga_set_thread_limit, NULL); + + if (output_fd_arg) { + const char * const end = output_fd_arg + strlen(output_fd_arg); + const char *rest = NULL; + const int output_fd = grn_atoi(output_fd_arg, end, &rest); + if (rest != end || output_fd == 0) { + fprintf(stderr, "invalid output FD: <%s>\n", output_fd_arg); + return EXIT_FAILURE; + } + output = fdopen(output_fd, "w"); + if (!output) { + fprintf(stderr, "can't open output FD: %d (%s)\n", + output_fd, strerror(errno)); + return EXIT_FAILURE; + } + } + + + if (bind_address_arg) { + const size_t bind_address_length = strlen(bind_address_arg); + if (bind_address_length > HOST_NAME_MAX) { + fprintf(stderr, "too long bind address: %s (%u bytes):" + " must not be longer than %u bytes\n", + bind_address_arg, (unsigned int)bind_address_length, HOST_NAME_MAX); + return EXIT_FAILURE; + } + grn_strcpy(bind_address, HOST_NAME_MAX + 1, bind_address_arg); + } else { + grn_strcpy(bind_address, HOST_NAME_MAX + 1, default_bind_address); + } + + if (hostname_arg) { + const size_t hostname_length = strlen(hostname_arg); + if (hostname_length > HOST_NAME_MAX) { + fprintf(stderr, "too long hostname: %s (%u bytes):" + " must not be longer than %u bytes\n", + hostname_arg, (unsigned int)hostname_length, HOST_NAME_MAX); + return EXIT_FAILURE; + } + grn_strcpy(hostname, HOST_NAME_MAX + 1, hostname_arg); + } else { + grn_strcpy(hostname, HOST_NAME_MAX + 1, default_hostname); + } + + if (document_root_arg) { + grn_document_root = document_root_arg; + } + + if (default_command_version_arg) { + const char * const end = default_command_version_arg + + strlen(default_command_version_arg); + const char *rest = NULL; + const int value = grn_atoi(default_command_version_arg, end, &rest); + if (end != rest || value < GRN_COMMAND_VERSION_MIN || + value > GRN_COMMAND_VERSION_MAX) { + fprintf(stderr, "invalid command version: <%s>\n", + default_command_version_arg); + return EXIT_FAILURE; + } + default_command_version = value; + } else { + default_command_version = default_default_command_version; + } + + if (default_match_escalation_threshold_arg) { + const char * const end = default_match_escalation_threshold_arg + + strlen(default_match_escalation_threshold_arg); + const char *rest = NULL; + const int64_t value = grn_atoll(default_match_escalation_threshold_arg, end, &rest); + if (end != rest) { + fprintf(stderr, "invalid match escalation threshold: <%s>\n", + default_match_escalation_threshold_arg); + return EXIT_FAILURE; + } + default_match_escalation_threshold = value; + } else { + default_match_escalation_threshold = default_default_match_escalation_threshold; + } + + if (cache_limit_arg) { + const char * const end = cache_limit_arg + strlen(cache_limit_arg); + const char *rest = NULL; + const uint32_t value = grn_atoui(cache_limit_arg, end, &rest); + if (end != rest) { + fprintf(stderr, "invalid --cache-limit value: <%s>\n", cache_limit_arg); + return EXIT_FAILURE; + } + cache_limit = value; + } + + if (default_request_timeout_arg) { + const char * const end = + default_request_timeout_arg + strlen(default_request_timeout_arg); + char *rest = NULL; + double value; + value = strtod(default_request_timeout_arg, &rest); + if (end != rest) { + fprintf(stderr, "invalid default request timeout: <%s>\n", + default_request_timeout_arg); + return EXIT_FAILURE; + } + default_request_timeout = value; + } else { + default_request_timeout = default_default_request_timeout; + } + + grn_gctx.errbuf[0] = '\0'; + if (grn_init()) { + fprintf(stderr, "failed to initialize Groonga: %s\n", grn_gctx.errbuf); + return EXIT_FAILURE; + } + + grn_set_default_encoding(encoding); + + if (default_command_version_arg) { + grn_set_default_command_version(default_command_version); + } + + if (default_match_escalation_threshold_arg) { + grn_set_default_match_escalation_threshold(default_match_escalation_threshold); + } + + if (default_request_timeout_arg) { + grn_set_default_request_timeout(default_request_timeout); + } + + grn_set_segv_handler(); + grn_set_int_handler(); + grn_set_term_handler(); + + if (cache_limit_arg) { + grn_cache *cache; + cache = grn_cache_current_get(&grn_gctx); + grn_cache_set_max_n_entries(&grn_gctx, cache, cache_limit); + } + + MUTEX_INIT(q_mutex); + COND_INIT(q_cond); + + if (input_path) { + input_reader = grn_file_reader_open(&grn_gctx, input_path); + if (!input_reader) { + fprintf(stderr, "can't open input file: %s (%s)\n", + input_path, strerror(errno)); + return EXIT_FAILURE; + } + batchmode = GRN_TRUE; + } else { + if (input_fd_arg) { + const char * const end = input_fd_arg + strlen(input_fd_arg); + const char *rest = NULL; + const int input_fd = grn_atoi(input_fd_arg, end, &rest); + if (rest != end || input_fd == 0) { + fprintf(stderr, "invalid input FD: <%s>\n", input_fd_arg); + return EXIT_FAILURE; + } + if (dup2(input_fd, STDIN_FILENO) == -1) { + fprintf(stderr, "can't open input FD: %d (%s)\n", + input_fd, strerror(errno)); + return EXIT_FAILURE; + } + input_reader = grn_file_reader_open(&grn_gctx, "-"); + if (!input_reader) { + fprintf(stderr, "%s", grn_gctx.errbuf); + return EXIT_FAILURE; + } + batchmode = GRN_TRUE; + } else { + input_reader = grn_file_reader_open(&grn_gctx, "-"); + if (!input_reader) { + fprintf(stderr, "%s", grn_gctx.errbuf); + return EXIT_FAILURE; + } + if (argc - i > 1) { + batchmode = GRN_TRUE; + } else { + batchmode = !grn_isatty(0); + } + } + } + + if ((flags & (FLAG_MODE_ALONE | FLAG_MODE_CLIENT)) && + !batchmode) { + need_line_editor = GRN_TRUE; + } + +#ifdef GRN_WITH_LIBEDIT + if (need_line_editor) { + line_editor_init(argc, argv); + } +#endif + + newdb = (flags & FLAG_NEW_DB); + is_daemon_mode = (flags & FLAG_MODE_DAEMON); + if (flags & FLAG_MODE_CLIENT) { + exit_code = do_client(argc - i, argv + i); + } else if (is_daemon_mode || (flags & FLAG_MODE_SERVER)) { + exit_code = do_server(argc > i ? argv[i] : NULL); + } else { + exit_code = do_alone(argc - i, argv + i); + } + + COND_FIN(q_cond); + MUTEX_FIN(q_mutex); + + if (input_reader) { + grn_file_reader_close(&grn_gctx, input_reader); + } +#ifdef GRN_WITH_LIBEDIT + if (need_line_editor) { + line_editor_fin(); + } +#endif + if (output != stdout) { + fclose(output); + } + grn_fin(); + return exit_code; +} diff --git a/storage/mroonga/vendor/groonga/src/groonga_benchmark.c b/storage/mroonga/vendor/groonga/src/groonga_benchmark.c new file mode 100644 index 00000000..267bb278 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/groonga_benchmark.c @@ -0,0 +1,3176 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2010-2014 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif /* HAVE_CONFIG_H */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> + +#ifdef HAVE_SYS_WAIT_H +# include <sys/wait.h> +#endif /* HAVE_SYS_WAIT_H */ +#ifdef HAVE_SYS_SOCKET_H +# include <sys/socket.h> +#endif /* HAVE_SYS_SOCKET_H */ +#ifndef WIN32 +# include <netinet/in.h> +#endif /* WIN32 */ + +#include <grn_str.h> +#include <grn_com.h> +#include <grn_db.h> + +#ifdef WIN32 +#include <windows.h> +#include <stddef.h> +#else +#include <sys/param.h> +#include <sys/utsname.h> +#include <sys/statvfs.h> +#include <libgen.h> +#endif /* WIN32 */ + +/* +#define DEBUG_FTP +#define DEBUG_HTTP +*/ + +#define FTPUSER "anonymous" +#define FTPPASSWD "grntest" +#define FTPSERVER "ftp.groonga.org" +#define FTPBUF 20000 +#define DEFAULT_PORT 10041 +#define DEFAULT_DEST "localhost" + +#define OUT_JSON 0 +#define OUT_TSV 1 + +static int grntest_outtype = OUT_JSON; + +static grn_critical_section grntest_cs; + +static int grntest_stop_flag = 0; +static int grntest_detail_on = 0; +static int grntest_remote_mode = 0; +static int grntest_localonly_mode = 0; +static int grntest_owndb_mode = 0; +static int grntest_onmemory_mode = 0; +static grn_bool grntest_ftp_mode = GRN_FALSE; +#define TMPFILE "_grntest.tmp" + +static grn_ctx grntest_server_context; +static FILE *grntest_log_file; + +#define OS_LINUX64 "LINUX64" +#define OS_LINUX32 "LINUX32" +#define OS_WINDOWS64 "WINDOWS64" +#define OS_WINDOWS32 "WINDOWS32" + +#ifdef WIN32 +typedef SOCKET socket_t; +#define SOCKETERROR INVALID_SOCKET +#define socketclose closesocket +static const char *groonga_path = "groonga.exe"; +static PROCESS_INFORMATION grntest_pi; +#else +static pid_t grntest_server_id = 0; +typedef int socket_t; +#define socketclose close +#define SOCKETERROR -1 +static const char *groonga_path = "groonga"; +#endif /* WIN32 */ + +static const char *groonga_protocol = "gqtp"; +static const char *grntest_osinfo; +static int grntest_sigint = 0; + + + +static grn_obj *grntest_db = NULL; + +#define MAX_CON_JOB 10 +#define MAX_CON 64 + +#define BUF_LEN 1024 +#define MAX_PATH_LEN 256 + +#define J_DO_LOCAL 1 /* do_local */ +#define J_DO_GQTP 2 /* do_gqtp */ +#define J_DO_HTTP 3 /* do_http */ +#define J_REP_LOCAL 4 /* rep_local */ +#define J_REP_GQTP 5 /* rep_gqtp */ +#define J_REP_HTTP 6 /* rep_http */ +#define J_OUT_LOCAL 7 /* out_local */ +#define J_OUT_GQTP 8 /* out_gqtp */ +#define J_OUT_HTTP 9 /* out_http */ +#define J_TEST_LOCAL 10 /* test_local */ +#define J_TEST_GQTP 11 /* test_gqtp */ +#define J_TEST_HTTP 12 /* test_http */ + +static char grntest_username[BUF_LEN]; +static char grntest_scriptname[BUF_LEN]; +static char grntest_date[BUF_LEN]; +static char grntest_serverhost[BUF_LEN]; +static int grntest_serverport; +static const char *grntest_dbpath; + +struct job { + char jobname[BUF_LEN]; + char commandfile[BUF_LEN]; + int qnum; + int jobtype; + int concurrency; + int ntimes; + int done; + long long int max; + long long int min; + FILE *outputlog; + grn_file_reader *inputlog; + char logfile[BUF_LEN]; +}; + +struct task { + char *file; + grn_obj *commands; + int jobtype; + int ntimes; + int qnum; + int job_id; + long long int max; + long long int min; + socket_t http_socket; + grn_obj http_response; +}; + +static struct task grntest_task[MAX_CON]; +static struct job grntest_job[MAX_CON]; +static int grntest_jobdone; +static int grntest_jobnum; +static grn_ctx grntest_ctx[MAX_CON]; +static grn_obj *grntest_owndb[MAX_CON]; + +static grn_obj grntest_starttime, grntest_jobs_start; + +static int +grntest_atoi(const char *str, const char *end, const char **rest) +{ + while (grn_isspace(str, GRN_ENC_UTF8) == 1) { + str++; + } + return grn_atoi(str, end, rest); +} + +static int +out_p(int jobtype) +{ + if (jobtype == J_OUT_LOCAL) { + return 1; + } + if (jobtype == J_OUT_GQTP) { + return 1; + } + if (jobtype == J_OUT_HTTP) { + return 1; + } + return 0; +} + +static int +test_p(int jobtype) +{ + if (jobtype == J_TEST_LOCAL) { + return 1; + } + if (jobtype == J_TEST_GQTP) { + return 1; + } + if (jobtype == J_TEST_HTTP) { + return 1; + } + return 0; +} + +static int +report_p(int jobtype) +{ + if (jobtype == J_REP_LOCAL) { + return 1; + } + if (jobtype == J_REP_GQTP) { + return 1; + } + if (jobtype == J_REP_HTTP) { + return 1; + } + return 0; +} + +static int +gqtp_p(int jobtype) +{ + if (jobtype == J_DO_GQTP) { + return 1; + } + if (jobtype == J_REP_GQTP) { + return 1; + } + if (jobtype == J_OUT_GQTP) { + return 1; + } + if (jobtype == J_TEST_GQTP) { + return 1; + } + return 0; +} + +static int +http_p(int jobtype) +{ + if (jobtype == J_DO_HTTP) { + return 1; + } + if (jobtype == J_REP_HTTP) { + return 1; + } + if (jobtype == J_OUT_HTTP) { + return 1; + } + if (jobtype == J_TEST_HTTP) { + return 1; + } + return 0; +} + +static int +error_exit_in_thread(intptr_t code) +{ + fprintf(stderr, + "Fatal error! Check script file or database!: %ld\n", (long)code); + fflush(stderr); + CRITICAL_SECTION_ENTER(grntest_cs); + grntest_stop_flag = 1; + CRITICAL_SECTION_LEAVE(grntest_cs); +#ifdef WIN32 + _endthreadex(code); +#else + pthread_exit((void *)code); +#endif /* WIN32 */ + return 0; +} + + +static void +escape_command(grn_ctx *ctx, const char *in, int ilen, grn_obj *escaped_command) +{ + int i = 0; + + while (i < ilen) { + if ((in[i] == '\\') || (in[i] == '\"') || (in[i] == '/')) { + GRN_TEXT_PUTC(ctx, escaped_command, '\\'); + GRN_TEXT_PUTC(ctx, escaped_command, in[i]); + i++; + } else { + switch (in[i]) { + case '\b': + GRN_TEXT_PUTS(ctx, escaped_command, "\\b"); + i++; + break; + case '\f': + GRN_TEXT_PUTS(ctx, escaped_command, "\\f"); + i++; + break; + case '\n': + GRN_TEXT_PUTS(ctx, escaped_command, "\\n"); + i++; + break; + case '\r': + GRN_TEXT_PUTS(ctx, escaped_command, "\\r"); + i++; + break; + case '\t': + GRN_TEXT_PUTS(ctx, escaped_command, "\\t"); + i++; + break; + default: + GRN_TEXT_PUTC(ctx, escaped_command, in[i]); + i++; + break; + } + } + } + GRN_TEXT_PUTC(ctx, escaped_command, '\0'); +} + +static int +report_command(grn_ctx *ctx, const char *command, const char *ret, int task_id, + grn_obj *start_time, grn_obj *end_time) +{ + int i, len, clen; + long long int start, end; + grn_obj result, escaped_command; + + GRN_TEXT_INIT(&result, 0); + if (strncmp(ret, "[[", 2) == 0) { + i = 2; + len = 1; + while (ret[i] != ']') { + i++; + len++; + if (ret[i] == '\0') { + fprintf(stderr, "Error results:command=[%s]\n", command); + error_exit_in_thread(3); + } + } + len++; + grn_text_esc(ctx, &result, ret + 1, len); + } else { + grn_text_esc(ctx, &result, ret, strlen(ret)); + } + + start = GRN_TIME_VALUE(start_time) - GRN_TIME_VALUE(&grntest_starttime); + end = GRN_TIME_VALUE(end_time) - GRN_TIME_VALUE(&grntest_starttime); + clen = strlen(command); + GRN_TEXT_INIT(&escaped_command, 0); + escape_command(ctx, command, clen, &escaped_command); + if (grntest_outtype == OUT_TSV) { + fprintf(grntest_log_file, "report\t%d\t%s\t%" GRN_FMT_LLD "\t%" GRN_FMT_LLD "\t%.*s\n", + task_id, GRN_TEXT_VALUE(&escaped_command), start, end, + (int)GRN_TEXT_LEN(&result), GRN_TEXT_VALUE(&result)); + } else { + fprintf(grntest_log_file, "[%d, \"%s\", %" GRN_FMT_LLD ", %" GRN_FMT_LLD ", %.*s],\n", + task_id, GRN_TEXT_VALUE(&escaped_command), start, end, + (int)GRN_TEXT_LEN(&result), GRN_TEXT_VALUE(&result)); + } + fflush(grntest_log_file); + GRN_OBJ_FIN(ctx, &escaped_command); + GRN_OBJ_FIN(ctx, &result); + return 0; +} + +static int +output_result_final(grn_ctx *ctx, int qnum) +{ + grn_obj end_time; + long long int latency, self; + double sec, qps; + + GRN_TIME_INIT(&end_time, 0); + GRN_TIME_NOW(ctx, &end_time); + + latency = GRN_TIME_VALUE(&end_time) - GRN_TIME_VALUE(&grntest_starttime); + self = latency; + sec = self / (double)1000000; + qps = (double)qnum / sec; + if (grntest_outtype == OUT_TSV) { + fprintf(grntest_log_file, "total\t%" GRN_FMT_LLD "\t%f\t%d\n", latency, qps, qnum); + } else { + fprintf(grntest_log_file, + "{\"total\": %" GRN_FMT_LLD ", \"qps\": %f, \"queries\": %d}]\n", latency, qps, qnum); + } + grn_obj_close(ctx, &end_time); + return 0; +} + +static int +output_sysinfo(char *sysinfo) +{ + if (grntest_outtype == OUT_TSV) { + fprintf(grntest_log_file, "%s", sysinfo); + } else { + fprintf(grntest_log_file, "[%s\n", sysinfo); + } + return 0; +} + +/* #define ENABLE_ERROR_REPORT 1 */ +#ifdef ENABLE_ERROR_REPORT +static int +error_command(grn_ctx *ctx, char *command, int task_id) +{ + fprintf(stderr, "error!:command=[%s] task_id = %d\n", command, task_id); + fflush(stderr); + error_exit_in_thread(1); + return 0; +} +#endif + +static void +normalize_output(char *output, int length, + char **normalized_output, int *normalized_length) +{ + int i; + + *normalized_output = NULL; + *normalized_length = length; + for (i = 0; i < length; i++) { + if (!strncmp(output + i, "],", 2)) { + *normalized_output = output + i + 2; + *normalized_length -= i + 2; + break; + } + } + + if (!*normalized_output) { + if (length > 2 && strncmp(output + length - 2, "]]", 2)) { + *normalized_output = output + length; + *normalized_length = 0; + } else { + *normalized_output = output; + } + } +} + +static grn_bool +same_result_p(char *expect, int expected_length, char *result, int result_length) +{ + char *normalized_expected, *normalized_result; + int normalized_expected_length, normalized_result_length; + + normalize_output(expect, expected_length, + &normalized_expected, &normalized_expected_length); + normalize_output(result, result_length, + &normalized_result, &normalized_result_length); + + return((normalized_expected_length == normalized_result_length) && + strncmp(normalized_expected, normalized_result, + normalized_expected_length) == 0); +} + +static socket_t +open_socket(const char *host, int port) +{ + socket_t sock; + struct hostent *servhost; + struct sockaddr_in server; + u_long inaddr; + int ret; + + servhost = gethostbyname(host); + if (servhost == NULL){ + fprintf(stderr, "Bad hostname [%s]\n", host); + return -1; + } + inaddr = *(u_long*)(servhost->h_addr_list[0]); + + memset(&server, 0, sizeof(struct sockaddr_in)); + server.sin_family = AF_INET; + server.sin_port = htons(port); + server.sin_addr = *(struct in_addr*)&inaddr; + + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock == -1) { + fprintf(stderr, "socket error\n"); + return -1; + } + ret = connect(sock, (struct sockaddr *)&server, sizeof(struct sockaddr_in)); + if (ret == -1) { + fprintf(stderr, "connect error\n"); + return -1; + } + return sock; +} + +static int +write_to_server(socket_t socket, const char *buf) +{ +#ifdef DEBUG_FTP + fprintf(stderr, "send:%s", buf); +#endif + send(socket, buf, strlen(buf), 0); + return 0; +} + +#define OUTPUT_TYPE "output_type" +#define OUTPUT_TYPE_LEN (sizeof(OUTPUT_TYPE) - 1) + +static void +command_line_to_uri_path(grn_ctx *ctx, grn_obj *uri, const char *command) +{ + char tok_type; + int offset = 0, have_key = 0; + const char *p, *e, *v; + grn_obj buf, *expr = NULL; + grn_expr_var *vars; + unsigned int nvars; + + GRN_TEXT_INIT(&buf, 0); + p = command; + e = command + strlen(command); + p = grn_text_unesc_tok(ctx, &buf, p, e, &tok_type); + if ((expr = grn_ctx_get(ctx, GRN_TEXT_VALUE(&buf), GRN_TEXT_LEN(&buf)))) { + grn_obj params, output_type; + + GRN_TEXT_INIT(¶ms, 0); + GRN_TEXT_INIT(&output_type, 0); + vars = ((grn_proc *)expr)->vars; + nvars = ((grn_proc *)expr)->nvars; + GRN_TEXT_PUTS(ctx, uri, "/d/"); + GRN_TEXT_PUT(ctx, uri, GRN_TEXT_VALUE(&buf), GRN_TEXT_LEN(&buf)); + while (p < e) { + GRN_BULK_REWIND(&buf); + p = grn_text_unesc_tok(ctx, &buf, p, e, &tok_type); + v = GRN_TEXT_VALUE(&buf); + switch (tok_type) { + case GRN_TOK_VOID : + p = e; + break; + case GRN_TOK_SYMBOL : + if (GRN_TEXT_LEN(&buf) > 2 && v[0] == '-' && v[1] == '-') { + int l = GRN_TEXT_LEN(&buf) - 2; + v += 2; + if (l == OUTPUT_TYPE_LEN && !memcmp(v, OUTPUT_TYPE, OUTPUT_TYPE_LEN)) { + GRN_BULK_REWIND(&output_type); + p = grn_text_unesc_tok(ctx, &output_type, p, e, &tok_type); + break; + } + if (GRN_TEXT_LEN(¶ms)) { + GRN_TEXT_PUTS(ctx, ¶ms, "&"); + } + grn_text_urlenc(ctx, ¶ms, v, l); + have_key = 1; + break; + } + /* fallthru */ + case GRN_TOK_STRING : + case GRN_TOK_QUOTE : + if (!have_key) { + if (offset < nvars) { + if (GRN_TEXT_LEN(¶ms)) { + GRN_TEXT_PUTS(ctx, ¶ms, "&"); + } + grn_text_urlenc(ctx, ¶ms, + vars[offset].name, vars[offset].name_size); + offset++; + } + } + GRN_TEXT_PUTS(ctx, ¶ms, "="); + grn_text_urlenc(ctx, ¶ms, GRN_TEXT_VALUE(&buf), GRN_TEXT_LEN(&buf)); + have_key = 0; + break; + } + } + GRN_TEXT_PUTS(ctx, uri, "."); + if (GRN_TEXT_LEN(&output_type)) { + GRN_TEXT_PUT(ctx, uri, + GRN_TEXT_VALUE(&output_type), GRN_TEXT_LEN(&output_type)); + } else { + GRN_TEXT_PUTS(ctx, uri, "json"); + } + if (GRN_TEXT_LEN(¶ms) > 0) { + GRN_TEXT_PUTS(ctx, uri, "?"); + GRN_TEXT_PUT(ctx, uri, GRN_TEXT_VALUE(¶ms), GRN_TEXT_LEN(¶ms)); + } + GRN_OBJ_FIN(ctx, ¶ms); + GRN_OBJ_FIN(ctx, &output_type); + } + GRN_OBJ_FIN(ctx, &buf); +} + +static void +command_send_http(grn_ctx *ctx, const char *command, int type, int task_id) +{ + socket_t http_socket; + grn_obj buf; + + http_socket = open_socket(grntest_serverhost, grntest_serverport); + if (http_socket == SOCKETERROR) { + fprintf(stderr, "failed to connect to groonga at %s:%d via HTTP: ", + grntest_serverhost, grntest_serverport); +#ifdef WIN32 + fprintf(stderr, "%lu\n", GetLastError()); +#else + fprintf(stderr, "%s\n", strerror(errno)); +#endif + error_exit_in_thread(100); + } + grntest_task[task_id].http_socket = http_socket; + GRN_BULK_REWIND(&grntest_task[task_id].http_response); + + GRN_TEXT_INIT(&buf, 0); + GRN_TEXT_PUTS(ctx, &buf, "GET "); + if (strncmp(command, "/d/", 3) == 0) { + GRN_TEXT_PUTS(ctx, &buf, command); + } else { + command_line_to_uri_path(ctx, &buf, command); + } +#ifdef DEBUG_HTTP + fprintf(stderr, "command: <%s>\n", command); + fprintf(stderr, "path: <%.*s>\n", + (int)GRN_TEXT_LEN(&buf), GRN_TEXT_VALUE(&buf)); +#endif + GRN_TEXT_PUTS(ctx, &buf, " HTTP/1.1\r\n"); + GRN_TEXT_PUTS(ctx, &buf, "Host: "); + GRN_TEXT_PUTS(ctx, &buf, grntest_serverhost); + GRN_TEXT_PUTS(ctx, &buf, "\r\n"); + GRN_TEXT_PUTS(ctx, &buf, "User-Agent: grntest/"); + GRN_TEXT_PUTS(ctx, &buf, grn_get_version()); + GRN_TEXT_PUTS(ctx, &buf, "\r\n"); + GRN_TEXT_PUTS(ctx, &buf, "Connection: close\r\n"); + GRN_TEXT_PUTS(ctx, &buf, "\r\n"); + GRN_TEXT_PUTC(ctx, &buf, '\0'); + write_to_server(http_socket, GRN_TEXT_VALUE(&buf)); + GRN_OBJ_FIN(ctx, &buf); +} + +static void +command_send_ctx(grn_ctx *ctx, const char *command, int type, int task_id) +{ + grn_ctx_send(ctx, command, strlen(command), 0); +/* fix me. + when command fails, ctx->rc is not 0 in local mode! + if (ctx->rc) { + fprintf(stderr, "ctx_send:rc=%d:command:%s\n", ctx->rc, command); + error_exit_in_thread(1); + } +*/ +} + +static void +command_send(grn_ctx *ctx, const char *command, int type, int task_id) +{ + if (http_p(type)) { + command_send_http(ctx, command, type, task_id); + } else { + command_send_ctx(ctx, command, type, task_id); + } +} + +static void +command_recv_http(grn_ctx *ctx, int type, int task_id, + char **res, unsigned int *res_len, int *flags) +{ + int len; + char buf[BUF_LEN]; + char *p, *e; + socket_t http_socket; + grn_obj *http_response; + + http_socket = grntest_task[task_id].http_socket; + http_response = &grntest_task[task_id].http_response; + while ((len = recv(http_socket, buf, BUF_LEN - 1, 0))) { +#ifdef DEBUG_HTTP + fprintf(stderr, "receive: <%.*s>\n", len, buf); +#endif + GRN_TEXT_PUT(ctx, http_response, buf, len); + } + + p = GRN_TEXT_VALUE(http_response); + e = p + GRN_TEXT_LEN(http_response); + while (p < e) { + if (p[0] != '\r') { + p++; + continue; + } + if (e - p >= 4) { + if (!memcmp(p, "\r\n\r\n", 4)) { + *res = p + 4; + *res_len = e - *res; +#ifdef DEBUG_HTTP + fprintf(stderr, "body: <%.*s>\n", *res_len, *res); +#endif + break; + } + p += 4; + } else { + *res = NULL; + *res_len = 0; + break; + } + } + + socketclose(http_socket); + grntest_task[task_id].http_socket = 0; +} + +static void +command_recv_ctx(grn_ctx *ctx, int type, int task_id, + char **res, unsigned int *res_len, int *flags) +{ + grn_ctx_recv(ctx, res, res_len, flags); + if (ctx->rc) { + fprintf(stderr, "ctx_recv:rc=%d\n", ctx->rc); + error_exit_in_thread(1); + } +} + +static void +command_recv(grn_ctx *ctx, int type, int task_id, + char **res, unsigned int *res_len, int *flags) +{ + if (http_p(type)) { + command_recv_http(ctx, type, task_id, res, res_len, flags); + } else { + command_recv_ctx(ctx, type, task_id, res, res_len, flags); + } +} + +static int +shutdown_server(void) +{ + char *res; + int flags; + unsigned int res_len; + int job_type; + int task_id = 0; + + if (grntest_remote_mode) { + return 0; + } + job_type = grntest_task[task_id].jobtype; + command_send(&grntest_server_context, "shutdown", job_type, task_id); + if (grntest_server_context.rc) { + fprintf(stderr, "ctx_send:rc=%d\n", grntest_server_context.rc); + exit(1); + } + command_recv(&grntest_server_context, job_type, task_id, + &res, &res_len, &flags); + + return 0; +} + +static int +do_load_command(grn_ctx *ctx, char *command, int type, int task_id, + long long int *load_start) +{ + char *res; + unsigned int res_len; + int flags, ret; + grn_obj start_time, end_time; + + GRN_TIME_INIT(&start_time, 0); + if (*load_start == 0) { + GRN_TIME_NOW(ctx, &start_time); + *load_start = GRN_TIME_VALUE(&start_time); + } else { + GRN_TIME_SET(ctx, &start_time, *load_start); + } + + command_send(ctx, command, type, task_id); + do { + command_recv(ctx, type, task_id, &res, &res_len, &flags); + if (res_len) { + long long int self; + GRN_TIME_INIT(&end_time, 0); + GRN_TIME_NOW(ctx, &end_time); + + self = GRN_TIME_VALUE(&end_time) - *load_start; + + if (grntest_task[task_id].max < self) { + grntest_task[task_id].max = self; + } + if (grntest_task[task_id].min > self) { + grntest_task[task_id].min = self; + } + + if (report_p(grntest_task[task_id].jobtype)) { + char tmpbuf[BUF_LEN]; + + if (res_len < BUF_LEN) { + strncpy(tmpbuf, res, res_len); + tmpbuf[res_len] = '\0'; + } else { + strncpy(tmpbuf, res, BUF_LEN - 2); + tmpbuf[BUF_LEN -2] = '\0'; + } + report_command(ctx, "load", tmpbuf, task_id, &start_time, &end_time); + } + if (out_p(grntest_task[task_id].jobtype)) { + fwrite(res, 1, res_len, grntest_job[grntest_task[task_id].job_id].outputlog); + fputc('\n', grntest_job[grntest_task[task_id].job_id].outputlog); + fflush(grntest_job[grntest_task[task_id].job_id].outputlog); + } + if (test_p(grntest_task[task_id].jobtype)) { + grn_obj log; + grn_file_reader *input; + FILE *output; + GRN_TEXT_INIT(&log, 0); + input = grntest_job[grntest_task[task_id].job_id].inputlog; + output = grntest_job[grntest_task[task_id].job_id].outputlog; + if (grn_file_reader_read_line(ctx, input, &log) != GRN_SUCCESS) { + GRN_LOG(ctx, GRN_ERROR, "Cannot get input-log"); + error_exit_in_thread(55); + } + if (GRN_TEXT_VALUE(&log)[GRN_TEXT_LEN(&log) - 1] == '\n') { + grn_bulk_truncate(ctx, &log, GRN_TEXT_LEN(&log) - 1); + } + + if (!same_result_p(GRN_TEXT_VALUE(&log), GRN_TEXT_LEN(&log), + res, res_len)) { + fprintf(output, "DIFF:command:%s\n", command); + fprintf(output, "DIFF:result:"); + fwrite(res, 1, res_len, output); + fputc('\n', output); + fprintf(output, "DIFF:expect:%.*s\n", + (int)GRN_TEXT_LEN(&log), GRN_TEXT_VALUE(&log)); + fflush(output); + } + GRN_OBJ_FIN(ctx, &log); + } + grn_obj_close(ctx, &end_time); + ret = 1; + break; + } else { + ret = 0; + break; + } + } while ((flags & GRN_CTX_MORE)); + grn_obj_close(ctx, &start_time); + + return ret; +} + + +static int +do_command(grn_ctx *ctx, char *command, int type, int task_id) +{ + char *res; + unsigned int res_len; + int flags; + grn_obj start_time, end_time; + + GRN_TIME_INIT(&start_time, 0); + GRN_TIME_NOW(ctx, &start_time); + + command_send(ctx, command, type, task_id); + do { + command_recv(ctx, type, task_id, &res, &res_len, &flags); + if (res_len) { + long long int self; + GRN_TIME_INIT(&end_time, 0); + GRN_TIME_NOW(ctx, &end_time); + + self = GRN_TIME_VALUE(&end_time) - GRN_TIME_VALUE(&start_time); + + if (grntest_task[task_id].max < self) { + grntest_task[task_id].max = self; + } + if (grntest_task[task_id].min > self) { + grntest_task[task_id].min = self; + } + + if (report_p(grntest_task[task_id].jobtype)) { + char tmpbuf[BUF_LEN]; + + if (res_len < BUF_LEN) { + strncpy(tmpbuf, res, res_len); + tmpbuf[res_len] = '\0'; + } else { + strncpy(tmpbuf, res, BUF_LEN - 2); + tmpbuf[BUF_LEN -2] = '\0'; + } + report_command(ctx, command, tmpbuf, task_id, &start_time, &end_time); + } + if (out_p(grntest_task[task_id].jobtype)) { + fwrite(res, 1, res_len, grntest_job[grntest_task[task_id].job_id].outputlog); + fputc('\n', grntest_job[grntest_task[task_id].job_id].outputlog); + fflush(grntest_job[grntest_task[task_id].job_id].outputlog); + } + if (test_p(grntest_task[task_id].jobtype)) { + grn_obj log; + grn_file_reader *input; + FILE *output; + GRN_TEXT_INIT(&log, 0); + input = grntest_job[grntest_task[task_id].job_id].inputlog; + output = grntest_job[grntest_task[task_id].job_id].outputlog; + if (grn_file_reader_read_line(ctx, input, &log) != GRN_SUCCESS) { + GRN_LOG(ctx, GRN_ERROR, "Cannot get input-log"); + error_exit_in_thread(55); + } + if (GRN_TEXT_VALUE(&log)[GRN_TEXT_LEN(&log) - 1] == '\n') { + grn_bulk_truncate(ctx, &log, GRN_TEXT_LEN(&log) - 1); + } + + if (!same_result_p(GRN_TEXT_VALUE(&log), GRN_TEXT_LEN(&log), + res, res_len)) { + fprintf(output, "DIFF:command:%s\n", command); + fprintf(output, "DIFF:result:"); + fwrite(res, 1, res_len, output); + fputc('\n', output); + fprintf(output, "DIFF:expect:%.*s\n", + (int)GRN_TEXT_LEN(&log), GRN_TEXT_VALUE(&log)); + fflush(output); + } + GRN_OBJ_FIN(ctx, &log); + } + grn_obj_close(ctx, &end_time); + break; + } else { +#ifdef ENABLE_ERROR_REPORT + error_command(ctx, command, task_id); +#endif + } + } while ((flags & GRN_CTX_MORE)); + + grn_obj_close(ctx, &start_time); + + return 0; +} + +static int +comment_p(char *command) +{ + if (command[0] == '#') { + return 1; + } + return 0; +} + +static int +load_command_p(char *command) +{ + int i = 0; + + while (grn_isspace(&command[i], GRN_ENC_UTF8) == 1) { + i++; + } + if (command[i] == '\0') { + return 0; + } + if (!strncmp(&command[i], "load", 4)) { + return 1; + } + return 0; +} + +static int +worker_sub(grn_ctx *ctx, grn_obj *log, int task_id) +{ + int i, load_mode, load_count; + grn_obj end_time; + long long int total_elapsed_time, job_elapsed_time; + double sec, qps; + long long int load_start; + struct task *task; + struct job *job; + + task = &(grntest_task[task_id]); + task->max = 0LL; + task->min = 9223372036854775807LL; + task->qnum = 0; + + for (i = 0; i < task->ntimes; i++) { + if (task->file != NULL) { + grn_file_reader *reader; + grn_obj line; + reader = grn_file_reader_open(ctx, task->file); + if (!reader) { + fprintf(stderr, "Cannot open %s\n",grntest_task[task_id].file); + error_exit_in_thread(1); + } + load_mode = 0; + load_count = 0; + load_start = 0LL; + GRN_TEXT_INIT(&line, 0); + while (grn_file_reader_read_line(ctx, reader, &line) == GRN_SUCCESS) { + if (GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] == '\n') { + grn_bulk_truncate(ctx, &line, GRN_TEXT_LEN(&line) - 1); + } + if (GRN_TEXT_LEN(&line) == 0) { + GRN_BULK_REWIND(&line); + continue; + } + GRN_TEXT_PUTC(ctx, &line, '\0'); + if (comment_p(GRN_TEXT_VALUE(&line))) { + GRN_BULK_REWIND(&line); + continue; + } + if (load_command_p(GRN_TEXT_VALUE(&line))) { + load_mode = 1; + load_count = 1; + } + if (load_mode == 1) { + if (do_load_command(&grntest_ctx[task_id], GRN_TEXT_VALUE(&line), + task->jobtype, + task_id, &load_start)) { + task->qnum += load_count; + load_mode = 0; + load_count = 0; + load_start = 0LL; + } + load_count++; + GRN_BULK_REWIND(&line); + continue; + } + do_command(&grntest_ctx[task_id], GRN_TEXT_VALUE(&line), + task->jobtype, + task_id); + task->qnum++; + GRN_BULK_REWIND(&line); + if (grntest_sigint) { + goto exit; + } + } + GRN_OBJ_FIN(ctx, &line); + grn_file_reader_close(ctx, reader); + } else { + int i, n_commands; + grn_obj *commands; + commands = task->commands; + if (!commands) { + error_exit_in_thread(1); + } + load_mode = 0; + n_commands = GRN_BULK_VSIZE(commands) / sizeof(grn_obj *); + for (i = 0; i < n_commands; i++) { + grn_obj *command; + command = GRN_PTR_VALUE_AT(commands, i); + if (load_command_p(GRN_TEXT_VALUE(command))) { + load_mode = 1; + } + if (load_mode == 1) { + if (do_load_command(&grntest_ctx[task_id], + GRN_TEXT_VALUE(command), + task->jobtype, task_id, &load_start)) { + load_mode = 0; + load_start = 0LL; + task->qnum++; + } + continue; + } + do_command(&grntest_ctx[task_id], + GRN_TEXT_VALUE(command), + task->jobtype, task_id); + task->qnum++; + if (grntest_sigint) { + goto exit; + } + } + } + } + +exit: + GRN_TIME_INIT(&end_time, 0); + GRN_TIME_NOW(&grntest_ctx[task_id], &end_time); + total_elapsed_time = GRN_TIME_VALUE(&end_time) - GRN_TIME_VALUE(&grntest_starttime); + job_elapsed_time = GRN_TIME_VALUE(&end_time) - GRN_TIME_VALUE(&grntest_jobs_start); + + CRITICAL_SECTION_ENTER(grntest_cs); + job = &(grntest_job[task->job_id]); + if (job->max < task->max) { + job->max = task->max; + } + if (job->min > task->min) { + job->min = task->min; + } + + job->qnum += task->qnum; + job->done++; + if (job->done == job->concurrency) { + char tmpbuf[BUF_LEN]; + sec = job_elapsed_time / (double)1000000; + qps = (double)job->qnum/ sec; + grntest_jobdone++; + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, + "job\t" + "%s\t" + "%" GRN_FMT_LLD "\t" + "%" GRN_FMT_LLD "\t" + "%f\t" + "%" GRN_FMT_LLD "\t" + "%" GRN_FMT_LLD "\t" + "%d\n", + job->jobname, + total_elapsed_time, + job_elapsed_time, + qps, + job->min, + job->max, + job->qnum); + } else { + sprintf(tmpbuf, + "{\"job\": \"%s\", " + "\"total_elapsed_time\": %" GRN_FMT_LLD ", " + "\"job_elapsed_time\": %" GRN_FMT_LLD ", " + "\"qps\": %f, " + "\"min\": %" GRN_FMT_LLD ", " + "\"max\": %" GRN_FMT_LLD ", " + "\"queries\": %d}", + job->jobname, + total_elapsed_time, + job_elapsed_time, + qps, + job->min, + job->max, + job->qnum); + if (grntest_jobdone < grntest_jobnum) { + grn_strcat(tmpbuf, BUF_LEN, ","); + } + } + GRN_TEXT_PUTS(ctx, log, tmpbuf); + if (grntest_jobdone == grntest_jobnum) { + if (grntest_outtype == OUT_TSV) { + fprintf(grntest_log_file, "%.*s", + (int)GRN_TEXT_LEN(log), GRN_TEXT_VALUE(log)); + } else { + if (grntest_detail_on) { + fseek(grntest_log_file, -2, SEEK_CUR); + fprintf(grntest_log_file, "],\n"); + } + fprintf(grntest_log_file, "\"summary\": ["); + fprintf(grntest_log_file, "%.*s", + (int)GRN_TEXT_LEN(log), GRN_TEXT_VALUE(log)); + fprintf(grntest_log_file, "]"); + } + fflush(grntest_log_file); + } + } + grn_obj_close(&grntest_ctx[task_id], &end_time); + CRITICAL_SECTION_LEAVE(grntest_cs); + + return 0; +} + +typedef struct _grntest_worker { + grn_ctx *ctx; + grn_obj log; + int task_id; +} grntest_worker; + +#ifdef WIN32 +static unsigned int +__stdcall +worker(void *val) +{ + grntest_worker *worker = val; + worker_sub(worker->ctx, &worker->log, worker->task_id); + return 0; +} +#else +static void * +worker(void *val) +{ + grntest_worker *worker = val; + worker_sub(worker->ctx, &worker->log, worker->task_id); + return NULL; +} +#endif /* WIN32 */ + +#ifdef WIN32 +static int +thread_main(grn_ctx *ctx, int num) +{ + int i; + int ret; + HANDLE pthread[MAX_CON]; + grntest_worker *workers[MAX_CON]; + + for (i = 0; i < num; i++) { + workers[i] = GRN_MALLOC(sizeof(grntest_worker)); + workers[i]->ctx = &grntest_ctx[i]; + GRN_TEXT_INIT(&workers[i]->log, 0); + workers[i]->task_id = i; + pthread[i] = (HANDLE)_beginthreadex(NULL, 0, worker, (void *)workers[i], + 0, NULL); + if (pthread[i]== (HANDLE)0) { + fprintf(stderr, "thread failed:%d\n", i); + error_exit_in_thread(1); + } + } + + ret = WaitForMultipleObjects(num, pthread, TRUE, INFINITE); + if (ret == WAIT_TIMEOUT) { + fprintf(stderr, "timeout\n"); + error_exit_in_thread(1); + } + + for (i = 0; i < num; i++) { + CloseHandle(pthread[i]); + GRN_OBJ_FIN(workers[i]->ctx, &workers[i]->log); + GRN_FREE(workers[i]); + } + return 0; +} +#else +static int +thread_main(grn_ctx *ctx, int num) +{ + intptr_t i; + int ret; + pthread_t pthread[MAX_CON]; + grntest_worker *workers[MAX_CON]; + + for (i = 0; i < num; i++) { + workers[i] = GRN_MALLOC(sizeof(grntest_worker)); + workers[i]->ctx = &grntest_ctx[i]; + GRN_TEXT_INIT(&workers[i]->log, 0); + workers[i]->task_id = i; + ret = pthread_create(&pthread[i], NULL, worker, (void *)workers[i]); + if (ret) { + fprintf(stderr, "Cannot create thread:ret=%d\n", ret); + error_exit_in_thread(1); + } + } + + for (i = 0; i < num; i++) { + ret = pthread_join(pthread[i], NULL); + GRN_OBJ_FIN(workers[i]->ctx, &workers[i]->log); + GRN_FREE(workers[i]); + if (ret) { + fprintf(stderr, "Cannot join thread:ret=%d\n", ret); + error_exit_in_thread(1); + } + } + return 0; +} +#endif + +static int +error_exit(grn_ctx *ctx, int ret) +{ + fflush(stderr); + shutdown_server(); + grn_ctx_fin(ctx); + grn_fin(); + exit(ret); +} + +static int +get_sysinfo(const char *path, char *result, int olen) +{ + char tmpbuf[256]; + +#ifdef WIN32 + ULARGE_INTEGER dinfo; + char cpustring[64]; + SYSTEM_INFO sinfo; + MEMORYSTATUSEX minfo; + OSVERSIONINFO osinfo; + + if (grntest_outtype == OUT_TSV) { + result[0] = '\0'; + sprintf(tmpbuf, "script\t%s\n", grntest_scriptname); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, "user\t%s\n", grntest_username); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, "date\t%s\n", grntest_date); + grn_strcat(result, olen, tmpbuf); + } else { + grn_strcpy(result, olen, "{"); + sprintf(tmpbuf, "\"script\": \"%s.scr\",\n", grntest_scriptname); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, " \"user\": \"%s\",\n", grntest_username); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, " \"date\": \"%s\",\n", grntest_date); + grn_strcat(result, olen, tmpbuf); + } + + memset(cpustring, 0, 64); +#ifndef __GNUC__ + { + int cinfo[4]; + __cpuid(cinfo, 0x80000002); + memcpy(cpustring, cinfo, 16); + __cpuid(cinfo, 0x80000003); + memcpy(cpustring+16, cinfo, 16); + __cpuid(cinfo, 0x80000004); + memcpy(cpustring+32, cinfo, 16); + } +#endif + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%s\n", cpustring); + } else { + sprintf(tmpbuf, " \"CPU\": \"%s\",\n", cpustring); + } + grn_strcat(result, olen, tmpbuf); + + if (sizeof(int *) == 8) { + grntest_osinfo = OS_WINDOWS64; + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "64BIT\n"); + } else { + sprintf(tmpbuf, " \"BIT\": 64,\n"); + } + } else { + grntest_osinfo = OS_WINDOWS32; + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "32BIT\n"); + } else { + sprintf(tmpbuf, " \"BIT\": 32,\n"); + } + } + grn_strcat(result, olen, tmpbuf); + + GetSystemInfo(&sinfo); + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "CORE\t%lu\n", sinfo.dwNumberOfProcessors); + } else { + sprintf(tmpbuf, " \"CORE\": %lu,\n", sinfo.dwNumberOfProcessors); + } + grn_strcat(result, olen, tmpbuf); + + minfo.dwLength = sizeof(MEMORYSTATUSEX); + GlobalMemoryStatusEx(&minfo); + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "RAM\t%I64dMByte\n", minfo.ullTotalPhys/(1024*1024)); + } else { + sprintf(tmpbuf, " \"RAM\": \"%I64dMByte\",\n", minfo.ullTotalPhys/(1024*1024)); + } + grn_strcat(result, olen, tmpbuf); + + GetDiskFreeSpaceEx(NULL, NULL, &dinfo, NULL); + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "HDD\t%I64dKBytes\n", dinfo.QuadPart/1024 ); + } else { + sprintf(tmpbuf, " \"HDD\": \"%I64dKBytes\",\n", dinfo.QuadPart/1024 ); + } + grn_strcat(result, olen, tmpbuf); + + osinfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); GetVersionEx(&osinfo); + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "Windows %ld.%ld\n", + osinfo.dwMajorVersion, osinfo.dwMinorVersion); + } else { + sprintf(tmpbuf, " \"OS\": \"Windows %lu.%lu\",\n", osinfo.dwMajorVersion, + osinfo.dwMinorVersion); + } + grn_strcat(result, olen, tmpbuf); + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%s\n", grntest_serverhost); + } else { + sprintf(tmpbuf, " \"HOST\": \"%s\",\n", grntest_serverhost); + } + grn_strcat(result, olen, tmpbuf); + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%d\n", grntest_serverport); + } else { + sprintf(tmpbuf, " \"PORT\": \"%d\",\n", grntest_serverport); + } + grn_strcat(result, olen, tmpbuf); + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%s\"\n", grn_get_version()); + } else { + sprintf(tmpbuf, " \"VERSION\": \"%s\"\n", grn_get_version()); + } + + grn_strcat(result, olen, tmpbuf); + if (grntest_outtype != OUT_TSV) { + grn_strcat(result, olen, "}"); + } + +#else /* linux only */ + FILE *fp; + int ret; + int cpunum = 0; + int minfo = 0; + int unevictable = 0; + int mlocked = 0; +#define CPU_STRING_SIZE 256 + char cpu_string[CPU_STRING_SIZE]; + struct utsname ubuf; + struct statvfs vfsbuf; + + if (grntest_outtype == OUT_TSV) { + result[0] = '\0'; + sprintf(tmpbuf, "sctipt\t%s\n", grntest_scriptname); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, "user\t%s\n", grntest_username); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, "date\t%s\n", grntest_date); + grn_strcat(result, olen, tmpbuf); + } else { + grn_strcpy(result, olen, "{"); + sprintf(tmpbuf, "\"script\": \"%s.scr\",\n", grntest_scriptname); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, " \"user\": \"%s\",\n", grntest_username); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, " \"date\": \"%s\",\n", grntest_date); + grn_strcat(result, olen, tmpbuf); + } + + fp = fopen("/proc/cpuinfo", "r"); + if (!fp) { + fprintf(stderr, "Cannot open cpuinfo\n"); + exit(1); + } + while (fgets(tmpbuf, 256, fp) != NULL) { + tmpbuf[strlen(tmpbuf)-1] = '\0'; + if (!strncmp(tmpbuf, "model name\t: ", 13)) { + grn_strcpy(cpu_string, CPU_STRING_SIZE, &tmpbuf[13]); + } + } + fclose(fp); +#undef CPU_STRING_SIZE + + cpunum = sysconf(_SC_NPROCESSORS_CONF); + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%s\n", cpu_string); + } else { + sprintf(tmpbuf, " \"CPU\": \"%s\",\n", cpu_string); + } + grn_strcat(result, olen, tmpbuf); + + if (sizeof(int *) == 8) { + grntest_osinfo = OS_LINUX64; + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "64BIT\n"); + } else { + sprintf(tmpbuf, " \"BIT\": 64,\n"); + } + } else { + grntest_osinfo = OS_LINUX32; + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "32BIT\n"); + } else { + sprintf(tmpbuf, " \"BIT\": 32,\n"); + } + } + grn_strcat(result, olen, tmpbuf); + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "CORE\t%d\n", cpunum); + } else { + sprintf(tmpbuf, " \"CORE\": %d,\n", cpunum); + } + grn_strcat(result, olen, tmpbuf); + + fp = fopen("/proc/meminfo", "r"); + if (!fp) { + fprintf(stderr, "Cannot open meminfo\n"); + exit(1); + } + while (fgets(tmpbuf, 256, fp) != NULL) { + tmpbuf[strlen(tmpbuf)-1] = '\0'; + if (!strncmp(tmpbuf, "MemTotal:", 9)) { + minfo = grntest_atoi(&tmpbuf[10], &tmpbuf[10] + 40, NULL); + } + if (!strncmp(tmpbuf, "Unevictable:", 12)) { + unevictable = grntest_atoi(&tmpbuf[13], &tmpbuf[13] + 40, NULL); + } + if (!strncmp(tmpbuf, "Mlocked:", 8)) { + mlocked = grntest_atoi(&tmpbuf[9], &tmpbuf[9] + 40, NULL); + } + } + fclose(fp); + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%dMBytes\n", minfo/1024); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, "%dMBytes_Unevictable\n", unevictable/1024); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, "%dMBytes_Mlocked\n", mlocked/1024); + grn_strcat(result, olen, tmpbuf); + } else { + sprintf(tmpbuf, " \"RAM\": \"%dMBytes\",\n", minfo/1024); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, " \"Unevictable\": \"%dMBytes\",\n", unevictable/1024); + grn_strcat(result, olen, tmpbuf); + sprintf(tmpbuf, " \"Mlocked\": \"%dMBytes\",\n", mlocked/1024); + grn_strcat(result, olen, tmpbuf); + } + + ret = statvfs(path, &vfsbuf); + if (ret) { + fprintf(stderr, "Cannot access %s\n", path); + exit(1); + } + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%" GRN_FMT_INT64U "KBytes\n", vfsbuf.f_blocks * 4); + } else { + sprintf(tmpbuf, + " \"HDD\": \"%" GRN_FMT_INT64U "KBytes\",\n", + vfsbuf.f_blocks * 4); + } + grn_strcat(result, olen, tmpbuf); + + uname(&ubuf); + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%s %s\n", ubuf.sysname, ubuf.release); + } else { + sprintf(tmpbuf, " \"OS\": \"%s %s\",\n", ubuf.sysname, ubuf.release); + } + grn_strcat(result, olen, tmpbuf); + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%s\n", grntest_serverhost); + } else { + sprintf(tmpbuf, " \"HOST\": \"%s\",\n", grntest_serverhost); + } + grn_strcat(result, olen, tmpbuf); + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%d\n", grntest_serverport); + } else { + sprintf(tmpbuf, " \"PORT\": \"%d\",\n", grntest_serverport); + } + grn_strcat(result, olen, tmpbuf); + + if (grntest_outtype == OUT_TSV) { + sprintf(tmpbuf, "%s\n", grn_get_version()); + } else { + sprintf(tmpbuf, " \"VERSION\": \"%s\"\n", grn_get_version()); + } + grn_strcat(result, olen, tmpbuf); + + if (grntest_outtype != OUT_TSV) { + grn_strcat(result, olen, "},"); + } +#endif /* WIN32 */ + if (strlen(result) >= olen) { + fprintf(stderr, "buffer overrun in get_sysinfo!\n"); + exit(1); + } + return 0; +} + +static int +start_server(const char *dbpath, int r) +{ + int ret; + char optbuf[BUF_LEN]; +#ifdef WIN32 + char tmpbuf[BUF_LEN]; + + STARTUPINFO si; + + if (strlen(dbpath) > BUF_LEN - 100) { + fprintf(stderr, "too long dbpath!\n"); + exit(1); + } + + grn_strcpy(tmpbuf, BUF_LEN, groonga_path); + grn_strcat(tmpbuf, BUF_LEN, " -s --protocol "); + grn_strcat(tmpbuf, BUF_LEN, groonga_protocol); + grn_strcat(tmpbuf, BUF_LEN, " -p "); + sprintf(optbuf, "%d ", grntest_serverport); + grn_strcat(tmpbuf, BUF_LEN, optbuf); + grn_strcat(tmpbuf, BUF_LEN, dbpath); + memset(&si, 0, sizeof(STARTUPINFO)); + si.cb=sizeof(STARTUPINFO); + ret = CreateProcess(NULL, tmpbuf, NULL, NULL, FALSE, + 0, NULL, NULL, &si, &grntest_pi); + + if (ret == 0) { + fprintf(stderr, "Cannot start groonga server: <%s>: error=%lu\n", + groonga_path, GetLastError()); + exit(1); + } + +#else + pid_t pid; + pid = fork(); + if (pid < 0) { + fprintf(stderr, "Cannot start groonga server:Cannot fork\n"); + exit(1); + } + sprintf(optbuf, "%d", grntest_serverport); + if (pid == 0) { + ret = execlp(groonga_path, groonga_path, + "-s", + "--protocol", groonga_protocol, + "-p", optbuf, + dbpath, (char*)NULL); + if (ret == -1) { + fprintf(stderr, "Cannot start groonga server: <%s>: errno=%d\n", + groonga_path, errno); + exit(1); + } + } + else { + grntest_server_id = pid; + } + +#endif /* WIN32 */ + + return 0; +} + +static int +parse_line(grn_ctx *ctx, char *buf, int start, int end, int num) +{ + int i, j, error_flag = 0, out_or_test = 0; + char tmpbuf[BUF_LEN]; + + grntest_job[num].concurrency = 1; + grntest_job[num].ntimes = 1; + grntest_job[num].done = 0; + grntest_job[num].qnum = 0; + grntest_job[num].max = 0LL; + grntest_job[num].min = 9223372036854775807LL; + grntest_job[num].outputlog = NULL; + grntest_job[num].inputlog = NULL; + + strncpy(grntest_job[num].jobname, &buf[start], end - start); + grntest_job[num].jobname[end - start] = '\0'; + i = start; + while (i < end) { + if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) { + i++; + continue; + } + if (!strncmp(&buf[i], "do_local", 8)) { + grntest_job[num].jobtype = J_DO_LOCAL; + i = i + 8; + break; + } + if (!strncmp(&buf[i], "do_gqtp", 7)) { + grntest_job[num].jobtype = J_DO_GQTP; + i = i + 7; + break; + } + if (!strncmp(&buf[i], "do_http", 7)) { + grntest_job[num].jobtype = J_DO_HTTP; + i = i + 7; + break; + } + if (!strncmp(&buf[i], "rep_local", 9)) { + grntest_job[num].jobtype = J_REP_LOCAL; + i = i + 9; + break; + } + if (!strncmp(&buf[i], "rep_gqtp", 8)) { + grntest_job[num].jobtype = J_REP_GQTP; + i = i + 8; + break; + } + if (!strncmp(&buf[i], "rep_http", 8)) { + grntest_job[num].jobtype = J_REP_HTTP; + i = i + 8; + break; + } + if (!strncmp(&buf[i], "out_local", 9)) { + grntest_job[num].jobtype = J_OUT_LOCAL; + i = i + 9; + out_or_test = 1; + break; + } + if (!strncmp(&buf[i], "out_gqtp", 8)) { + grntest_job[num].jobtype = J_OUT_GQTP; + i = i + 8; + out_or_test = 1; + break; + } + if (!strncmp(&buf[i], "out_http", 8)) { + grntest_job[num].jobtype = J_OUT_HTTP; + i = i + 8; + out_or_test = 1; + break; + } + if (!strncmp(&buf[i], "test_local", 10)) { + grntest_job[num].jobtype = J_TEST_LOCAL; + i = i + 10; + out_or_test = 1; + break; + } + if (!strncmp(&buf[i], "test_gqtp", 9)) { + grntest_job[num].jobtype = J_TEST_GQTP; + i = i + 9; + out_or_test = 1; + break; + } + if (!strncmp(&buf[i], "test_http", 9)) { + grntest_job[num].jobtype = J_TEST_HTTP; + i = i + 9; + out_or_test = 1; + break; + } + error_flag = 1; + i++; + } + + if (error_flag) { + return 3; + } + if (i == end) { + return 1; + } + + if (grn_isspace(&buf[i], GRN_ENC_UTF8) != 1) { + return 4; + } + i++; + + while (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) { + i++; + continue; + } + j = 0; + while (i < end) { + if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) { + break; + } + grntest_job[num].commandfile[j] = buf[i]; + i++; + j++; + if (j > 255) { + return 5; + } + } + grntest_job[num].commandfile[j] = '\0'; + + while (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) { + i++; + } + + if (i == end) { + if (out_or_test) { + fprintf(stderr, "log(test)_local(gqtp|http) needs log(test)_filename\n"); + return 11; + } + return 0; + } + + j = 0; + while (i < end) { + if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) { + break; + } + tmpbuf[j] = buf[i]; + i++; + j++; + if (j >= BUF_LEN) { + return 6; + } + } + tmpbuf[j] ='\0'; + if (out_or_test) { + if (out_p(grntest_job[num].jobtype)) { + grntest_job[num].outputlog = fopen(tmpbuf, "wb"); + if (grntest_job[num].outputlog == NULL) { + fprintf(stderr, "Cannot open %s\n", tmpbuf); + return 13; + } + } else { + char outlog[BUF_LEN]; + grntest_job[num].inputlog = grn_file_reader_open(ctx, tmpbuf); + if (grntest_job[num].inputlog == NULL) { + fprintf(stderr, "Cannot open %s\n", tmpbuf); + return 14; + } + sprintf(outlog, "%s.diff", tmpbuf); + grntest_job[num].outputlog = fopen(outlog, "wb"); + if (grntest_job[num].outputlog == NULL) { + fprintf(stderr, "Cannot open %s\n", outlog); + return 15; + } + } + grn_strcpy(grntest_job[num].logfile, BUF_LEN, tmpbuf); + return 0; + } else { + grntest_job[num].concurrency = grntest_atoi(tmpbuf, tmpbuf + j, NULL); + if (grntest_job[num].concurrency == 0) { + return 7; + } + } + + while (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) { + i++; + } + + if (i == end) { + return 0; + } + + j = 0; + while (i < end) { + if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) { + break; + } + tmpbuf[j] = buf[i]; + i++; + j++; + if (j > 16) { + return 8; + } + } + tmpbuf[j] ='\0'; + grntest_job[num].ntimes = grntest_atoi(tmpbuf, tmpbuf + j, NULL); + if (grntest_job[num].ntimes == 0) { + return 9; + } + if (i == end) { + return 0; + } + + while (i < end) { + if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) { + i++; + continue; + } + return 10; + } + return 0; +} + +static int +get_jobs(grn_ctx *ctx, char *buf, int line) +{ + int i, len, start, end, ret; + int jnum = 0; + + len = strlen(buf); + i = 0; + while (i < len) { + if ((buf[i] == '#') || (buf[i] == '\r') || (buf[i] == '\n')) { + buf[i] = '\0'; + len = i; + break; + } + i++; + } + + i = 0; + start = 0; + while (i < len) { + if (buf[i] == ';') { + end = i; + ret = parse_line(ctx, buf, start, end, jnum); + if (ret) { + if (ret > 1) { + fprintf(stderr, "Syntax error:line=%d:ret=%d:%s\n", line, ret, buf); + error_exit(ctx, 1); + } + } else { + jnum++; + } + start = end + 1; + } + i++; + } + end = len; + ret = parse_line(ctx, buf, start, end, jnum); + if (ret) { + if (ret > 1) { + fprintf(stderr, "Syntax error:line=%d:ret=%d:%s\n", line, ret, buf); + error_exit(ctx, 1); + } + } else { + jnum++; + } + return jnum; +} + +static int +make_task_table(grn_ctx *ctx, int jobnum) +{ + int i, j; + int tid = 0; + grn_obj *commands = NULL; + + for (i = 0; i < jobnum; i++) { + if ((grntest_job[i].concurrency == 1) && (!grntest_onmemory_mode)) { + grntest_task[tid].file = grntest_job[i].commandfile; + grntest_task[tid].commands = NULL; + grntest_task[tid].ntimes = grntest_job[i].ntimes; + grntest_task[tid].jobtype = grntest_job[i].jobtype; + grntest_task[tid].job_id = i; + tid++; + continue; + } + for (j = 0; j < grntest_job[i].concurrency; j++) { + if (j == 0) { + grn_file_reader *reader; + grn_obj line; + GRN_TEXT_INIT(&line, 0); + commands = grn_obj_open(ctx, GRN_PVECTOR, 0, GRN_VOID); + if (!commands) { + fprintf(stderr, "Cannot alloc commands\n"); + error_exit(ctx, 1); + } + reader = grn_file_reader_open(ctx, grntest_job[i].commandfile); + if (!reader) { + fprintf(stderr, "Cannot alloc commandfile:%s\n", + grntest_job[i].commandfile); + error_exit(ctx, 1); + } + while (grn_file_reader_read_line(ctx, reader, &line) == GRN_SUCCESS) { + grn_obj *command; + if (GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] == '\n') { + grn_bulk_truncate(ctx, &line, GRN_TEXT_LEN(&line) - 1); + } + if (GRN_TEXT_LEN(&line) == 0) { + GRN_BULK_REWIND(&line); + continue; + } + GRN_TEXT_PUTC(ctx, &line, '\0'); + if (comment_p(GRN_TEXT_VALUE(&line))) { + GRN_BULK_REWIND(&line); + continue; + } + command = grn_obj_open(ctx, GRN_BULK, 0, GRN_VOID); + if (!command) { + fprintf(stderr, "Cannot alloc command: %s: %s\n", + grntest_job[i].commandfile, GRN_TEXT_VALUE(&line)); + GRN_OBJ_FIN(ctx, &line); + error_exit(ctx, 1); + } + GRN_TEXT_SET(ctx, command, GRN_TEXT_VALUE(&line), GRN_TEXT_LEN(&line)); + GRN_PTR_PUT(ctx, commands, command); + GRN_BULK_REWIND(&line); + } + grn_file_reader_close(ctx, reader); + GRN_OBJ_FIN(ctx, &line); + } + grntest_task[tid].file = NULL; + grntest_task[tid].commands = commands; + grntest_task[tid].ntimes = grntest_job[i].ntimes; + grntest_task[tid].jobtype = grntest_job[i].jobtype; + grntest_task[tid].job_id = i; + tid++; + } + } + return tid; +} + +/* +static int +print_commandlist(int task_id) +{ + int i; + + for (i = 0; i < GRN_TEXT_LEN(grntest_task[task_id].commands); i++) { + grn_obj *command; + command = GRN_PTR_VALUE_AT(grntest_task[task_id].commands, i); + printf("%s\n", GRN_TEXT_VALUE(command)); + } + return 0; +} +*/ + +/* return num of query */ +static int +do_jobs(grn_ctx *ctx, int jobnum, int line) +{ + int i, task_num, ret, qnum = 0, thread_num = 0; + + for (i = 0; i < jobnum; i++) { +/* +printf("%d:type =%d:file=%s:con=%d:ntimes=%d\n", i, grntest_job[i].jobtype, + grntest_job[i].commandfile, JobTable[i].concurrency, JobTable[i].ntimes); + +*/ + thread_num = thread_num + grntest_job[i].concurrency; + } + + if (thread_num >= MAX_CON) { + fprintf(stderr, "Too many threads requested(MAX=64):line=%d\n", line); + error_exit(ctx, 1); + } + + task_num = make_task_table(ctx, jobnum); + if (task_num != thread_num) { + fprintf(stderr, "Logical error\n"); + error_exit(ctx, 9); + } + + grntest_detail_on = 0; + for (i = 0; i < task_num; i++) { + grn_ctx_init(&grntest_ctx[i], 0); + grntest_owndb[i] = NULL; + if (gqtp_p(grntest_task[i].jobtype)) { + ret = grn_ctx_connect(&grntest_ctx[i], grntest_serverhost, grntest_serverport, 0); + if (ret) { + fprintf(stderr, "Cannot connect groonga server:host=%s:port=%d:ret=%d\n", + grntest_serverhost, grntest_serverport, ret); + error_exit(ctx, 1); + } + } else if (http_p(grntest_task[i].jobtype)) { + grntest_task[i].http_socket = 0; + GRN_TEXT_INIT(&grntest_task[i].http_response, 0); + if (grntest_owndb_mode) { + grntest_owndb[i] = grn_db_open(&grntest_ctx[i], grntest_dbpath); + if (grntest_owndb[i] == NULL) { + fprintf(stderr, "Cannot open db:%s\n", grntest_dbpath); + exit(1); + } + } else { + grntest_owndb[i] = grn_db_create(&grntest_ctx[i], NULL, NULL); + } + } else { + if (grntest_owndb_mode) { + grntest_owndb[i] = grn_db_open(&grntest_ctx[i], grntest_dbpath); + if (grntest_owndb[i] == NULL) { + fprintf(stderr, "Cannot open db:%s\n", grntest_dbpath); + exit(1); + } + } + else { + grn_ctx_use(&grntest_ctx[i], grntest_db); + } + } + if (report_p(grntest_task[i].jobtype)) { + grntest_detail_on++; + } + } + if (grntest_detail_on) { + if (grntest_outtype == OUT_TSV) { + ; + } + else { + fprintf(grntest_log_file, "\"detail\": [\n"); + } + fflush(grntest_log_file); + } + + thread_main(ctx, task_num); + + for (i = 0; i < task_num; i++) { + if (grntest_owndb[i]) { + grn_obj_close(&grntest_ctx[i], grntest_owndb[i]); + } + if (http_p(grntest_task[i].jobtype)) { + GRN_OBJ_FIN(&grntest_ctx[i], &grntest_task[i].http_response); + } + grn_ctx_fin(&grntest_ctx[i]); + qnum = qnum + grntest_task[i].qnum; + } + + i = 0; + while (i < task_num) { + int job_id; + if (grntest_task[i].commands) { + job_id = grntest_task[i].job_id; + GRN_OBJ_FIN(ctx, grntest_task[i].commands); + while (job_id == grntest_task[i].job_id) { + i++; + } + } else { + i++; + } + } + for (i = 0; i < jobnum; i++) { + if (grntest_job[i].outputlog) { + int ret; + ret = fclose(grntest_job[i].outputlog); + if (ret) { + fprintf(stderr, "Cannot close %s\n", grntest_job[i].logfile); + exit(1); + } + } + if (grntest_job[i].inputlog) { + grn_file_reader_close(ctx, grntest_job[i].inputlog); + } + } + return qnum; +} + +/* return num of query */ +static int +do_script(grn_ctx *ctx, const char *script_file_path) +{ + int n_lines = 0; + int n_jobs; + int n_queries, total_n_queries = 0; + grn_file_reader *script_file; + grn_obj line; + + script_file = grn_file_reader_open(ctx, script_file_path); + if (script_file == NULL) { + fprintf(stderr, "Cannot open script file: <%s>\n", script_file_path); + error_exit(ctx, 1); + } + + GRN_TEXT_INIT(&line, 0); + while (grn_file_reader_read_line(ctx, script_file, &line) == GRN_SUCCESS) { + if (grntest_sigint) { + break; + } + n_lines++; + grntest_jobdone = 0; + n_jobs = get_jobs(ctx, GRN_TEXT_VALUE(&line), n_lines); + grntest_jobnum = n_jobs; + + if (n_jobs > 0) { + GRN_TIME_INIT(&grntest_jobs_start, 0); + GRN_TIME_NOW(ctx, &grntest_jobs_start); + if (grntest_outtype == OUT_TSV) { + fprintf(grntest_log_file, "jobs-start\t%s\n", GRN_TEXT_VALUE(&line)); + } else { + fprintf(grntest_log_file, "{\"jobs\": \"%s\",\n", GRN_TEXT_VALUE(&line)); + } + n_queries = do_jobs(ctx, n_jobs, n_lines); + if (grntest_outtype == OUT_TSV) { + fprintf(grntest_log_file, "jobs-end\t%s\n", GRN_TEXT_VALUE(&line)); + } else { + fprintf(grntest_log_file, "},\n"); + } + total_n_queries += n_queries; + + grn_obj_close(ctx, &grntest_jobs_start); + } + if (grntest_stop_flag) { + fprintf(stderr, "Error:Quit\n"); + break; + } + GRN_BULK_REWIND(&line); + } + grn_obj_unlink(ctx, &line); + + grn_file_reader_close(ctx, script_file); + + return total_n_queries; +} + +static int +start_local(grn_ctx *ctx, const char *dbpath) +{ + grntest_db = grn_db_open(ctx, dbpath); + if (!grntest_db) { + grntest_db = grn_db_create(ctx, dbpath, NULL); + } + if (!grntest_db) { + fprintf(stderr, "Cannot open db:%s\n", dbpath); + exit(1); + } + return 0; +} + +static int +check_server(grn_ctx *ctx) +{ + int ret, retry = 0; + while (1) { + ret = grn_ctx_connect(ctx, grntest_serverhost, grntest_serverport, 0); + if (ret == GRN_CONNECTION_REFUSED) { + grn_sleep(1); + retry++; + if (retry > 5) { + fprintf(stderr, "Cannot connect groonga server:host=%s:port=%d:ret=%d\n", + grntest_serverhost, grntest_serverport, ret); + return 1; + } + continue; + } + if (ret) { + fprintf(stderr, "Cannot connect groonga server:host=%s:port=%d:ret=%d\n", + grntest_serverhost, grntest_serverport, ret); + return 1; + } + break; + } + return 0; +} + +#define MODE_LIST 1 +#define MODE_GET 2 +#define MODE_PUT 3 +#define MODE_TIME 4 + +static int +check_response(char *buf) +{ + if (buf[0] == '1') { + return 1; + } + if (buf[0] == '2') { + return 1; + } + if (buf[0] == '3') { + return 1; + } + return 0; +} + +static int +read_response(socket_t socket, char *buf) +{ + int ret; + ret = recv(socket, buf, BUF_LEN - 1, 0); + if (ret == -1) { + fprintf(stderr, "recv error:3\n"); + exit(1); + } + buf[ret] ='\0'; +#ifdef DEBUG_FTP + fprintf(stderr, "recv:%s", buf); +#endif + return ret; +} + +static int +put_file(socket_t socket, const char *filename) +{ + FILE *fp; + int c, ret, size = 0; + char buf[1]; + + fp = fopen(filename, "rb"); + if (!fp) { + fprintf(stderr, "LOCAL:no such file:%s\n", filename); + return 0; + } + + while ((c = fgetc(fp)) != EOF) { + buf[0] = c; + ret = send(socket, buf, 1, 0); + if (ret == -1) { + fprintf(stderr, "send error\n"); + exit(1); + } + size++; + } + fclose(fp); + return size; +} + +static int +ftp_list(socket_t data_socket) +{ + int ret; + char buf[BUF_LEN]; + + while (1) { + ret = recv(data_socket, buf, BUF_LEN - 2, 0); + if (ret == 0) { + fflush(stdout); + return 0; + } + buf[ret] = '\0'; + fprintf(stdout, "%s", buf); + } + + return 0; +} + +static int +get_file(socket_t socket, const char *filename, int size) +{ + FILE *fp; + int ret, total; + char buf[FTPBUF]; + + fp = fopen(filename, "wb"); + if (!fp) { + fprintf(stderr, "Cannot open %s\n", filename); + return -1; + } + + total = 0; + while (total != size) { + ret = recv(socket, buf, FTPBUF, 0); + if (ret == -1) { + fprintf(stderr, "recv error:2:ret=%d:size=%d:total\n", ret, size); + return -1; + } + if (ret == 0) { + break; + } + fwrite(buf, ret, 1, fp); + total = total + ret; + } + + fclose(fp); + return size; +} + +static int +get_port(char *buf, char *host, int *port) +{ + int ret,d1,d2,d3,d4,d5,d6; + ret = sscanf(buf, "227 Entering Passive Mode (%d,%d,%d,%d,%d,%d)", + &d1, &d2, &d3, &d4, &d5, &d6); + if (ret != 6) { + fprintf(stderr, "Cannot enter passsive mode\n"); + return 0; + } + + *port = d5 * 256 + d6; + sprintf(host, "%d.%d.%d.%d", d1, d2, d3, d4); + return 1; +} + +static char * +get_ftp_date(char *buf) +{ + while (*buf !=' ') { + buf++; + if (*buf == '\0') { + return NULL; + } + } + buf++; + + return buf; +} + +static int +get_size(char *buf) +{ + int size; + + while (*buf !='(') { + buf++; + if (*buf == '\0') { + return 0; + } + } + buf++; + size = grntest_atoi(buf, buf + strlen(buf), NULL); + + return size; +} + +int +ftp_sub(const char *user, const char *passwd, const char *host, + const char *filename, int mode, + const char *cd_dirname, char *retval) +{ + int size = 0; + int status = 0; + socket_t command_socket, data_socket; + int data_port; + char data_host[BUF_LEN]; + char send_mesg[BUF_LEN]; + char buf[BUF_LEN]; +#ifdef WIN32 + char base[BUF_LEN]; + char fname[BUF_LEN]; + char ext[BUF_LEN]; +#else + char *base; +#endif /* WIN32 */ + +#ifdef WIN32 + WSADATA ws; + + WSAStartup(MAKEWORD(2,0), &ws); +#endif /* WIN32 */ + + if ((filename != NULL) && (strlen(filename) >= MAX_PATH_LEN)) { + fprintf(stderr, "too long filename\n"); + exit(1); + } + + if ((cd_dirname != NULL) && (strlen(cd_dirname) >= MAX_PATH_LEN)) { + fprintf(stderr, "too long dirname\n"); + exit(1); + } + + command_socket = open_socket(host, 21); + if (command_socket == SOCKETERROR) { + return 0; + } + + read_response(command_socket, buf); + if (!check_response(buf)) { + goto exit; + } + + /* send username */ + sprintf(send_mesg, "USER %s\r\n", user); + write_to_server(command_socket, send_mesg); + read_response(command_socket, buf); + if (!check_response(buf)) { + goto exit; + } + + /* send passwd */ + sprintf(send_mesg, "PASS %s\r\n", passwd); + write_to_server(command_socket, send_mesg); + read_response(command_socket, buf); + if (!check_response(buf)) { + goto exit; + } + + /* send TYPE I */ + sprintf(send_mesg, "TYPE I\r\n"); + write_to_server(command_socket, send_mesg); + read_response(command_socket, buf); + if (!check_response(buf)) { + goto exit; + } + + /* send PASV */ + sprintf(send_mesg, "PASV\r\n"); + write_to_server(command_socket, send_mesg); + read_response(command_socket, buf); + if (!check_response(buf)) { + goto exit; + } + + if (!get_port(buf, data_host, &data_port)) { + goto exit; + } + + data_socket = open_socket(data_host, data_port); + if (data_socket == SOCKETERROR) { + goto exit; + } + + if (cd_dirname) { + sprintf(send_mesg, "CWD %s\r\n", cd_dirname); + write_to_server(command_socket, send_mesg); + } + + read_response(command_socket, buf); + if (!check_response(buf)) { + socketclose(data_socket); + goto exit; + } + +#ifdef WIN32 + _splitpath(filename, NULL, NULL, fname, ext); + grn_strcpy(base, BUF_LEN, fname); + strcat(base, ext); +#else + grn_strcpy(buf, BUF_LEN, filename); + base = basename(buf); +#endif /* WIN32 */ + + switch (mode) { + case MODE_LIST: + if (filename) { + sprintf(send_mesg, "LIST %s\r\n", filename); + } else { + sprintf(send_mesg, "LIST \r\n"); + } + write_to_server(command_socket, send_mesg); + break; + case MODE_PUT: + sprintf(send_mesg, "STOR %s\r\n", base); + write_to_server(command_socket, send_mesg); + break; + case MODE_GET: + sprintf(send_mesg, "RETR %s\r\n", base); + write_to_server(command_socket, send_mesg); + break; + case MODE_TIME: + sprintf(send_mesg, "MDTM %s\r\n", base); + write_to_server(command_socket, send_mesg); + break; + default: + fprintf(stderr, "invalid mode\n"); + socketclose(data_socket); + goto exit; + } + + read_response(command_socket, buf); + if (!check_response(buf)) { + socketclose(data_socket); + goto exit; + } + if (!strncmp(buf, "150", 3)) { + size = get_size(buf); + } + if (!strncmp(buf, "213", 3)) { + retval[BUF_LEN-2] = '\0'; + grn_strcpy(retval, BUF_LEN - 2, get_ftp_date(buf)); + if (retval[BUF_LEN-2] != '\0' ) { + fprintf(stderr, "buffer over run in ftp\n"); + exit(1); + } + } + + switch (mode) { + case MODE_LIST: + ftp_list(data_socket); + break; + case MODE_GET: + if (get_file(data_socket, filename, size) == -1) { + socketclose(data_socket); + goto exit; + } + fprintf(stderr, "get:%s\n", filename); + break; + case MODE_PUT: + if (put_file(data_socket, filename) == -1) { + socketclose(data_socket); + goto exit; + } + fprintf(stderr, "put:%s\n", filename); + break; + default: + break; + } + + socketclose(data_socket); + if ((mode == MODE_GET) || (mode == MODE_PUT)) { + read_response(command_socket, buf); + } + write_to_server(command_socket, "QUIT\n"); + status = 1; +exit: + socketclose(command_socket); + +#ifdef WIN32 + WSACleanup(); +#endif + return status; +} + +/* +static int +ftp_main(int argc, char **argv) +{ + char val[BUF_LEN]; + val[0] = '\0'; + ftp_sub(FTPUSER, FTPPASSWD, FTPSERVER, argv[2], + grntest_atoi(argv[3], argv[3] + strlen(argv[3]), NULL), argv[4], val); + if (val[0] != '\0') { + printf("val=%s\n", val); + } + return 0; +} +*/ + +static int +get_username(char *name, int maxlen) +{ + char *env=NULL; + grn_strcpy(name, maxlen, "nobody"); +#ifdef WIN32 + env = getenv("USERNAME"); +#else + env = getenv("USER"); +#endif /* WIN32 */ + if (strlen(env) > maxlen) { + fprintf(stderr, "too long username:%s\n", env); + exit(1); + } + if (env) { + grn_strcpy(name, maxlen, env); + } + return 0; +} + +static int +get_date(char *date, time_t *sec) +{ +#if defined(WIN32) && !defined(__GNUC__) + struct tm tmbuf; + struct tm *tm = &tmbuf; + localtime_s(tm, sec); +#else /* defined(WIN32) && !defined(__GNUC__) */ +# ifdef HAVE_LOCALTIME_R + struct tm result; + struct tm *tm = &result; + localtime_r(sec, tm); +# else /* HAVE_LOCALTIME_R */ + struct tm *tm = localtime(sec); +# endif /* HAVE_LOCALTIME_R */ +#endif /* defined(WIN32) && !defined(__GNUC__) */ + +#ifdef WIN32 + strftime(date, 128, "%Y-%m-%d %H:%M:%S", tm); +#else + strftime(date, 128, "%F %T", tm); +#endif /* WIN32 */ + + return 1; +} + +static int +get_scriptname(const char *path, char *name, size_t name_len, const char *suffix) +{ + int slen = strlen(suffix); + int len = strlen(path); + + if (len >= BUF_LEN) { + fprintf(stderr, "too long script name\n"); + exit(1); + } + if (slen > len) { + fprintf(stderr, "too long suffux\n"); + exit(1); + } + + grn_strcpy(name, name_len, path); + if (strncmp(&name[len-slen], suffix, slen)) { + name[0] = '\0'; + return 0; + } + name[len-slen] = '\0'; + return 1; +} + +#ifdef WIN32 +static int +get_tm_from_serverdate(char *serverdate, struct tm *tm) +{ + int res; + int year, month, day, hour, minute, second; + + res = sscanf(serverdate, "%4d%2d%2d%2d%2d%2d", + &year, &month, &day, &hour, &minute, &second); + +/* + printf("%d %d %d %d %d %d\n", year, month, day, hour, minute, second); +*/ + + tm->tm_sec = second; + tm->tm_min = minute; + tm->tm_hour = hour; + tm->tm_mday = day; + tm->tm_mon = month - 1; + tm->tm_year = year - 1900; + tm->tm_isdst = -1; + + return 0; +} +#endif /* WIN32 */ + + + +static int +sync_sub(grn_ctx *ctx, const char *filename) +{ + int ret; + char serverdate[BUF_LEN]; +#ifdef WIN32 + struct _stat statbuf; +#else + struct stat statbuf; +#endif /* WIN32 */ + time_t st, lt; + struct tm stm; + + ret = ftp_sub(FTPUSER, FTPPASSWD, FTPSERVER, filename, MODE_TIME, "data", + serverdate); + if (ret == 0) { + fprintf(stderr, "[%s] does not exist in server\n", filename); + return 0; + } +#ifdef WIN32 + get_tm_from_serverdate(serverdate, &stm); +#else + strptime(serverdate, "%Y%m%d %H%M%S", &stm); +#endif /* WIN32 */ + + /* fixme! needs timezone info */ + st = mktime(&stm) + 3600 * 9; + lt = st; + +#ifdef WIN32 + ret = _stat(filename, &statbuf); +#else + ret = stat(filename, &statbuf); +#endif /* WIN32 */ + + if (!ret) { + lt = statbuf.st_mtime; + if (lt < st) { + fprintf(stderr, "newer [%s] exists in server\n", filename); + fflush(stderr); + ret = ftp_sub(FTPUSER, FTPPASSWD, FTPSERVER, filename, MODE_GET, "data", + NULL); + return ret; + } + } else { + fprintf(stderr, "[%s] does not exist in local\n", filename); + fflush(stderr); + ret = ftp_sub(FTPUSER, FTPPASSWD, FTPSERVER, filename, MODE_GET, "data", + NULL); + return ret; + } + return 0; +} + +static int +cache_file(grn_ctx *ctx, char **flist, char *file, int fnum) +{ + int i; + + for (i = 0; i < fnum; i++) { + if (!strcmp(flist[i], file) ) { + return fnum; + } + } + flist[fnum] = GRN_STRDUP(file); + fnum++; + if (fnum >= BUF_LEN) { + fprintf(stderr, "too many uniq commands file!\n"); + exit(1); + } + return fnum; +} + +static int +sync_datafile(grn_ctx *ctx, const char *script_file_path) +{ + int line = 0; + int fnum = 0; + int i, job_num; + FILE *fp; + char buf[BUF_LEN]; + char *filelist[BUF_LEN]; + + fp = fopen(script_file_path, "r"); + if (fp == NULL) { + fprintf(stderr, "Cannot open script file: <%s>\n", script_file_path); + error_exit(ctx, 1); + } + buf[BUF_LEN-2] = '\0'; + while (fgets(buf, BUF_LEN, fp) != NULL) { + line++; + if (buf[BUF_LEN-2] != '\0') { + fprintf(stderr, "Too long line in script file:%d\n", line); + error_exit(ctx, 1); + } + job_num = get_jobs(ctx, buf, line); + + if (job_num > 0) { + for (i = 0; i < job_num; i++) { +/* +printf("commandfile=[%s]:buf=%s\n", grntest_job[i].commandfile, buf); +*/ + fnum = cache_file(ctx, filelist, grntest_job[i].commandfile, fnum); + } + } + } + for (i = 0; i < fnum; i++) { + if (sync_sub(ctx, filelist[i])) { + fprintf(stderr, "updated!:%s\n", filelist[i]); + fflush(stderr); + } + GRN_FREE(filelist[i]); + } + fclose(fp); + return fnum; +} + +static int +sync_script(grn_ctx *ctx, const char *filename) +{ + int ret, filenum; + + ret = sync_sub(ctx, filename); + if (!ret) { + return 0; + } + + fprintf(stderr, "updated!:%s\n", filename); + fflush(stderr); + filenum = sync_datafile(ctx, filename); + return 1; +} + +static void +usage(void) +{ + fprintf(stderr, + "Usage: grntest [options...] [script] [db]\n" + "options:\n" + " --dir: show script files on ftp server\n" + " -i, --host <ip/hostname>: server address to listen (default: %s)\n" + " --localonly: omit server connection\n" + " --log-output-dir: specify output dir (default: current)\n" + " --ftp: connect to ftp server\n" + " --onmemory: load all commands into memory\n" + " --output-type <tsv/json>: specify output-type (default: json)\n" + " --owndb: open dbs for each ctx\n" + " -p, --port <port number>: server port number (default: %d)\n" + " --groonga <groonga_path>: groonga command path (default: %s)\n" + " --protocol <gqtp|http>: groonga server protocol (default: %s)\n" + " --log-path <path>: specify log file path\n" + " --pid-path <path>: specify file path to store PID file\n", + DEFAULT_DEST, DEFAULT_PORT, + groonga_path, groonga_protocol); + exit(1); +} + +enum { + mode_default = 0, + mode_list, + mode_usage, +}; + +#define MODE_MASK 0x007f +#define MODE_FTP 0x0080 +#define MODE_LOCALONLY 0x0100 +#define MODE_OWNDB 0x0800 +#define MODE_ONMEMORY 0x1000 + + +static int +get_token(char *line, char *token, int maxlen, char **next) +{ + int i = 0; + + *next = NULL; + token[i] = '\0'; + + while (*line) { + if (grn_isspace(line, GRN_ENC_UTF8) == 1) { + line++; + continue; + } + if (*line == ';') { + token[0] = ';'; + token[1] = '\0'; + *next = line + 1; + return 1; + } + if (*line == '#') { + token[0] = ';'; + token[1] = '\0'; + *next = line + 1; + return 1; + } + break; + } + + while (*line) { + token[i] = *line; + i++; + if (grn_isspace(line + 1, GRN_ENC_UTF8) == 1) { + token[i] = '\0'; + *next = line + 1; + return 1; + } + if (*(line + 1) == ';') { + token[i] = '\0'; + *next = line + 1; + return 1; + } + if (*(line + 1) == '#') { + token[i] = '\0'; + *next = line + 1; + return 1; + } + if (*(line + 1) == '\0') { + token[i] = '\0'; + return 1; + } + + line++; + } + return 0; +} + +/* SET_PORT and SET_HOST */ +static grn_bool +check_script(grn_ctx *ctx, const char *script_file_path) +{ + grn_file_reader *script_file; + grn_obj line; + char token[BUF_LEN]; + char prev[BUF_LEN]; + char *next = NULL; + + script_file = grn_file_reader_open(ctx, script_file_path); + if (!script_file) { + fprintf(stderr, "Cannot open script file: <%s>\n", script_file_path); + return GRN_FALSE; + } + + GRN_TEXT_INIT(&line, 0); + while (grn_file_reader_read_line(ctx, script_file, &line) == GRN_SUCCESS) { + GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] = '\0'; + get_token(GRN_TEXT_VALUE(&line), token, BUF_LEN, &next); + grn_strcpy(prev, BUF_LEN, token); + + while (next) { + get_token(next, token, BUF_LEN, &next); + if (!strncmp(prev, "SET_PORT", 8)) { + grntest_serverport = grn_atoi(token, token + strlen(token), NULL); + } + if (!strncmp(prev, "SET_HOST", 8)) { + grn_strcpy(grntest_serverhost, BUF_LEN, token); + grntest_remote_mode = 1; + } + grn_strcpy(prev, BUF_LEN, token); + } + } + grn_obj_unlink(ctx, &line); + + grn_file_reader_close(ctx, script_file); + return GRN_TRUE; +} + +#ifndef WIN32 +static void +timeout(int sig) +{ + fprintf(stderr, "timeout:groonga server cannot shutdown!!\n"); + fprintf(stderr, "Use \"kill -9 %d\"\n", grntest_server_id); + alarm(0); +} + +static void +setexit(int sig) +{ + grntest_sigint = 1; +} + +static int +setsigalarm(int sec) +{ + int ret; + struct sigaction sig; + + alarm(sec); + sig.sa_handler = timeout; + sig.sa_flags = 0; + sigemptyset(&sig.sa_mask); + ret = sigaction(SIGALRM, &sig, NULL); + if (ret == -1) { + fprintf(stderr, "setsigalarm:errno= %d\n", errno); + } + return ret; +} + +static int +setsigint(void) +{ + int ret; + struct sigaction sig; + + sig.sa_handler = setexit; + sig.sa_flags = 0; + sigemptyset(&sig.sa_mask); + ret = sigaction(SIGINT, &sig, NULL); + if (ret == -1) { + fprintf(stderr, "setsigint:errno= %d\n", errno); + } + return ret; +} +#endif /* WIN32 */ + +int +main(int argc, char **argv) +{ + int qnum, i, mode = 0; + int exit_code = EXIT_SUCCESS; + grn_ctx context; + char sysinfo[BUF_LEN]; + char log_path_buffer[BUF_LEN]; + const char *log_path = NULL; + const char *pid_path = NULL; + const char *portstr = NULL, *hoststr = NULL, *dbname = NULL, *scrname = NULL, *outdir = NULL, *outtype = NULL; + time_t sec; + + static grn_str_getopt_opt opts[] = { + {'i', "host", NULL, 0, GETOPT_OP_NONE}, + {'p', "port", NULL, 0, GETOPT_OP_NONE}, + {'\0', "log-output-dir", NULL, 0, GETOPT_OP_NONE}, + {'\0', "output-type", NULL, 0, GETOPT_OP_NONE}, + {'\0', "dir", NULL, mode_list, GETOPT_OP_UPDATE}, + {'\0', "ftp", NULL, MODE_FTP, GETOPT_OP_ON}, + {'h', "help", NULL, mode_usage, GETOPT_OP_UPDATE}, + {'\0', "localonly", NULL, MODE_LOCALONLY, GETOPT_OP_ON}, + {'\0', "onmemory", NULL, MODE_ONMEMORY, GETOPT_OP_ON}, + {'\0', "owndb", NULL, MODE_OWNDB, GETOPT_OP_ON}, + {'\0', "groonga", NULL, 0, GETOPT_OP_NONE}, + {'\0', "protocol", NULL, 0, GETOPT_OP_NONE}, + {'\0', "log-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "pid-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', NULL, NULL, 0, 0} + }; + + opts[0].arg = &hoststr; + opts[1].arg = &portstr; + opts[2].arg = &outdir; + opts[3].arg = &outtype; + opts[10].arg = &groonga_path; + opts[11].arg = &groonga_protocol; + opts[12].arg = &log_path; + opts[13].arg = &pid_path; + + i = grn_str_getopt(argc, argv, opts, &mode); + if (i < 0) { + usage(); + } + + switch (mode & MODE_MASK) { + case mode_list : + ftp_sub(FTPUSER, FTPPASSWD, FTPSERVER, "*.scr", 1, "data", + NULL); + return 0; + break; + case mode_usage : + usage(); + break; + default : + break; + } + + if (pid_path) { + FILE *pid_file; + pid_file = fopen(pid_path, "w"); + if (pid_file) { + fprintf(pid_file, "%d", grn_getpid()); + fclose(pid_file); + } else { + fprintf(stderr, + "failed to open PID file: <%s>: %s\n", + pid_path, strerror(errno)); + } + } + + if (i < argc) { + scrname = argv[i]; + } + if (i < argc - 1) { + dbname = argv[i+1]; + } + grntest_dbpath = dbname; + + if (mode & MODE_LOCALONLY) { + grntest_localonly_mode = 1; + grntest_remote_mode = 1; + } + + if (mode & MODE_OWNDB) { + grntest_localonly_mode = 1; + grntest_remote_mode = 1; + grntest_owndb_mode = 1; + } + + if (mode & MODE_ONMEMORY) { + grntest_onmemory_mode= 1; + } + + if (mode & MODE_FTP) { + grntest_ftp_mode = GRN_TRUE; + } + + if ((scrname == NULL) || (dbname == NULL)) { + usage(); + } + + grn_strcpy(grntest_serverhost, BUF_LEN, DEFAULT_DEST); + if (hoststr) { + grntest_remote_mode = 1; + grn_strcpy(grntest_serverhost, BUF_LEN, hoststr); + } + grntest_serverport = DEFAULT_PORT; + if (portstr) { + grntest_serverport = grn_atoi(portstr, portstr + strlen(portstr), NULL); + } + + if (outtype && !strcmp(outtype, "tsv")) { + grntest_outtype = OUT_TSV; + } + + grn_default_logger_set_path(GRN_LOG_PATH); + + grn_init(); + CRITICAL_SECTION_INIT(grntest_cs); + + grn_ctx_init(&context, 0); + grn_ctx_init(&grntest_server_context, 0); + grn_db_create(&grntest_server_context, NULL, NULL); + grn_set_default_encoding(GRN_ENC_UTF8); + + if (grntest_ftp_mode) { + sync_script(&context, scrname); + } + if (!check_script(&context, scrname)) { + exit_code = EXIT_FAILURE; + goto exit; + } + + start_local(&context, dbname); + if (!grntest_remote_mode) { + start_server(dbname, 0); + } + + if (!grntest_localonly_mode) { + if (check_server(&grntest_server_context)) { + goto exit; + } + } + + get_scriptname(scrname, grntest_scriptname, BUF_LEN, ".scr"); + get_username(grntest_username, 256); + + GRN_TIME_INIT(&grntest_starttime, 0); + GRN_TIME_NOW(&context, &grntest_starttime); + sec = (time_t)(GRN_TIME_VALUE(&grntest_starttime)/1000000); + get_date(grntest_date, &sec); + + if (!log_path) { + if (outdir) { + sprintf(log_path_buffer, + "%s/%s-%s-%" GRN_FMT_LLD "-%s.log", outdir, grntest_scriptname, + grntest_username, + GRN_TIME_VALUE(&grntest_starttime), grn_get_version()); + } else { + sprintf(log_path_buffer, + "%s-%s-%" GRN_FMT_LLD "-%s.log", grntest_scriptname, + grntest_username, + GRN_TIME_VALUE(&grntest_starttime), grn_get_version()); + } + log_path = log_path_buffer; + } + + grntest_log_file = fopen(log_path, "w+b"); + if (!grntest_log_file) { + fprintf(stderr, "Cannot open log file: <%s>\n", log_path); + goto exit; + } + + get_sysinfo(dbname, sysinfo, BUF_LEN); + output_sysinfo(sysinfo); + +#ifndef WIN32 + setsigint(); +#endif /* WIN32 */ + qnum = do_script(&context, scrname); + output_result_final(&context, qnum); + fclose(grntest_log_file); + + if (grntest_ftp_mode) { + ftp_sub(FTPUSER, FTPPASSWD, FTPSERVER, log_path, 3, + "report", NULL); + } + fprintf(stderr, "grntest done. logfile=%s\n", log_path); + +exit: + if (pid_path) { + remove(pid_path); + } + + shutdown_server(); +#ifdef WIN32 + if (!grntest_remote_mode) { + int ret; + ret = WaitForSingleObject(grntest_pi.hProcess, 20000); + if (ret == WAIT_TIMEOUT) { + fprintf(stderr, "timeout:groonga server cannot shutdown!!\n"); + fprintf(stderr, "Cannot wait\n"); + exit(1); + } + } +#else + if (grntest_server_id) { + int ret, pstatus; + setsigalarm(20); + ret = waitpid(grntest_server_id, &pstatus, 0); + if (ret < 0) { + fprintf(stderr, "Cannot wait\n"); + exit(1); + } +/* + else { + fprintf(stderr, "pstatus = %d\n", pstatus); + } +*/ + alarm(0); + } +#endif /* WIN32 */ + CRITICAL_SECTION_FIN(grntest_cs); + grn_obj_close(&context, &grntest_starttime); + grn_obj_close(&context, grntest_db); + grn_ctx_fin(&context); + grn_obj_close(&grntest_server_context, grn_ctx_db(&grntest_server_context)); + grn_ctx_fin(&grntest_server_context); + grn_fin(); + return exit_code; +} diff --git a/storage/mroonga/vendor/groonga/src/groonga_benchmark_sources.am b/storage/mroonga/vendor/groonga/src/groonga_benchmark_sources.am new file mode 100644 index 00000000..bf05fbff --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/groonga_benchmark_sources.am @@ -0,0 +1,2 @@ +groonga_benchmark_SOURCES = \ + groonga_benchmark.c diff --git a/storage/mroonga/vendor/groonga/src/groonga_mruby.c b/storage/mroonga/vendor/groonga/src/groonga_mruby.c new file mode 100644 index 00000000..3aa73aaf --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/groonga_mruby.c @@ -0,0 +1,86 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2014 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#include <grn_mrb.h> + +#include <stdio.h> +#include <stdlib.h> + +static int +run(grn_ctx *ctx, const char *db_path, const char *ruby_script_path) +{ + grn_obj *db; + + db = grn_db_open(ctx, db_path); + if (!db) { + if (ctx->rc == GRN_NO_SUCH_FILE_OR_DIRECTORY) { + db = grn_db_create(ctx, db_path, NULL); + if (!db) { + fprintf(stderr, "Failed to create database: <%s>: %s", + db_path, ctx->errbuf); + return EXIT_FAILURE; + } + } else { + fprintf(stderr, "Failed to open database: <%s>: %s", + db_path, ctx->errbuf); + return EXIT_FAILURE; + } + } + + grn_mrb_load(ctx, ruby_script_path); + if (ctx->rc != GRN_SUCCESS) { + fprintf(stderr, "Failed to load Ruby script: <%s>: %s", + ruby_script_path, ctx->errbuf); + } + + grn_obj_close(ctx, db); + + if (ctx->rc == GRN_SUCCESS) { + return EXIT_SUCCESS; + } else { + return EXIT_FAILURE; + } +} + +int +main(int argc, char **argv) +{ + int exit_code = EXIT_SUCCESS; + + if (argc != 3) { + fprintf(stderr, "Usage: %s DB_PATH RUBY_SCRIPT_PATH\n", argv[0]); + return EXIT_FAILURE; + } + + grn_default_logger_set_path(GRN_LOG_PATH); + + if (grn_init() != GRN_SUCCESS) { + return EXIT_FAILURE; + } + + { + grn_ctx ctx; + grn_ctx_init(&ctx, 0); + exit_code = run(&ctx, argv[1], argv[2]); + grn_ctx_fin(&ctx); + } + + grn_fin(); + + return exit_code; +} diff --git a/storage/mroonga/vendor/groonga/src/groonga_mruby_sources.am b/storage/mroonga/vendor/groonga/src/groonga_mruby_sources.am new file mode 100644 index 00000000..c9006755 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/groonga_mruby_sources.am @@ -0,0 +1,2 @@ +groonga_mruby_SOURCES = \ + groonga_mruby.c diff --git a/storage/mroonga/vendor/groonga/src/groonga_sources.am b/storage/mroonga/vendor/groonga/src/groonga_sources.am new file mode 100644 index 00000000..308fdc3d --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/groonga_sources.am @@ -0,0 +1,2 @@ +groonga_SOURCES = \ + groonga.c diff --git a/storage/mroonga/vendor/groonga/src/httpd/Makefile.am b/storage/mroonga/vendor/groonga/src/httpd/Makefile.am new file mode 100644 index 00000000..736dd1cf --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/httpd/Makefile.am @@ -0,0 +1,32 @@ +NGINX_DIR = $(top_builddir)/vendor/nginx-$(NGINX_VERSION) + +EXTRA_DIST = \ + nginx-module \ + configure + +if WITH_GROONGA_HTTPD +NGINX_MAKEILE = $(NGINX_DIR)/Makefile +NGINX_MAKEFILE_DEPEND_FILES = \ + configure \ + Makefile.am \ + $(top_builddir)/config.status \ + $(srcdir)/nginx-module/config + +$(NGINX_MAKEILE): $(NGINX_MAKEFILE_DEPEND_FILES) + $(srcdir)/configure --srcdir="$(srcdir)" `../../config.status --config` + +# nginx's Makefile specify 'build' as the default rule. +# This isn't compatible with the 'all' default rule generated by Automake +# So, override the all rule. +all-nginx: $(NGINX_MAKEILE) + (cd $(NGINX_DIR) && $(MAKE) build) +all-local: all-nginx + +clean-nginx: $(NGINX_MAKEILE) + (cd $(NGINX_DIR) && $(MAKE) clean) +clean-local: clean-nginx + +install-exec-nginx: $(NGINX_MAKEILE) + (cd $(NGINX_DIR) && $(MAKE) install) +install-exec-local: install-exec-nginx +endif diff --git a/storage/mroonga/vendor/groonga/src/httpd/nginx-module/config b/storage/mroonga/vendor/groonga/src/httpd/nginx-module/config new file mode 100644 index 00000000..b79eef6b --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/httpd/nginx-module/config @@ -0,0 +1,56 @@ +# -*- sh -*- + +groonga_strip_switch() +{ + # skip "-I" from "-I/usr/..." + tail -c +3 +} + +if [ "$GROONGA_HTTPD_IN_TREE" = yes ]; then + groonga_cflags="-I${GROONGA_HTTPD_IN_TREE_INCLUDE_PATH}" + groonga_cflags="${groonga_cflags} -DNGX_HTTP_GROONGA_LOG_PATH=\\\"\"${GROONGA_HTTPD_GROONGA_LOG_PATH}\"\\\"" + groonga_cflags="${groonga_cflags} -DNGX_HTTP_GROONGA_QUERY_LOG_PATH=\\\"\"${GROONGA_HTTPD_GROONGA_QUERY_LOG_PATH}\"\\\"" + groonga_libs="-L${GROONGA_HTTPD_IN_TREE_LINK_PATH}" + if [ "${GROONGA_HTTPD_WITH_ONIGMO}" = "yes" ]; then + groonga_libs="$groonga_libs -L${GROONGA_HTTPD_ONIGMO_IN_TREE_LINK_PATH}" + fi + groonga_libs="$groonga_libs -lgroonga" + if [ -n "${GROONGA_HTTPD_RPATH}" ]; then + groonga_libs="$groonga_libs -Wl,-rpath -Wl,${GROONGA_HTTPD_RPATH}" + fi + + ngx_addon_name=ngx_http_groonga_module + HTTP_MODULES="$HTTP_MODULES ngx_http_groonga_module" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_groonga_module.c" + CFLAGS="$CFLAGS $groonga_cflags" + CORE_LIBS="$CORE_LIBS $groonga_libs" + + return 0 +fi + +groonga_cflags="$(pkg-config --cflags groonga)" +groonga_feature_path="$(pkg-config --cflags-only-I groonga | + groonga_strip_switch)" +groonga_libs="$(pkg-config --libs groonga)" + +ngx_feature="groonga" +ngx_feature_name= +ngx_feature_run=no +ngx_feature_incs="#include <groonga.h>" +ngx_feature_path="$groonga_feature_path" +ngx_feature_libs="$groonga_libs" +ngx_feature_test="grn_get_version()" +. auto/feature + +if [ $ngx_found = yes ]; then + ngx_addon_name=ngx_http_groonga_module + HTTP_MODULES="$HTTP_MODULES ngx_http_groonga_module" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_groonga_module.c" + CFLAGS="$CFLAGS $groonga_cflags" + CORE_LIBS="$CORE_LIBS $groonga_libs" +else + cat << END +$0: error: the groonga module requires the Groonga library. +END + exit 1 +fi diff --git a/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c b/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c new file mode 100644 index 00000000..07853906 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c @@ -0,0 +1,1678 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2012-2017 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#ifndef WIN32 +# define NGX_GRN_SUPPORT_STOP_BY_COMMAND +#endif + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_http.h> + +#include <groonga.h> +#include <groonga/plugin.h> + +#include <sys/stat.h> + +#ifdef NGX_GRN_SUPPORT_STOP_BY_COMMAND +# include <sys/types.h> +# include <unistd.h> +#endif + +#define GRN_NO_FLAGS 0 + +typedef struct { + ngx_flag_t enabled; + ngx_str_t database_path; + char *database_path_cstr; + ngx_flag_t database_auto_create; + ngx_str_t base_path; + ngx_str_t log_path; + ngx_open_file_t *log_file; + grn_log_level log_level; + ngx_str_t query_log_path; + ngx_open_file_t *query_log_file; + size_t cache_limit; + ngx_msec_t default_request_timeout_msec; + char *config_file; + int config_line; + char *name; + grn_obj *database; + grn_cache *cache; + ngx_str_t cache_base_path; +} ngx_http_groonga_loc_conf_t; + +typedef struct { + ngx_log_t *log; + ngx_pool_t *pool; + ngx_int_t rc; +} ngx_http_groonga_database_callback_data_t; + +typedef struct { + grn_bool initialized; + grn_rc rc; + struct { + grn_bool processed; + grn_bool header_sent; + ngx_http_request_t *r; + ngx_int_t rc; + ngx_chain_t *free_chain; + ngx_chain_t *busy_chain; + } raw; + struct { + grn_obj head; + grn_obj body; + grn_obj foot; + } typed; +} ngx_http_groonga_handler_data_t; + +typedef void (*ngx_http_groonga_loc_conf_callback_pt)(ngx_http_groonga_loc_conf_t *conf, void *user_data); + +ngx_module_t ngx_http_groonga_module; + +static grn_ctx ngx_http_groonga_context; +static grn_ctx *context = &ngx_http_groonga_context; +static ngx_http_groonga_loc_conf_t *ngx_http_groonga_current_location_conf = NULL; + +static char * +ngx_str_null_terminate(ngx_pool_t *pool, const ngx_str_t *string) +{ + char *null_terminated_c_string; + + null_terminated_c_string = ngx_pnalloc(pool, string->len + 1); + if (!null_terminated_c_string) { + return NULL; + } + + memcpy(null_terminated_c_string, string->data, string->len); + null_terminated_c_string[string->len] = '\0'; + + return null_terminated_c_string; +} + +static grn_bool +ngx_str_equal_c_string(ngx_str_t *string, const char *c_string) +{ + if (string->len != strlen(c_string)) { + return GRN_FALSE; + } + + return memcmp(c_string, string->data, string->len) == 0; +} + +static grn_bool +ngx_str_is_custom_path(ngx_str_t *string) +{ + if (string->len == 0) { + return GRN_FALSE; + } + + if (strncmp((const char *)(string->data), "off", string->len) == 0) { + return GRN_FALSE; + } + + return GRN_TRUE; +} + +static uint32_t +ngx_http_groonga_get_thread_limit(void *data) +{ + return 1; +} + +static ngx_int_t +ngx_http_groonga_grn_rc_to_http_status(grn_rc rc) +{ + switch (rc) { + case GRN_SUCCESS : + return NGX_HTTP_OK; + case GRN_INVALID_ARGUMENT : + case GRN_FUNCTION_NOT_IMPLEMENTED : + case GRN_SYNTAX_ERROR : + return NGX_HTTP_BAD_REQUEST; + case GRN_CANCEL : + return NGX_HTTP_REQUEST_TIME_OUT; + default : + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } +} + +static void +ngx_http_groonga_write_fd(ngx_fd_t fd, + u_char *buffer, size_t buffer_size, + const char *message, size_t message_size) +{ + size_t rest_message_size = message_size; + const char *current_message = message; + + while (rest_message_size > 0) { + size_t current_message_size; + + if (rest_message_size > NGX_MAX_ERROR_STR) { + current_message_size = NGX_MAX_ERROR_STR; + } else { + current_message_size = rest_message_size; + } + + grn_memcpy(buffer, current_message, current_message_size); + ngx_write_fd(fd, buffer, current_message_size); + rest_message_size -= current_message_size; + current_message += current_message_size; + } +} + +static void +ngx_http_groonga_logger_log(grn_ctx *ctx, grn_log_level level, + const char *timestamp, const char *title, + const char *message, const char *location, + void *user_data) +{ + ngx_open_file_t *file = user_data; + char level_marks[] = " EACewnid-"; + u_char buffer[NGX_MAX_ERROR_STR]; + + if (!file) { + return; + } + + ngx_http_groonga_write_fd(file->fd, + buffer, NGX_MAX_ERROR_STR, + timestamp, strlen(timestamp)); + ngx_write_fd(file->fd, "|", 1); + ngx_write_fd(file->fd, level_marks + level, 1); + ngx_write_fd(file->fd, "|", 1); + if (location && *location) { + ngx_http_groonga_write_fd(file->fd, + buffer, NGX_MAX_ERROR_STR, + location, strlen(location)); + ngx_write_fd(file->fd, ": ", 2); + if (title && *title) { + ngx_http_groonga_write_fd(file->fd, + buffer, NGX_MAX_ERROR_STR, + title, strlen(title)); + ngx_write_fd(file->fd, " ", 1); + } + } else { + ngx_http_groonga_write_fd(file->fd, + buffer, NGX_MAX_ERROR_STR, + title, strlen(title)); + ngx_write_fd(file->fd, " ", 1); + } + ngx_http_groonga_write_fd(file->fd, + buffer, NGX_MAX_ERROR_STR, + message, strlen(message)); + ngx_write_fd(file->fd, "\n", 1); +} + +static void +ngx_http_groonga_logger_reopen(grn_ctx *ctx, void *user_data) +{ + GRN_LOG(ctx, GRN_LOG_NOTICE, "log will be closed."); + ngx_reopen_files((ngx_cycle_t *)ngx_cycle, -1); + GRN_LOG(ctx, GRN_LOG_NOTICE, "log opened."); +} + +static void +ngx_http_groonga_logger_fin(grn_ctx *ctx, void *user_data) +{ +} + +static grn_logger ngx_http_groonga_logger = { + GRN_LOG_DEFAULT_LEVEL, + GRN_LOG_TIME | GRN_LOG_MESSAGE | GRN_LOG_PID, + NULL, + ngx_http_groonga_logger_log, + ngx_http_groonga_logger_reopen, + ngx_http_groonga_logger_fin +}; + +static ngx_int_t +ngx_http_groonga_context_init_logger(ngx_http_groonga_loc_conf_t *location_conf, + ngx_pool_t *pool, + ngx_log_t *log) +{ + if (ngx_http_groonga_current_location_conf) { + ngx_http_groonga_current_location_conf->log_level = + grn_logger_get_max_level(context); + } + + ngx_http_groonga_logger.max_level = location_conf->log_level; + ngx_http_groonga_logger.user_data = location_conf->log_file; + grn_logger_set(context, &ngx_http_groonga_logger); + + return NGX_OK; +} + +static void +ngx_http_groonga_query_logger_log(grn_ctx *ctx, unsigned int flag, + const char *timestamp, const char *info, + const char *message, void *user_data) +{ + ngx_open_file_t *file = user_data; + u_char buffer[NGX_MAX_ERROR_STR]; + u_char *last; + + if (!file) { + return; + } + + last = ngx_slprintf(buffer, buffer + NGX_MAX_ERROR_STR, + "%s|%s%s\n", + timestamp, info, message); + ngx_write_fd(file->fd, buffer, last - buffer); +} + +static void +ngx_http_groonga_query_logger_reopen(grn_ctx *ctx, void *user_data) +{ + ngx_reopen_files((ngx_cycle_t *)ngx_cycle, -1); +} + +static void +ngx_http_groonga_query_logger_fin(grn_ctx *ctx, void *user_data) +{ +} + +static grn_query_logger ngx_http_groonga_query_logger = { + GRN_QUERY_LOG_DEFAULT, + NULL, + ngx_http_groonga_query_logger_log, + ngx_http_groonga_query_logger_reopen, + ngx_http_groonga_query_logger_fin +}; + +static ngx_int_t +ngx_http_groonga_context_init_query_logger(ngx_http_groonga_loc_conf_t *location_conf, + ngx_pool_t *pool, + ngx_log_t *log) +{ + ngx_http_groonga_query_logger.user_data = location_conf->query_log_file; + grn_query_logger_set(context, &ngx_http_groonga_query_logger); + + return NGX_OK; +} + +static ngx_int_t +ngx_http_groonga_context_init(ngx_http_groonga_loc_conf_t *location_conf, + ngx_pool_t *pool, + ngx_log_t *log) +{ + ngx_int_t status; + + if (location_conf == ngx_http_groonga_current_location_conf) { + return NGX_OK; + } + + status = ngx_http_groonga_context_init_logger(location_conf, + pool, + log); + if (status == NGX_ERROR) { + return status; + } + + status = ngx_http_groonga_context_init_query_logger(location_conf, + pool, + log); + if (status == NGX_ERROR) { + return status; + } + + grn_ctx_use(context, location_conf->database); + grn_cache_current_set(context, location_conf->cache); + + /* TODO: It doesn't work yet. We need to implement request timeout + * handler. */ + if (location_conf->default_request_timeout_msec == NGX_CONF_UNSET_MSEC) { + grn_set_default_request_timeout(0.0); + } else { + double timeout; + timeout = location_conf->default_request_timeout_msec / 1000.0; + grn_set_default_request_timeout(timeout); + } + + ngx_http_groonga_current_location_conf = location_conf; + + return status; +} + +static void +ngx_http_groonga_context_log_error(ngx_log_t *log) +{ + if (context->rc == GRN_SUCCESS) { + return; + } + + ngx_log_error(NGX_LOG_ERR, log, 0, "%s", context->errbuf); +} + +static ngx_int_t +ngx_http_groonga_context_check_error(ngx_log_t *log) +{ + if (context->rc == GRN_SUCCESS) { + return NGX_OK; + } else { + ngx_http_groonga_context_log_error(log); + return NGX_HTTP_BAD_REQUEST; + } +} + +static ngx_buf_t * +ngx_http_groonga_grn_obj_to_ngx_buf(ngx_pool_t *pool, grn_obj *object) +{ + ngx_buf_t *buffer; + buffer = ngx_pcalloc(pool, sizeof(ngx_buf_t)); + if (buffer == NULL) { + return NULL; + } + + /* adjust the pointers of the buffer */ + buffer->pos = (u_char *)GRN_TEXT_VALUE(object); + buffer->last = (u_char *)GRN_TEXT_VALUE(object) + GRN_TEXT_LEN(object); + buffer->memory = 1; /* this buffer is in memory */ + buffer->in_file = 0; + + return buffer; +} + +static void +ngx_http_groonga_handler_cleanup(void *user_data) +{ + ngx_http_groonga_handler_data_t *data = user_data; + + if (!data->initialized) { + return; + } + + GRN_OBJ_FIN(context, &(data->typed.head)); + GRN_OBJ_FIN(context, &(data->typed.body)); + GRN_OBJ_FIN(context, &(data->typed.foot)); +} + +static void +ngx_http_groonga_handler_set_content_type(ngx_http_request_t *r, + const char *content_type) +{ + r->headers_out.content_type.len = strlen(content_type); + r->headers_out.content_type.data = (u_char *)content_type; + r->headers_out.content_type_len = r->headers_out.content_type.len; +} + +static void +ngx_http_groonga_context_receive_handler_raw(grn_ctx *context, + int flags, + ngx_http_groonga_handler_data_t *data) +{ + char *chunk = NULL; + unsigned int chunk_size = 0; + int recv_flags; + ngx_http_request_t *r; + ngx_log_t *log; + grn_bool is_last_chunk; + + grn_ctx_recv(context, &chunk, &chunk_size, &recv_flags); + data->raw.processed = GRN_TRUE; + + if (data->raw.rc != NGX_OK) { + return; + } + + r = data->raw.r; + log = r->connection->log; + is_last_chunk = (flags & GRN_CTX_TAIL); + + if (!data->raw.header_sent) { + ngx_http_groonga_handler_set_content_type(r, grn_ctx_get_mime_type(context)); + r->headers_out.status = NGX_HTTP_OK; + if (is_last_chunk) { + r->headers_out.content_length_n = chunk_size; + if (chunk_size == 0) { + r->header_only = 1; + } + } else { + r->headers_out.content_length_n = -1; + } + data->raw.rc = ngx_http_send_header(r); + data->raw.header_sent = GRN_TRUE; + + if (data->raw.rc != NGX_OK) { + return; + } + } + + if (chunk_size > 0 || is_last_chunk) { + ngx_chain_t *chain; + + chain = ngx_chain_get_free_buf(r->pool, &(data->raw.free_chain)); + if (!chain) { + ngx_log_error(NGX_LOG_ERR, log, 0, + "http_groonga: failed to allocate memory for chunked body"); + data->raw.rc = NGX_ERROR; + return; + } + if (chunk_size == 0) { + chain->buf->pos = NULL; + chain->buf->last = NULL; + chain->buf->memory = 0; + } else { + chain->buf->pos = (u_char *)chunk; + chain->buf->last = (u_char *)chunk + chunk_size; + chain->buf->memory = 1; + } + chain->buf->tag = (ngx_buf_tag_t)&ngx_http_groonga_module; + chain->buf->flush = 1; + chain->buf->temporary = 0; + chain->buf->in_file = 0; + if (is_last_chunk) { + chain->buf->last_buf = 1; + } else { + chain->buf->last_buf = 0; + } + chain->next = NULL; + + data->raw.rc = ngx_http_output_filter(r, chain); + ngx_chain_update_chains(r->pool, + &(data->raw.free_chain), + &(data->raw.busy_chain), + &chain, + (ngx_buf_tag_t)&ngx_http_groonga_module); + } +} + +static void +ngx_http_groonga_context_receive_handler_typed(grn_ctx *context, + int flags, + ngx_http_groonga_handler_data_t *data) +{ + char *result = NULL; + unsigned int result_size = 0; + int recv_flags; + + if (!(flags & GRN_CTX_TAIL)) { + return; + } + + grn_ctx_recv(context, &result, &result_size, &recv_flags); + +#ifdef NGX_GRN_SUPPORT_STOP_BY_COMMAND + if (recv_flags == GRN_CTX_QUIT) { + ngx_int_t ngx_rc; + ngx_int_t ngx_pid; + + if (ngx_process == NGX_PROCESS_SINGLE) { + ngx_pid = getpid(); + } else { + ngx_pid = getppid(); + } + + ngx_rc = ngx_os_signal_process((ngx_cycle_t *)ngx_cycle, + "quit", + ngx_pid); + if (ngx_rc == NGX_OK) { + context->stat &= ~GRN_CTX_QUIT; + grn_ctx_recv(context, &result, &result_size, &recv_flags); + context->stat |= GRN_CTX_QUIT; + } else { + context->rc = GRN_OPERATION_NOT_PERMITTED; + result = "false"; + result_size = strlen(result); + context->stat &= ~GRN_CTX_QUIT; + } + } +#endif + + if (result_size > 0 || + GRN_TEXT_LEN(&(data->typed.body)) > 0 || + context->rc != GRN_SUCCESS) { + if (result_size > 0) { + GRN_TEXT_PUT(context, &(data->typed.body), result, result_size); + } + + grn_output_envelope(context, + context->rc, + &(data->typed.head), + &(data->typed.body), + &(data->typed.foot), + NULL, + 0); + } +} + +static void +ngx_http_groonga_context_receive_handler(grn_ctx *context, + int flags, + void *callback_data) +{ + ngx_http_groonga_handler_data_t *data = callback_data; + + switch (grn_ctx_get_output_type(context)) { + case GRN_CONTENT_GROONGA_COMMAND_LIST : + case GRN_CONTENT_NONE : + ngx_http_groonga_context_receive_handler_raw(context, flags, data); + break; + default : + ngx_http_groonga_context_receive_handler_typed(context, flags, data); + break; + } +} + +static ngx_int_t +ngx_http_groonga_extract_command_path(ngx_http_request_t *r, + ngx_str_t *command_path) +{ + size_t base_path_length; + + ngx_http_core_loc_conf_t *http_location_conf; + ngx_http_groonga_loc_conf_t *location_conf; + + http_location_conf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + location_conf = ngx_http_get_module_loc_conf(r, ngx_http_groonga_module); + + command_path->data = r->unparsed_uri.data; + command_path->len = r->unparsed_uri.len; + base_path_length = http_location_conf->name.len; + if (location_conf->base_path.len > 0) { + if (command_path->len < location_conf->base_path.len) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "requested URI is shorter than groonga_base_path: " + "URI: <%V>, groonga_base_path: <%V>", + &(r->unparsed_uri), &(location_conf->base_path)); + } else if (strncmp((const char *)command_path->data, + (const char *)(location_conf->base_path.data), + location_conf->base_path.len) < 0) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "groonga_base_path doesn't match requested URI: " + "URI: <%V>, groonga_base_path: <%V>", + &(r->unparsed_uri), &(location_conf->base_path)); + } else { + base_path_length = location_conf->base_path.len; + } + } + command_path->data += base_path_length; + command_path->len -= base_path_length; + if (command_path->len > 0 && command_path->data[0] == '/') { + command_path->data += 1; + command_path->len -= 1; + } + if (command_path->len == 0) { + return NGX_HTTP_BAD_REQUEST; + } + + return NGX_OK; +} + +static ngx_int_t +ngx_http_groonga_handler_create_data(ngx_http_request_t *r, + ngx_http_groonga_handler_data_t **data_return) +{ + ngx_int_t rc; + + ngx_http_groonga_loc_conf_t *location_conf; + + ngx_http_cleanup_t *cleanup; + ngx_http_groonga_handler_data_t *data; + + location_conf = ngx_http_get_module_loc_conf(r, ngx_http_groonga_module); + + rc = ngx_http_groonga_context_init(location_conf, r->pool, r->connection->log); + if (rc != NGX_OK) { + return rc; + } + + cleanup = ngx_http_cleanup_add(r, sizeof(ngx_http_groonga_handler_data_t)); + cleanup->handler = ngx_http_groonga_handler_cleanup; + data = cleanup->data; + *data_return = data; + + data->initialized = GRN_TRUE; + data->rc = GRN_SUCCESS; + + data->raw.processed = GRN_FALSE; + data->raw.header_sent = GRN_FALSE; + data->raw.r = r; + data->raw.rc = NGX_OK; + data->raw.free_chain = NULL; + data->raw.busy_chain = NULL; + + GRN_TEXT_INIT(&(data->typed.head), GRN_NO_FLAGS); + GRN_TEXT_INIT(&(data->typed.body), GRN_NO_FLAGS); + GRN_TEXT_INIT(&(data->typed.foot), GRN_NO_FLAGS); + + grn_ctx_use(context, location_conf->database); + rc = ngx_http_groonga_context_check_error(r->connection->log); + if (rc != NGX_OK) { + return rc; + } + + grn_ctx_recv_handler_set(context, + ngx_http_groonga_context_receive_handler, + data); + + return NGX_OK; +} + +static void +ngx_http_groonga_handler_process_command_path(ngx_http_request_t *r, + ngx_str_t *command_path, + ngx_http_groonga_handler_data_t *data, + int flags) +{ + grn_obj uri; + + GRN_TEXT_INIT(&uri, 0); + GRN_TEXT_PUTS(context, &uri, "/d/"); + GRN_TEXT_PUT(context, &uri, command_path->data, command_path->len); + grn_ctx_send(context, GRN_TEXT_VALUE(&uri), GRN_TEXT_LEN(&uri), flags); + data->rc = context->rc; + ngx_http_groonga_context_log_error(r->connection->log); + GRN_OBJ_FIN(context, &uri); +} + +static grn_bool +ngx_http_groonga_handler_validate_post_command(ngx_http_request_t *r, + ngx_str_t *command_path, + ngx_http_groonga_handler_data_t *data) +{ + ngx_str_t command; + + command.data = command_path->data; + if (r->args.len == 0) { + command.len = command_path->len; + } else { + command.len = command_path->len - r->args.len - strlen("?"); + } + if (ngx_str_equal_c_string(&command, "load")) { + return GRN_TRUE; + } + + data->rc = GRN_INVALID_ARGUMENT; + ngx_http_groonga_handler_set_content_type(r, "text/plain"); + GRN_TEXT_PUTS(context, &(data->typed.body), + "command for POST must be <load>: <"); + GRN_TEXT_PUT(context, &(data->typed.body), command.data, command.len); + GRN_TEXT_PUTS(context, &(data->typed.body), ">"); + + return GRN_FALSE; +} + +static void +ngx_http_groonga_send_body(ngx_http_request_t *r, + ngx_http_groonga_handler_data_t *data) +{ + ngx_log_t *log; + grn_obj line_buffer; + size_t line_start_offset; + size_t line_check_start_offset; + ngx_chain_t *chain; + size_t line_buffer_chunk_size = 4096; + + log = r->connection->log; + + GRN_TEXT_INIT(&line_buffer, 0); + line_start_offset = 0; + line_check_start_offset = 0; + for (chain = r->request_body->bufs; chain; chain = chain->next) { + ngx_buf_t *buffer; + size_t rest_buffer_size; + off_t offset; + + buffer = chain->buf; + rest_buffer_size = ngx_buf_size(buffer); + offset = 0; + while (rest_buffer_size > 0) { + size_t current_buffer_size; + + if (rest_buffer_size > line_buffer_chunk_size) { + current_buffer_size = line_buffer_chunk_size; + } else { + current_buffer_size = rest_buffer_size; + } + + if (ngx_buf_in_memory(buffer)) { + GRN_TEXT_PUT(context, + &line_buffer, + buffer->pos + offset, + current_buffer_size); + } else { + ngx_int_t rc; + grn_bulk_reserve(context, &line_buffer, current_buffer_size); + rc = ngx_read_file(buffer->file, + (u_char *)GRN_BULK_CURR(&line_buffer), + current_buffer_size, + offset); + if (rc < 0) { + GRN_PLUGIN_ERROR(context, + GRN_INPUT_OUTPUT_ERROR, + "[nginx][post][body][read] " + "failed to read a request body from file"); + goto exit; + } + GRN_BULK_INCR_LEN(&line_buffer, current_buffer_size); + } + offset += current_buffer_size; + rest_buffer_size -= current_buffer_size; + + { + const char *line_start; + const char *line_current; + const char *line_end; + + line_start = GRN_TEXT_VALUE(&line_buffer) + line_start_offset; + line_end = GRN_TEXT_VALUE(&line_buffer) + GRN_TEXT_LEN(&line_buffer); + for (line_current = line_start + line_check_start_offset; + line_current < line_end; + line_current++) { + size_t line_length; + int flags = GRN_NO_FLAGS; + + if (*line_current != '\n') { + continue; + } + + line_length = line_current - line_start + 1; + if (line_current + 1 == line_end && + !chain->next && + rest_buffer_size == 0) { + flags |= GRN_CTX_TAIL; + } + grn_ctx_send(context, line_start, line_length, flags); + line_start_offset += line_length; + line_start += line_length; + ngx_http_groonga_context_log_error(log); + if (context->rc != GRN_SUCCESS && data->rc == GRN_SUCCESS) { + data->rc = context->rc; + } + } + + if (line_start_offset == 0) { + line_buffer_chunk_size *= 2; + line_check_start_offset = GRN_TEXT_LEN(&line_buffer); + } else if ((size_t)GRN_TEXT_LEN(&line_buffer) == line_start_offset) { + GRN_BULK_REWIND(&line_buffer); + line_start_offset = 0; + line_check_start_offset = 0; + } else { + size_t rest_line_size; + rest_line_size = GRN_TEXT_LEN(&line_buffer) - line_start_offset; + grn_memmove(GRN_TEXT_VALUE(&line_buffer), + GRN_TEXT_VALUE(&line_buffer) + line_start_offset, + rest_line_size); + grn_bulk_truncate(context, &line_buffer, rest_line_size); + line_start_offset = 0; + line_check_start_offset = GRN_TEXT_LEN(&line_buffer); + } + } + } + } + + if (GRN_TEXT_LEN(&line_buffer) > 0) { + grn_ctx_send(context, + GRN_TEXT_VALUE(&line_buffer), + GRN_TEXT_LEN(&line_buffer), + GRN_CTX_TAIL); + ngx_http_groonga_context_log_error(log); + if (context->rc != GRN_SUCCESS && data->rc == GRN_SUCCESS) { + data->rc = context->rc; + } + } + +exit : + GRN_OBJ_FIN(context, &line_buffer); +} + +static void +ngx_http_groonga_handler_process_body(ngx_http_request_t *r, + ngx_http_groonga_handler_data_t *data) +{ + ngx_buf_t *body; + + body = r->request_body->bufs->buf; + if (!body) { + data->rc = GRN_INVALID_ARGUMENT; + ngx_http_groonga_handler_set_content_type(r, "text/plain"); + GRN_TEXT_PUTS(context, &(data->typed.body), "must send load data as body"); + return; + } + + ngx_http_groonga_send_body(r, data); +} + + +static void +ngx_http_groonga_handler_process_load(ngx_http_request_t *r, + ngx_str_t *command_path, + ngx_http_groonga_handler_data_t *data) +{ + if (!ngx_http_groonga_handler_validate_post_command(r, command_path, data)) { + return; + } + + ngx_http_groonga_handler_process_command_path(r, + command_path, + data, + GRN_NO_FLAGS); + if (data->rc != GRN_SUCCESS) { + return; + } + + ngx_http_groonga_handler_process_body(r, data); +} + +static ngx_chain_t * +ngx_http_groonga_attach_chain(ngx_chain_t *chain, ngx_chain_t *new_chain) +{ + ngx_chain_t *last_chain; + + if (new_chain->buf->last == new_chain->buf->pos) { + return chain; + } + + new_chain->buf->last_buf = 1; + new_chain->next = NULL; + if (!chain) { + return new_chain; + } + + chain->buf->last_buf = 0; + last_chain = chain; + while (last_chain->next) { + last_chain = last_chain->next; + } + last_chain->next = new_chain; + return chain; +} + +static ngx_int_t +ngx_http_groonga_handler_send_response(ngx_http_request_t *r, + ngx_http_groonga_handler_data_t *data) +{ + ngx_int_t rc; + const char *content_type; + ngx_buf_t *head_buf, *body_buf, *foot_buf; + ngx_chain_t head_chain, body_chain, foot_chain; + ngx_chain_t *output_chain = NULL; + + if (data->raw.processed) { + return data->raw.rc; + } + + /* set the 'Content-type' header */ + if (r->headers_out.content_type.len == 0) { + grn_obj *foot = &(data->typed.foot); + if (grn_ctx_get_output_type(context) == GRN_CONTENT_JSON && + GRN_TEXT_LEN(foot) > 0 && + GRN_TEXT_VALUE(foot)[GRN_TEXT_LEN(foot) - 1] == ';') { + content_type = "application/javascript"; + } else { + content_type = grn_ctx_get_mime_type(context); + } + ngx_http_groonga_handler_set_content_type(r, content_type); + } + + /* allocate buffers for a response body */ + head_buf = ngx_http_groonga_grn_obj_to_ngx_buf(r->pool, &(data->typed.head)); + if (!head_buf) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + body_buf = ngx_http_groonga_grn_obj_to_ngx_buf(r->pool, &(data->typed.body)); + if (!body_buf) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + foot_buf = ngx_http_groonga_grn_obj_to_ngx_buf(r->pool, &(data->typed.foot)); + if (!foot_buf) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + /* attach buffers to the buffer chain */ + head_chain.buf = head_buf; + output_chain = ngx_http_groonga_attach_chain(output_chain, &head_chain); + body_chain.buf = body_buf; + output_chain = ngx_http_groonga_attach_chain(output_chain, &body_chain); + foot_chain.buf = foot_buf; + output_chain = ngx_http_groonga_attach_chain(output_chain, &foot_chain); + + /* set the status line */ + r->headers_out.status = ngx_http_groonga_grn_rc_to_http_status(data->rc); + r->headers_out.content_length_n = GRN_TEXT_LEN(&(data->typed.head)) + + GRN_TEXT_LEN(&(data->typed.body)) + + GRN_TEXT_LEN(&(data->typed.foot)); + if (r->headers_out.content_length_n == 0) { + r->header_only = 1; + } + + /* send the headers of your response */ + rc = ngx_http_send_header(r); + + if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { + return rc; + } + + /* send the buffer chain of your response */ + rc = ngx_http_output_filter(r, output_chain); + + return rc; +} + +static ngx_int_t +ngx_http_groonga_handler_get(ngx_http_request_t *r) +{ + ngx_int_t rc; + ngx_str_t command_path; + ngx_http_groonga_handler_data_t *data; + + rc = ngx_http_groonga_extract_command_path(r, &command_path); + if (rc != NGX_OK) { + return rc; + } + + rc = ngx_http_groonga_handler_create_data(r, &data); + if (rc != NGX_OK) { + return rc; + } + + ngx_http_groonga_handler_process_command_path(r, + &command_path, + data, + GRN_CTX_TAIL); + + /* discard request body, since we don't need it here */ + rc = ngx_http_discard_request_body(r); + if (rc != NGX_OK) { + return rc; + } + + rc = ngx_http_groonga_handler_send_response(r, data); + + return rc; +} + +static void +ngx_http_groonga_handler_post_send_error_response(ngx_http_request_t *r, + ngx_int_t rc) +{ + r->headers_out.status = rc; + r->headers_out.content_length_n = 0; + r->header_only = 1; + rc = ngx_http_send_header(r); + ngx_http_finalize_request(r, rc); +} + +static void +ngx_http_groonga_handler_post(ngx_http_request_t *r) +{ + ngx_int_t rc; + ngx_str_t command_path; + ngx_http_groonga_handler_data_t *data = NULL; + + rc = ngx_http_groonga_extract_command_path(r, &command_path); + if (rc != NGX_OK) { + ngx_http_groonga_handler_post_send_error_response(r, rc); + return; + } + + rc = ngx_http_groonga_handler_create_data(r, &data); + if (rc != NGX_OK) { + ngx_http_groonga_handler_post_send_error_response(r, rc); + return; + } + + ngx_http_groonga_handler_process_load(r, &command_path, data); + rc = ngx_http_groonga_handler_send_response(r, data); + ngx_http_finalize_request(r, rc); +} + +static ngx_int_t +ngx_http_groonga_handler(ngx_http_request_t *r) +{ + ngx_int_t rc; + + switch (r->method) { + case NGX_HTTP_GET: + case NGX_HTTP_HEAD: + rc = ngx_http_groonga_handler_get(r); + break; + case NGX_HTTP_POST: + rc = ngx_http_read_client_request_body(r, ngx_http_groonga_handler_post); + if (rc < NGX_HTTP_SPECIAL_RESPONSE) { + rc = NGX_DONE; + } + break; + default: + rc = NGX_HTTP_NOT_ALLOWED; + break; + } + + return rc; +} + +static char * +ngx_http_groonga_conf_set_groonga_slot(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + char *status; + ngx_http_core_loc_conf_t *location_conf; + ngx_http_groonga_loc_conf_t *groonga_location_conf = conf; + + status = ngx_conf_set_flag_slot(cf, cmd, conf); + if (status != NGX_CONF_OK) { + return status; + } + + location_conf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module); + if (groonga_location_conf->enabled) { + location_conf->handler = ngx_http_groonga_handler; + groonga_location_conf->name = + ngx_str_null_terminate(cf->pool, &(location_conf->name)); + groonga_location_conf->config_file = + ngx_str_null_terminate(cf->pool, &(cf->conf_file->file.name)); + groonga_location_conf->config_line = cf->conf_file->line; + } else { + location_conf->handler = NULL; + } + + return NGX_CONF_OK; +} + +static char * +ngx_http_groonga_conf_set_log_path_slot(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + char *status; + ngx_http_groonga_loc_conf_t *groonga_location_conf = conf; + + status = ngx_conf_set_str_slot(cf, cmd, conf); + if (status != NGX_CONF_OK) { + return status; + } + + if (!groonga_location_conf->log_path.data) { + return NGX_CONF_OK; + } + + if (!ngx_str_is_custom_path(&(groonga_location_conf->log_path))) { + return NGX_CONF_OK; + } + + groonga_location_conf->log_file = + ngx_conf_open_file(cf->cycle, &(groonga_location_conf->log_path)); + if (!groonga_location_conf->log_file) { + ngx_log_error(NGX_LOG_ERR, cf->cycle->log, 0, + "http_groonga: failed to open groonga log file: <%V>", + &(groonga_location_conf->log_path)); + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + +static char * +ngx_http_groonga_conf_set_log_level_slot(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + char *status = NGX_CONF_OK; + ngx_http_groonga_loc_conf_t *groonga_location_conf = conf; + char *value; + + value = ngx_str_null_terminate(cf->cycle->pool, + ((ngx_str_t *)cf->args->elts) + 1); + if (!grn_log_level_parse(value, &(groonga_location_conf->log_level))) { + status = "must be one of 'none', 'emergency', 'alert', " + "'critical', 'error', 'warning', 'notice', 'info', 'debug' and 'dump'"; + } + ngx_pfree(cf->cycle->pool, value); + + return status; +} + +static char * +ngx_http_groonga_conf_set_query_log_path_slot(ngx_conf_t *cf, + ngx_command_t *cmd, + void *conf) +{ + char *status; + ngx_http_groonga_loc_conf_t *groonga_location_conf = conf; + + status = ngx_conf_set_str_slot(cf, cmd, conf); + if (status != NGX_CONF_OK) { + return status; + } + + if (!groonga_location_conf->query_log_path.data) { + return NGX_CONF_OK; + } + + if (!ngx_str_is_custom_path(&(groonga_location_conf->query_log_path))) { + return NGX_CONF_OK; + } + + groonga_location_conf->query_log_file = + ngx_conf_open_file(cf->cycle, &(groonga_location_conf->query_log_path)); + if (!groonga_location_conf->query_log_file) { + ngx_log_error(NGX_LOG_ERR, cf->cycle->log, 0, + "http_groonga: failed to open Groonga query log file: <%V>", + &(groonga_location_conf->query_log_path)); + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + +static void * +ngx_http_groonga_create_loc_conf(ngx_conf_t *cf) +{ + ngx_http_groonga_loc_conf_t *conf; + conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_groonga_loc_conf_t)); + if (conf == NULL) { + return NGX_CONF_ERROR; + } + + conf->enabled = NGX_CONF_UNSET; + conf->database_path.data = NULL; + conf->database_path.len = 0; + conf->database_path_cstr = NULL; + conf->database_auto_create = NGX_CONF_UNSET; + conf->base_path.data = NULL; + conf->base_path.len = 0; + conf->log_path.data = NULL; + conf->log_path.len = 0; + conf->log_file = NULL; + conf->log_level = GRN_LOG_DEFAULT_LEVEL; + conf->query_log_path.data = NULL; + conf->query_log_path.len = 0; + conf->query_log_file = NULL; + conf->cache_limit = NGX_CONF_UNSET_SIZE; + conf->config_file = NULL; + conf->config_line = 0; + conf->cache = NULL; + conf->cache_base_path.data = NULL; + conf->cache_base_path.len = 0; + + return conf; +} + +static char * +ngx_http_groonga_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) +{ + ngx_http_groonga_loc_conf_t *prev = parent; + ngx_http_groonga_loc_conf_t *conf = child; + ngx_flag_t enabled = 0; + + if (conf->enabled != NGX_CONF_UNSET) { + enabled = conf->enabled; + } + + ngx_conf_merge_str_value(conf->database_path, prev->database_path, NULL); + ngx_conf_merge_value(conf->database_auto_create, + prev->database_auto_create, + GRN_TRUE); + ngx_conf_merge_size_value(conf->cache_limit, prev->cache_limit, + GRN_CACHE_DEFAULT_MAX_N_ENTRIES); + +#ifdef NGX_HTTP_GROONGA_LOG_PATH + ngx_conf_merge_str_value(conf->log_path, prev->log_path, + NGX_HTTP_GROONGA_LOG_PATH); + if (!conf->log_file && + ngx_str_is_custom_path(&(conf->log_path)) && + enabled) { + conf->log_file = ngx_conf_open_file(cf->cycle, &(conf->log_path)); + if (!conf->log_file) { + ngx_log_error(NGX_LOG_ERR, cf->cycle->log, 0, + "http_groonga: " + "failed to open the default Groonga log file: <%V>", + &(conf->log_path)); + return NGX_CONF_ERROR; + } + } +#endif + + ngx_conf_merge_str_value(conf->query_log_path, prev->query_log_path, + NGX_HTTP_GROONGA_QUERY_LOG_PATH); + if (!conf->query_log_file && + ngx_str_is_custom_path(&(conf->query_log_path)) && + enabled) { + conf->query_log_file = ngx_conf_open_file(cf->cycle, + &(conf->query_log_path)); + if (!conf->query_log_file) { + ngx_log_error(NGX_LOG_ERR, cf->cycle->log, 0, + "http_groonga: " + "failed to open the default Groonga query log file: <%V>", + &(conf->query_log_path)); + return NGX_CONF_ERROR; + } + } + + ngx_conf_merge_str_value(conf->cache_base_path, + prev->cache_base_path, + NULL); + + return NGX_CONF_OK; +} + +static void +ngx_http_groonga_each_loc_conf_in_tree(ngx_http_location_tree_node_t *node, + ngx_http_groonga_loc_conf_callback_pt callback, + void *user_data) +{ + if (!node) { + return; + } + + if (node->exact && node->exact->handler == ngx_http_groonga_handler) { + callback(node->exact->loc_conf[ngx_http_groonga_module.ctx_index], + user_data); + } + + if (node->inclusive && node->inclusive->handler == ngx_http_groonga_handler) { + callback(node->inclusive->loc_conf[ngx_http_groonga_module.ctx_index], + user_data); + } + + ngx_http_groonga_each_loc_conf_in_tree(node->left, callback, user_data); + ngx_http_groonga_each_loc_conf_in_tree(node->right, callback, user_data); + ngx_http_groonga_each_loc_conf_in_tree(node->tree, callback, user_data); +} + +static void +ngx_http_groonga_each_loc_conf(ngx_http_conf_ctx_t *http_conf, + ngx_http_groonga_loc_conf_callback_pt callback, + void *user_data) +{ + ngx_http_core_main_conf_t *main_conf; + ngx_http_core_srv_conf_t **server_confs; + ngx_uint_t i; + + if (!http_conf) { + return; + } + + main_conf = http_conf->main_conf[ngx_http_core_module.ctx_index]; + server_confs = main_conf->servers.elts; + for (i = 0; i < main_conf->servers.nelts; i++) { + ngx_http_core_srv_conf_t *server_conf; + ngx_http_core_loc_conf_t *location_conf; + + server_conf = server_confs[i]; + location_conf = server_conf->ctx->loc_conf[ngx_http_core_module.ctx_index]; + ngx_http_groonga_each_loc_conf_in_tree(location_conf->static_locations, + callback, + user_data); + +#if NGX_PCRE + if (location_conf->regex_locations) { + ngx_uint_t j; + for (j = 0; location_conf->regex_locations[j]; j++) { + ngx_http_core_loc_conf_t *regex_location_conf; + + regex_location_conf = location_conf->regex_locations[j]; + if (regex_location_conf->handler == ngx_http_groonga_handler) { + callback(regex_location_conf->loc_conf[ngx_http_groonga_module.ctx_index], + user_data); + } + } + } +#endif + } +} + +static void +ngx_http_groonga_set_logger_callback(ngx_http_groonga_loc_conf_t *location_conf, + void *user_data) +{ + ngx_http_groonga_database_callback_data_t *data = user_data; + + data->rc = ngx_http_groonga_context_init_logger(location_conf, + data->pool, + data->log); + if (data->rc != NGX_OK) { + return; + } + data->rc = ngx_http_groonga_context_init_query_logger(location_conf, + data->pool, + data->log); + if (data->rc != NGX_OK) { + return; + } +} + +static ngx_int_t +ngx_http_groonga_mkdir_p(ngx_log_t *log, const char *dir_name) +{ + char sub_path[PATH_MAX]; + size_t i, dir_name_length; + + dir_name_length = strlen(dir_name); + sub_path[0] = dir_name[0]; + for (i = 1; i < dir_name_length + 1; i++) { + if (dir_name[i] == '/' || dir_name[i] == '\0') { + struct stat stat_buffer; + sub_path[i] = '\0'; + if (stat(sub_path, &stat_buffer) == -1) { + if (ngx_create_dir(sub_path, 0700) == -1) { + ngx_log_error(NGX_LOG_EMERG, log, 0, + "failed to create directory: %s (%s): %s", + sub_path, dir_name, + strerror(errno)); + return NGX_ERROR; + } + } + } + sub_path[i] = dir_name[i]; + } + + return NGX_OK; +} + +static void +ngx_http_groonga_create_database(ngx_http_groonga_loc_conf_t *location_conf, + ngx_http_groonga_database_callback_data_t *data) +{ + const char *database_base_name; + + database_base_name = strrchr(location_conf->database_path_cstr, '/'); + if (database_base_name) { + char database_dir[PATH_MAX]; + database_dir[0] = '\0'; + strncat(database_dir, + location_conf->database_path_cstr, + database_base_name - location_conf->database_path_cstr); + data->rc = ngx_http_groonga_mkdir_p(data->log, database_dir); + if (data->rc != NGX_OK) { + return; + } + } + + location_conf->database = + grn_db_create(context, location_conf->database_path_cstr, NULL); + if (context->rc == GRN_SUCCESS) { + return; + } + + ngx_log_error(NGX_LOG_EMERG, data->log, 0, + "failed to create Groonga database: %s", + context->errbuf); + data->rc = NGX_ERROR; +} + +static void +ngx_http_groonga_open_database_callback(ngx_http_groonga_loc_conf_t *location_conf, + void *user_data) +{ + ngx_http_groonga_database_callback_data_t *data = user_data; + + data->rc = ngx_http_groonga_context_init_logger(location_conf, + data->pool, + data->log); + if (data->rc != NGX_OK) { + return; + } + data->rc = ngx_http_groonga_context_init_query_logger(location_conf, + data->pool, + data->log); + if (data->rc != NGX_OK) { + return; + } + + if (!location_conf->database_path.data) { + ngx_log_error(NGX_LOG_EMERG, data->log, 0, + "%s: \"groonga_database\" must be specified in block at %s:%d", + location_conf->name, + location_conf->config_file, + location_conf->config_line); + data->rc = NGX_ERROR; + return; + } + + if (!location_conf->database_path_cstr) { + location_conf->database_path_cstr = + ngx_str_null_terminate(data->pool, &(location_conf->database_path)); + } + + location_conf->database = + grn_db_open(context, location_conf->database_path_cstr); + if (context->rc != GRN_SUCCESS) { + if (location_conf->database_auto_create) { + ngx_http_groonga_create_database(location_conf, data); + } else { + ngx_log_error(NGX_LOG_EMERG, data->log, 0, + "failed to open Groonga database: %s", + context->errbuf); + data->rc = NGX_ERROR; + } + if (data->rc != NGX_OK) { + return; + } + } + + if (location_conf->cache_base_path.data && + ngx_str_is_custom_path(&(location_conf->cache_base_path))) { + char cache_base_path[PATH_MAX]; + grn_memcpy(cache_base_path, + location_conf->cache_base_path.data, + location_conf->cache_base_path.len); + cache_base_path[location_conf->cache_base_path.len] = '\0'; + location_conf->cache = grn_persistent_cache_open(context, cache_base_path); + } else { + location_conf->cache = grn_cache_open(context); + } + if (!location_conf->cache) { + ngx_log_error(NGX_LOG_EMERG, data->log, 0, + "failed to open Groonga cache: %s", + context->errbuf); + data->rc = NGX_ERROR; + return; + } + + if (location_conf->cache_limit != NGX_CONF_UNSET_SIZE) { + grn_cache_set_max_n_entries(context, + location_conf->cache, + location_conf->cache_limit); + } +} + +static void +ngx_http_groonga_close_database_callback(ngx_http_groonga_loc_conf_t *location_conf, + void *user_data) +{ + ngx_http_groonga_database_callback_data_t *data = user_data; + + ngx_http_groonga_context_init_logger(location_conf, + data->pool, + data->log); + ngx_http_groonga_context_init_query_logger(location_conf, + data->pool, + data->log); + grn_cache_current_set(context, location_conf->cache); + + grn_obj_close(context, location_conf->database); + ngx_http_groonga_context_log_error(data->log); + + grn_cache_current_set(context, NULL); + grn_cache_close(context, location_conf->cache); +} + +static ngx_int_t +ngx_http_groonga_init_process(ngx_cycle_t *cycle) +{ + grn_rc rc; + ngx_http_conf_ctx_t *http_conf; + ngx_http_groonga_database_callback_data_t data; + + grn_thread_set_get_limit_func(ngx_http_groonga_get_thread_limit, NULL); + +#ifdef NGX_HTTP_GROONGA_LOG_PATH + grn_default_logger_set_path(NGX_HTTP_GROONGA_LOG_PATH); +#endif + + http_conf = + (ngx_http_conf_ctx_t *)ngx_get_conf(cycle->conf_ctx, ngx_http_module); + + data.log = cycle->log; + data.pool = cycle->pool; + data.rc = NGX_OK; + ngx_http_groonga_each_loc_conf(http_conf, + ngx_http_groonga_set_logger_callback, + &data); + + if (data.rc != NGX_OK) { + return data.rc; + } + + rc = grn_init(); + if (rc != GRN_SUCCESS) { + return NGX_ERROR; + } + + grn_set_segv_handler(); + + rc = grn_ctx_init(context, GRN_NO_FLAGS); + if (rc != GRN_SUCCESS) { + return NGX_ERROR; + } + + ngx_http_groonga_each_loc_conf(http_conf, + ngx_http_groonga_open_database_callback, + &data); + + return data.rc; +} + +static void +ngx_http_groonga_exit_process(ngx_cycle_t *cycle) +{ + ngx_http_conf_ctx_t *http_conf; + ngx_http_groonga_database_callback_data_t data; + + http_conf = + (ngx_http_conf_ctx_t *)ngx_get_conf(cycle->conf_ctx, ngx_http_module); + data.log = cycle->log; + data.pool = cycle->pool; + ngx_http_groonga_each_loc_conf(http_conf, + ngx_http_groonga_close_database_callback, + &data); + + grn_ctx_fin(context); + + grn_fin(); + + return; +} + +/* entry point */ +static ngx_command_t ngx_http_groonga_commands[] = { + { ngx_string("groonga"), + NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_http_groonga_conf_set_groonga_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, enabled), + NULL }, + + { ngx_string("groonga_database"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, database_path), + NULL }, + + { ngx_string("groonga_database_auto_create"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, database_auto_create), + NULL }, + + { ngx_string("groonga_base_path"), + NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, base_path), + NULL }, + + { ngx_string("groonga_log_path"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_http_groonga_conf_set_log_path_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, log_path), + NULL }, + + { ngx_string("groonga_log_level"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_http_groonga_conf_set_log_level_slot, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("groonga_query_log_path"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_http_groonga_conf_set_query_log_path_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, query_log_path), + NULL }, + + { ngx_string("groonga_cache_limit"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, cache_limit), + NULL }, + + { ngx_string("groonga_default_request_timeout"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, default_request_timeout_msec), + NULL }, + + { ngx_string("groonga_cache_base_path"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, cache_base_path), + NULL }, + + ngx_null_command +}; + +static ngx_http_module_t ngx_http_groonga_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + ngx_http_groonga_create_loc_conf, /* create location configuration */ + ngx_http_groonga_merge_loc_conf, /* merge location configuration */ +}; + +ngx_module_t ngx_http_groonga_module = { + NGX_MODULE_V1, + &ngx_http_groonga_module_ctx, /* module context */ + ngx_http_groonga_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + ngx_http_groonga_init_process, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + ngx_http_groonga_exit_process, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; diff --git a/storage/mroonga/vendor/groonga/src/suggest/CMakeLists.txt b/storage/mroonga/vendor/groonga/src/suggest/CMakeLists.txt new file mode 100644 index 00000000..ec85c1fb --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/CMakeLists.txt @@ -0,0 +1,87 @@ +# Copyright(C) 2012-2013 Brazil +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/../../lib + ${MRUBY_INCLUDE_DIRS} + ${MESSAGE_PACK_INCLUDE_DIRS}) + +read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/create_dataset_sources.am + GROONGA_SUGGEST_CREATE_DATASET_SOURCES) +add_executable(groonga-suggest-create-dataset + ${GROONGA_SUGGEST_CREATE_DATASET_SOURCES}) +set_source_files_properties(${GROONGA_SUGGEST_CREATE_DATASET_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") +target_link_libraries(groonga-suggest-create-dataset libgroonga) +install( + TARGETS groonga-suggest-create-dataset + DESTINATION ${BIN_DIR}) + +if(GRN_WITH_LIBEVENT AND GRN_WITH_ZEROMQ AND GRN_WITH_MESSAGE_PACK) + set(GRN_WITH_SUGGEST_LEARNER TRUE) +else() + set(GRN_WITH_SUGGEST_LEARNER FALSE) +endif() + +if(GRN_WITH_SUGGEST_LEARNER) + include_directories( + ${LIBEVENT_INCLUDE_DIRS} + ${ZEROMQ_INCLUDE_DIRS} + ${MESSAGE_PACK_INCLUDE_DIRS} + ) + link_directories( + ${LIBEVENT_LIBRARY_DIRS} + ${ZEROMQ_LIBRARY_DIRS} + ${MESSAGE_PACK_LIBRARY_DIRS} + ) + + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/util_sources.am + GROONGA_SUGGEST_UTIL_SOURCES) + add_library(groonga-suggest-util STATIC ${GROONGA_SUGGEST_UTIL_SOURCES}) + set_source_files_properties(${GROONGA_SUGGEST_UTIL_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") + + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/learner_sources.am + GROONGA_SUGGEST_LEARNER_SOURCES) + add_executable(groonga-suggest-learner ${GROONGA_SUGGEST_LEARNER_SOURCES}) + set_source_files_properties(${GROONGA_SUGGEST_LEARNER_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") + target_link_libraries(groonga-suggest-learner + groonga-suggest-util + libgroonga + ${LIBEVENT_LIBRARIES} + ${ZEROMQ_LIBRARIES} + ${MESSAGE_PACK_LIBRARIES}) + + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/httpd_sources.am + GROONGA_SUGGEST_HTTPD_SOURCES) + add_executable(groonga-suggest-httpd ${GROONGA_SUGGEST_HTTPD_SOURCES}) + set_source_files_properties(${GROONGA_SUGGEST_HTTPD_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") + target_link_libraries(groonga-suggest-httpd + groonga-suggest-util + libgroonga + ${LIBEVENT_LIBRARIES} + ${ZEROMQ_LIBRARIES} + ${MESSAGE_PACK_LIBRARIES}) + + install( + TARGETS groonga-suggest-learner groonga-suggest-httpd + DESTINATION ${BIN_DIR}) +endif() diff --git a/storage/mroonga/vendor/groonga/src/suggest/Makefile.am b/storage/mroonga/vendor/groonga/src/suggest/Makefile.am new file mode 100644 index 00000000..91260016 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/Makefile.am @@ -0,0 +1,74 @@ +bin_PROGRAMS = + +NONEXISTENT_CXX_SOURCE = nonexistent.cpp + +bin_PROGRAMS += \ + groonga-suggest-create-dataset + +if ENABLE_SUGGEST_LEARNER +bin_PROGRAMS += \ + groonga-suggest-learner \ + groonga-suggest-httpd +noinst_LTLIBRARIES = libutil.la +endif + +EXTRA_DIST = \ + CMakeLists.txt + +AM_CFLAGS = \ + $(NO_STRICT_ALIASING_CFLAGS) \ + $(COVERAGE_CFLAGS) \ + $(GRN_CFLAGS) \ + $(MESSAGE_PACK_CFLAGS) \ + $(MRUBY_CFLAGS) + +DEFS += $(GRN_DEFS) + +AM_LDFLAGS = -no-undefined + +DEFAULT_INCLUDES = \ + -I$(top_builddir) \ + -I$(srcdir) \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/lib \ + $(GROONGA_INCLUDEDIR) + +include learner_sources.am +nodist_EXTRA_groonga_suggest_learner_SOURCES = $(NONEXISTENT_CXX_SOURCE) +groonga_suggest_learner_CFLAGS = \ + $(AM_CFLAGS) \ + $(LIBEVENT_CFLAGS) \ + $(LIBZMQ_CFLAGS) \ + $(MESSAGE_PACK_CFLAGS) +groonga_suggest_learner_LDADD = \ + libutil.la \ + $(top_builddir)/lib/libgroonga.la \ + $(LIBEVENT_LIBS) \ + $(LIBZMQ_LIBS) \ + $(MESSAGE_PACK_LIBS) + +include httpd_sources.am +nodist_EXTRA_groonga_suggest_httpd_SOURCES = $(NONEXISTENT_CXX_SOURCE) +groonga_suggest_httpd_CFLAGS = \ + $(AM_CFLAGS) \ + $(LIBEVENT_CFLAGS) \ + $(LIBZMQ_CFLAGS) \ + $(MESSAGE_PACK_CFLAGS) +groonga_suggest_httpd_LDADD = \ + libutil.la \ + $(top_builddir)/lib/libgroonga.la \ + $(LIBEVENT_LIBS) \ + $(LIBZMQ_LIBS) \ + $(MESSAGE_PACK_LIBS) + +include create_dataset_sources.am +nodist_EXTRA_groonga_suggest_create_dataset_SOURCES = $(NONEXISTENT_CXX_SOURCE) +groonga_suggest_create_dataset_CFLAGS = \ + $(AM_CFLAGS) +groonga_suggest_create_dataset_LDADD = \ + $(top_builddir)/lib/libgroonga.la + +include util_sources.am +libutil_la_CFLAGS = \ + $(AM_CFLAGS) \ + $(LIBEVENT_CFLAGS) diff --git a/storage/mroonga/vendor/groonga/src/suggest/create_dataset_sources.am b/storage/mroonga/vendor/groonga/src/suggest/create_dataset_sources.am new file mode 100644 index 00000000..cfecd650 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/create_dataset_sources.am @@ -0,0 +1,2 @@ +groonga_suggest_create_dataset_SOURCES = \ + groonga_suggest_create_dataset.c diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_create_dataset.c b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_create_dataset.c new file mode 100644 index 00000000..7cec2922 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_create_dataset.c @@ -0,0 +1,223 @@ +/* -*- c-basic-offset: 2 -*- */ +/* Copyright(C) 2010-2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +/* For grn_str_getopt() */ +#include <grn_str.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <groonga.h> + +typedef enum { + MODE_NONE, + MODE_USAGE +} ModeFlags; + +static const char *DEFAULT_DEFAULT_TOKENIZER = "TokenBigram"; + +static void +usage(FILE *output, int argc, char **argv) +{ +#define OUTPUT(...) fprintf(output, __VA_ARGS__) + + OUTPUT("Usage: %s [OPTIONS] DB_PATH DATASET_NAME\n", argv[0]); + OUTPUT(" e.g.: %s /tmp/db shops\n", argv[0]); + OUTPUT("\n"); + OUTPUT("Options:\n"); + OUTPUT(" --default-tokenizer=TOKENIZER Use TOKENIZER as the default\n"); + OUTPUT(" tokenizer for item name\n"); + OUTPUT(" (default: %s)\n", + DEFAULT_DEFAULT_TOKENIZER); + OUTPUT(" -h, --help Show this message and exit\n"); + +#undef OUTPUT +} + +static void +output(grn_ctx *ctx) +{ + int flags = 0; + char *str; + unsigned int str_len; + + do { + grn_ctx_recv(ctx, &str, &str_len, &flags); + if (str_len > 0 || ctx->rc) { + if (ctx->rc) { + printf("ERROR (%d): %s\n", ctx->rc, ctx->errbuf); + } + if (str_len > 0) { + printf("%.*s\n", str_len, str); + } + } + } while (flags & GRN_CTX_MORE); +} + +static void +send_command(grn_ctx *ctx, grn_obj *buffer, const char *command, + const char *dataset_name) +{ + const char *p = command; + const char *dataset_place_holder = "${DATASET}"; + char *dataset_place_holder_position; + + if (ctx->rc != GRN_SUCCESS) { + return; + } + + GRN_BULK_REWIND(buffer); + while ((dataset_place_holder_position = strstr(p, dataset_place_holder))) { + GRN_TEXT_PUT(ctx, buffer, p, dataset_place_holder_position - p); + GRN_TEXT_PUTS(ctx, buffer, dataset_name); + p = dataset_place_holder_position + strlen(dataset_place_holder); + } + GRN_TEXT_PUTS(ctx, buffer, p); + printf("> %.*s\n", (int)GRN_TEXT_LEN(buffer), GRN_TEXT_VALUE(buffer)); + grn_ctx_send(ctx, GRN_TEXT_VALUE(buffer), GRN_TEXT_LEN(buffer), 0); + output(ctx); +} + + +int +main(int argc, char **argv) +{ + const char *db_path; + const char *dataset_name; + grn_ctx ctx_, *ctx; + grn_obj *db; + grn_bool success = GRN_TRUE; + int parsed_argc, rest_argc; + int flags = MODE_NONE; + const char *default_tokenizer = NULL; + static grn_str_getopt_opt opts[] = { + {'\0', "default-tokenizer", NULL, 0, GETOPT_OP_NONE}, + {'h', "help", NULL, MODE_USAGE, GETOPT_OP_UPDATE} + }; + + opts[0].arg = &default_tokenizer; + + parsed_argc = grn_str_getopt(argc, argv, opts, &flags); + if (parsed_argc < 0) { + usage(stderr, argc, argv); + return EXIT_FAILURE; + } + + if (flags & MODE_USAGE) { + usage(stdout, argc, argv); + return EXIT_SUCCESS; + } + + rest_argc = argc - parsed_argc; + if (rest_argc != 2) { + usage(stderr, argc, argv); + return EXIT_FAILURE; + } + + db_path = argv[parsed_argc]; + dataset_name = argv[parsed_argc + 1]; + + grn_init(); + + ctx = &ctx_; + grn_ctx_init(ctx, 0); + db = grn_db_open(ctx, db_path); + if (!db) { + if (ctx->rc == GRN_NO_SUCH_FILE_OR_DIRECTORY) { + db = grn_db_create(ctx, db_path, NULL); + if (!db) { + fprintf(stderr, "DB create failed (%s): %s\n", db_path, ctx->errbuf); + } + } else { + fprintf(stderr, "DB open failed (%s): %s\n", db_path, ctx->errbuf); + } + } + + if (db) { + grn_obj text; + GRN_TEXT_INIT(&text, 0); +#define SEND(string) send_command(ctx, &text, string, dataset_name) + SEND("plugin_register suggest/suggest"); + SEND("table_create event_type TABLE_HASH_KEY ShortText"); + { + grn_obj query; + GRN_TEXT_INIT(&query, 0); + GRN_TEXT_PUTS(ctx, &query, + "table_create bigram TABLE_PAT_KEY ShortText " + "--default_tokenizer "); + if (default_tokenizer) { + GRN_TEXT_PUTS(ctx, &query, default_tokenizer); + } else { + GRN_TEXT_PUTS(ctx, &query, DEFAULT_DEFAULT_TOKENIZER); + } + GRN_TEXT_PUTS(ctx, &query, " --normalizer NormalizerAuto"); + GRN_TEXT_PUTC(ctx, &query, '\0'); + SEND(GRN_TEXT_VALUE(&query)); + GRN_OBJ_FIN(ctx, &query); + } + SEND("table_create kana TABLE_PAT_KEY ShortText " + "--normalizer NormalizerAuto"); + SEND("table_create item_${DATASET} TABLE_PAT_KEY " + "ShortText --default_tokenizer TokenDelimit " + "--normalizer NormalizerAuto"); + SEND("column_create bigram item_${DATASET}_key " + "COLUMN_INDEX|WITH_POSITION item_${DATASET} _key"); + SEND("column_create item_${DATASET} kana COLUMN_VECTOR kana"); + SEND("column_create kana item_${DATASET}_kana COLUMN_INDEX " + "item_${DATASET} kana"); + SEND("column_create item_${DATASET} freq COLUMN_SCALAR Int32"); + SEND("column_create item_${DATASET} last COLUMN_SCALAR Time"); + SEND("column_create item_${DATASET} boost COLUMN_SCALAR Int32"); + SEND("column_create item_${DATASET} freq2 COLUMN_SCALAR Int32"); + SEND("column_create item_${DATASET} buzz COLUMN_SCALAR Int32"); + + SEND("table_create pair_${DATASET} TABLE_HASH_KEY UInt64"); + SEND("column_create pair_${DATASET} pre COLUMN_SCALAR item_${DATASET}"); + SEND("column_create pair_${DATASET} post COLUMN_SCALAR item_${DATASET}"); + SEND("column_create pair_${DATASET} freq0 COLUMN_SCALAR Int32"); + SEND("column_create pair_${DATASET} freq1 COLUMN_SCALAR Int32"); + SEND("column_create pair_${DATASET} freq2 COLUMN_SCALAR Int32"); + SEND("column_create item_${DATASET} co COLUMN_INDEX pair_${DATASET} pre"); + + SEND("table_create sequence_${DATASET} TABLE_HASH_KEY ShortText"); + SEND("table_create event_${DATASET} TABLE_NO_KEY"); + SEND("column_create sequence_${DATASET} events " + "COLUMN_VECTOR|RING_BUFFER event_${DATASET}"); + SEND("column_create event_${DATASET} type COLUMN_SCALAR event_type"); + SEND("column_create event_${DATASET} time COLUMN_SCALAR Time"); + SEND("column_create event_${DATASET} item COLUMN_SCALAR item_${DATASET}"); + SEND("column_create event_${DATASET} sequence COLUMN_SCALAR " + "sequence_${DATASET}"); + + SEND("table_create configuration TABLE_HASH_KEY ShortText"); + SEND("column_create configuration weight COLUMN_SCALAR UInt32"); + SEND("load --table configuration"); + SEND("["); + SEND("{\"_key\": \"${DATASET}\", \"weight\": 1}"); + SEND("]"); +#undef SEND + success = ctx->rc == GRN_SUCCESS; + GRN_OBJ_FIN(ctx, &text); + GRN_OBJ_FIN(ctx, db); + } else { + success = GRN_FALSE; + } + grn_ctx_fin(ctx); + grn_fin(); + + return success ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_ddl.txt b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_ddl.txt new file mode 100644 index 00000000..f82763f8 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_ddl.txt @@ -0,0 +1,62 @@ +suggest向けDDL解説 +================== + +suggest機能で必要となるスキーマを説明する。なお、このドキュメントはsuggest機能完成とともにdocsディレクトリ内に移動される。 + +1. 概要 + + suggestデータベースには、学習過程でのみ必要になるテーブルと、 + 学習時およびサジェスト時に必要になるテーブルがあります。 + 前者を作業用テーブルと呼び、後者を学習結果テーブルと呼びます。 + +2. 学習結果テーブル + + 学習結果テーブルは二つのファクトテーブルと二つの語彙テーブルから構成されます。 + +2.1. 学習結果ファクトテーブル + +item: suggest候補として提示する個々の文字列を格納するテーブルです。 + + 以下のカラムを用意しています。 + + _key: itemテーブルの主キーは、サジェスト対象文字列そのものとなります。 + kana: 対象文字列の読み仮名を格納します。必須ではありませんが、セットされていればローマ字入力途中の文字列に対しても補完候補を提示可能となります。 + freq: ユーザクエリにおいて入力された回数を記録します。 + last: ユーザクエリに最後に入力された時刻を記録します。 + freq2: ユーザクエリにおいてサブミットされた回数を記録します。 + boost: 当該文字列の露出を制御します。正値を指定すれば露出が頻繁になり、-1を指定すれば、露出されません。 + buzz: 入力回数の差分を記録します。 + co: 共起する他の文字列を参照するための索引です。 + +pair: item間の共起関係を管理するテーブルです。 + + 以下のカラムを用意しています。 + + _kay: preおよびpostのIDを結合した数値が主キーとなります。 + pre: 事前に出現するアイティムです。 + post: 事後に出現するアイティムです。 + freq0: completeイベントにおける共起頻度を記録します。 + freq1: correctイベントにおける共起頻度を記録します。 + freq2: suggestイベントにおける共起頻度を記録します。 + +2.2. 学習結果語彙テーブル + + bigram: itemの主キーによって全文検索を行うための語彙表です。 + kana: itemのkanaカラムの入力補完を行うための語彙表です。 + +3. 作業用テーブル + + 作業用テーブルにはeventとsequenceの2テーブルがあります。 + +event: ユーザの個々のクエリイベントに対応します。レコードの追加のみが行われます。 + 適宜dropして再作成することが望ましいです。 + + type: submitの有無を記録します。 + time: イベントが発生した時刻を記録します。 + item: ユーザが入力した文字列を記録します。 + sequence: ユーザの識別子(cookie, ipアドレス等)を記録します。 + +sequence: 同一ユーザの一連の操作を記録します。レコードの追加のみが行われます。 + 適宜dropして再作成することが望ましいです。 + + events: 当該ユーザの操作履歴を記録します。 diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c new file mode 100644 index 00000000..4f542f21 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c @@ -0,0 +1,860 @@ +/* -*- c-basic-offset: 2 -*- */ +/* Copyright(C) 2010-2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +/* groonga origin headers */ +#include <grn_str.h> +#include <grn_msgpack.h> + +#include <stdio.h> +#include <signal.h> +#include <string.h> +#include <sys/types.h> +#include <sys/time.h> +#include <time.h> +#include <stdlib.h> +#include <unistd.h> +#include <err.h> + +#include <fcntl.h> +#include <sys/queue.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <netinet/in.h> +#include <sys/resource.h> + +#include "zmq_compatible.h" +#include <event.h> +#include <evhttp.h> +#include <groonga.h> +#include <pthread.h> + +#include "util.h" + +#define DEFAULT_PORT 8080 +#define DEFAULT_MAX_THREADS 8 + +#define CONST_STR_LEN(x) x, x ? sizeof(x) - 1 : 0 + +#define LISTEN_BACKLOG 756 +#define MIN_MAX_FDS 2048 +#define MAX_THREADS 128 /* max 256 */ + +typedef enum { + run_mode_none = 0, + run_mode_usage, + run_mode_daemon, + run_mode_error +} run_mode; + +#define RUN_MODE_MASK 0x007f +#define RUN_MODE_ENABLE_MAX_FD_CHECK 0x0080 + + +typedef struct { + grn_ctx *ctx; + grn_obj *db; + void *zmq_sock; + grn_obj cmd_buf; + grn_obj pass_through_parameters; + pthread_t thd; + uint32_t thread_id; + struct event_base *base; + struct evhttp *httpd; + struct event pulse; + const char *log_base_path; + FILE *log_file; + uint32_t log_count; + grn_bool request_reopen_log_file; +} thd_data; + +typedef struct { + const char *db_path; + const char *recv_endpoint; + pthread_t thd; + void *zmq_ctx; +} recv_thd_data; + +#define CMD_BUF_SIZE 1024 + +static thd_data threads[MAX_THREADS]; +static uint32_t default_max_threads = DEFAULT_MAX_THREADS; +static uint32_t max_threads; +static volatile sig_atomic_t loop = 1; +static grn_obj *db; +static uint32_t n_lines_per_log_file = 1000000; + +static int +suggest_result(grn_ctx *ctx, + struct evbuffer *res_buf, const char *types, const char *query, + const char *target_name, int frequency_threshold, + double conditional_probability_threshold, int limit, + grn_obj *cmd_buf, grn_obj *pass_through_parameters) +{ + if (target_name && types && query) { + GRN_BULK_REWIND(cmd_buf); + GRN_TEXT_PUTS(ctx, cmd_buf, "/d/suggest?table=item_"); + grn_text_urlenc(ctx, cmd_buf, target_name, strlen(target_name)); + GRN_TEXT_PUTS(ctx, cmd_buf, "&column=kana&types="); + grn_text_urlenc(ctx, cmd_buf, types, strlen(types)); + GRN_TEXT_PUTS(ctx, cmd_buf, "&query="); + grn_text_urlenc(ctx, cmd_buf, query, strlen(query)); + GRN_TEXT_PUTS(ctx, cmd_buf, "&frequency_threshold="); + grn_text_itoa(ctx, cmd_buf, frequency_threshold); + GRN_TEXT_PUTS(ctx, cmd_buf, "&conditional_probability_threshold="); + grn_text_ftoa(ctx, cmd_buf, conditional_probability_threshold); + GRN_TEXT_PUTS(ctx, cmd_buf, "&limit="); + grn_text_itoa(ctx, cmd_buf, limit); + if (GRN_TEXT_LEN(pass_through_parameters) > 0) { + GRN_TEXT_PUTS(ctx, cmd_buf, "&"); + GRN_TEXT_PUT(ctx, cmd_buf, + GRN_TEXT_VALUE(pass_through_parameters), + GRN_TEXT_LEN(pass_through_parameters)); + } + { + char *res; + int flags; + unsigned int res_len; + + grn_ctx_send(ctx, GRN_TEXT_VALUE(cmd_buf), GRN_TEXT_LEN(cmd_buf), 0); + grn_ctx_recv(ctx, &res, &res_len, &flags); + + evbuffer_add(res_buf, res, res_len); + return res_len; + } + } else { + evbuffer_add(res_buf, "{}", 2); + return 2; + } +} + +static void +log_send(struct evkeyvalq *output_headers, struct evbuffer *res_buf, + thd_data *thd, struct evkeyvalq *get_args) +{ + uint64_t millisec; + int frequency_threshold, limit; + double conditional_probability_threshold; + const char *callback, *types, *query, *client_id, *target_name, + *learn_target_name; + + GRN_BULK_REWIND(&(thd->pass_through_parameters)); + parse_keyval(thd->ctx, get_args, &query, &types, &client_id, &target_name, + &learn_target_name, &callback, &millisec, &frequency_threshold, + &conditional_probability_threshold, &limit, + &(thd->pass_through_parameters)); + + /* send data to learn client */ + if (thd->zmq_sock && millisec && client_id && query && learn_target_name) { + char c; + size_t l; + msgpack_packer pk; + msgpack_sbuffer sbuf; + int cnt, submit_flag = 0; + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + cnt = 4; + if (types && !strcmp(types, "submit")) { + cnt++; + types = NULL; + submit_flag = 1; + } + msgpack_pack_map(&pk, cnt); + + c = 'i'; + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); + l = strlen(client_id); + msgpack_pack_str(&pk, l); + msgpack_pack_str_body(&pk, client_id, l); + + c = 'q'; + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); + l = strlen(query); + msgpack_pack_str(&pk, l); + msgpack_pack_str_body(&pk, query, l); + + c = 's'; + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); + msgpack_pack_uint64(&pk, millisec); + + c = 'l'; + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); + l = strlen(learn_target_name); + msgpack_pack_str(&pk, l); + msgpack_pack_str_body(&pk, learn_target_name, l); + + if (submit_flag) { + c = 't'; + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); + msgpack_pack_true(&pk); + } + { + zmq_msg_t msg; + if (!zmq_msg_init_size(&msg, sbuf.size)) { + memcpy((void *)zmq_msg_data(&msg), sbuf.data, sbuf.size); + if (zmq_msg_send(&msg, thd->zmq_sock, 0) == -1) { + print_error("zmq_msg_send() error"); + } + zmq_msg_close(&msg); + } + } + msgpack_sbuffer_destroy(&sbuf); + } + /* make result */ + { + int content_length; + if (callback) { + evhttp_add_header(output_headers, + "Content-Type", "text/javascript; charset=UTF-8"); + content_length = strlen(callback); + evbuffer_add(res_buf, callback, content_length); + evbuffer_add(res_buf, "(", 1); + content_length += suggest_result(thd->ctx, + res_buf, types, query, target_name, + frequency_threshold, + conditional_probability_threshold, + limit, + &(thd->cmd_buf), + &(thd->pass_through_parameters)) + 3; + evbuffer_add(res_buf, ");", 2); + } else { + evhttp_add_header(output_headers, + "Content-Type", "application/json; charset=UTF-8"); + content_length = suggest_result(thd->ctx, + res_buf, types, query, target_name, + frequency_threshold, + conditional_probability_threshold, + limit, + &(thd->cmd_buf), + &(thd->pass_through_parameters)); + } + if (content_length >= 0) { +#define NUM_BUF_SIZE 16 + char num_buf[NUM_BUF_SIZE]; + grn_snprintf(num_buf, NUM_BUF_SIZE, NUM_BUF_SIZE, "%d", content_length); + evhttp_add_header(output_headers, "Content-Length", num_buf); +#undef NUM_BUF_SIZE + } + } +} + +static void +cleanup_httpd_thread(thd_data *thd) { + if (thd->log_file) { + fclose(thd->log_file); + } + if (thd->httpd) { + evhttp_free(thd->httpd); + } + if (thd->zmq_sock) { + zmq_close(thd->zmq_sock); + } + grn_obj_unlink(thd->ctx, &(thd->cmd_buf)); + grn_obj_unlink(thd->ctx, &(thd->pass_through_parameters)); + if (thd->ctx) { + grn_ctx_close(thd->ctx); + } + event_base_free(thd->base); +} + +static void +close_log_file(thd_data *thread) +{ + fclose(thread->log_file); + thread->log_file = NULL; + thread->request_reopen_log_file = GRN_FALSE; +} + +static void +generic_handler(struct evhttp_request *req, void *arg) +{ + struct evkeyvalq args; + thd_data *thd = arg; + + if (!loop) { + event_base_loopexit(thd->base, NULL); + return; + } + if (!req->uri) { return; } + + evhttp_parse_query(req->uri, &args); + { + struct evbuffer *res_buf; + if (!(res_buf = evbuffer_new())) { + err(1, "failed to create response buffer"); + } + + evhttp_add_header(req->output_headers, "Connection", "close"); + + log_send(req->output_headers, res_buf, thd, &args); + evhttp_send_reply(req, HTTP_OK, "OK", res_buf); + evbuffer_free(res_buf); + /* logging */ + { + if (thd->log_base_path) { + if (thd->log_file && thd->request_reopen_log_file) { + close_log_file(thd); + } + if (!thd->log_file) { + time_t n; + struct tm *t_st; + char p[PATH_MAX + 1]; + + time(&n); + t_st = localtime(&n); + + grn_snprintf(p, + PATH_MAX, + PATH_MAX, + "%s%04d%02d%02d%02d%02d%02d-%02d", + thd->log_base_path, + t_st->tm_year + 1900, + t_st->tm_mon + 1, + t_st->tm_mday, + t_st->tm_hour, + t_st->tm_min, + t_st->tm_sec, + thd->thread_id); + + if (!(thd->log_file = fopen(p, "a"))) { + print_error("cannot open log_file %s.", p); + } else { + thd->log_count = 0; + } + } + if (thd->log_file) { + fprintf(thd->log_file, "%s\n", req->uri); + thd->log_count++; + if (n_lines_per_log_file > 0 && + thd->log_count >= n_lines_per_log_file) { + close_log_file(thd); + } + } + } + } + } + evhttp_clear_headers(&args); +} + +static int +bind_socket(int port) +{ + int nfd; + if ((nfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + print_error("cannot open socket for http."); + return -1; + } else { + int r, one = 1; + struct sockaddr_in addr; + + r = setsockopt(nfd, SOL_SOCKET, SO_REUSEADDR, (char *)&one, sizeof(int)); + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + addr.sin_port = htons(port); + + if ((r = bind(nfd, (struct sockaddr *)&addr, sizeof(addr))) < 0) { + print_error("cannot bind socket for http."); + return r; + } + if ((r = listen(nfd, LISTEN_BACKLOG)) < 0) { + print_error("cannot listen socket for http."); + return r; + } + if ((r = fcntl(nfd, F_GETFL, 0)) < 0 || fcntl(nfd, F_SETFL, r | O_NONBLOCK) < 0 ) { + print_error("cannot fcntl socket for http."); + return -1; + } + return nfd; + } +} + +static void +signal_handler(int sig) +{ + loop = 0; +} + +static void +signal_reopen_log_file(int sig) +{ + uint32_t i; + + for (i = 0; i < max_threads; i++) { + threads[i].request_reopen_log_file = GRN_TRUE; + } +} + +void +timeout_handler(int fd, short events, void *arg) { + thd_data *thd = arg; + if (!loop) { + event_base_loopexit(thd->base, NULL); + } else { + struct timeval tv = {1, 0}; + evtimer_add(&(thd->pulse), &tv); + } +} + +static void * +dispatch(void *arg) +{ + event_base_dispatch((struct event_base *)arg); + return NULL; +} + +static void +msgpack2json(msgpack_object *o, grn_ctx *ctx, grn_obj *buf) +{ + switch (o->type) { + case MSGPACK_OBJECT_POSITIVE_INTEGER: + grn_text_ulltoa(ctx, buf, o->via.u64); + break; + case MSGPACK_OBJECT_STR: + grn_text_esc(ctx, buf, + MSGPACK_OBJECT_STR_PTR(o), + MSGPACK_OBJECT_STR_SIZE(o)); + break; + case MSGPACK_OBJECT_ARRAY: + GRN_TEXT_PUTC(ctx, buf, '['); + { + int i; + for (i = 0; i < o->via.array.size; i++) { + msgpack2json(o->via.array.ptr, ctx, buf); + } + } + GRN_TEXT_PUTC(ctx, buf, ']'); + break; + case MSGPACK_OBJECT_FLOAT: + grn_text_ftoa(ctx, buf, MSGPACK_OBJECT_FLOAT_VALUE(o)); + break; + default: + print_error("cannot handle this msgpack type."); + } +} + +static void +load_from_learner(msgpack_object *o, grn_ctx *ctx, grn_obj *cmd_buf) +{ + if (o->type == MSGPACK_OBJECT_MAP && o->via.map.size) { + msgpack_object_kv *kv; + msgpack_object *key; + msgpack_object *value; + kv = &(o->via.map.ptr[0]); + key = &(kv->key); + value = &(kv->val); + if (key->type == MSGPACK_OBJECT_STR && MSGPACK_OBJECT_STR_SIZE(key) == 6 && + !memcmp(MSGPACK_OBJECT_STR_PTR(key), CONST_STR_LEN("target"))) { + if (value->type == MSGPACK_OBJECT_STR) { + int i; + GRN_BULK_REWIND(cmd_buf); + GRN_TEXT_PUTS(ctx, cmd_buf, "load --table "); + GRN_TEXT_PUT(ctx, cmd_buf, + MSGPACK_OBJECT_STR_PTR(value), + MSGPACK_OBJECT_STR_SIZE(value)); + grn_ctx_send(ctx, GRN_TEXT_VALUE(cmd_buf), GRN_TEXT_LEN(cmd_buf), GRN_CTX_MORE); + grn_ctx_send(ctx, CONST_STR_LEN("["), GRN_CTX_MORE); + if (MSGPACK_OBJECT_STR_SIZE(value) > 5) { + if (!memcmp(MSGPACK_OBJECT_STR_PTR(value), CONST_STR_LEN("item_")) || + !memcmp(MSGPACK_OBJECT_STR_PTR(value), CONST_STR_LEN("pair_"))) { + char delim = '{'; + GRN_BULK_REWIND(cmd_buf); + for (i = 1; i < o->via.map.size; i++) { + GRN_TEXT_PUTC(ctx, cmd_buf, delim); + kv = &(o->via.map.ptr[i]); + msgpack2json(&(kv->key), ctx, cmd_buf); + GRN_TEXT_PUTC(ctx, cmd_buf, ':'); + msgpack2json(&(kv->val), ctx, cmd_buf); + delim = ','; + } + GRN_TEXT_PUTC(ctx, cmd_buf, '}'); + /* printf("msg: %.*s\n", GRN_TEXT_LEN(cmd_buf), GRN_TEXT_VALUE(cmd_buf)); */ + grn_ctx_send(ctx, GRN_TEXT_VALUE(cmd_buf), GRN_TEXT_LEN(cmd_buf), GRN_CTX_MORE); + } + } + grn_ctx_send(ctx, CONST_STR_LEN("]"), 0); + { + char *res; + int flags; + unsigned int res_len; + grn_ctx_recv(ctx, &res, &res_len, &flags); + } + } + } + } +} + +static void +recv_handler(grn_ctx *ctx, void *zmq_recv_sock, msgpack_zone *mempool, grn_obj *cmd_buf) +{ + zmq_msg_t msg; + + if (zmq_msg_init(&msg)) { + print_error("cannot init zmq message."); + } else { + if (zmq_msg_recv(&msg, zmq_recv_sock, 0) == -1) { + print_error("cannot recv zmq message."); + } else { + msgpack_object obj; + msgpack_unpack_return ret; + + ret = msgpack_unpack(zmq_msg_data(&msg), zmq_msg_size(&msg), NULL, mempool, &obj); + if (MSGPACK_UNPACK_SUCCESS == ret) { + load_from_learner(&obj, ctx, cmd_buf); + } else { + print_error("invalid recv data."); + } + msgpack_zone_clear(mempool); + } + zmq_msg_close(&msg); + } +} + +static void * +recv_from_learner(void *arg) +{ + void *zmq_recv_sock; + recv_thd_data *thd = arg; + + if ((zmq_recv_sock = zmq_socket(thd->zmq_ctx, ZMQ_SUB))) { + if (!zmq_connect(zmq_recv_sock, thd->recv_endpoint)) { + grn_ctx ctx; + if (!grn_ctx_init(&ctx, 0)) { + if ((!grn_ctx_use(&ctx, db))) { + msgpack_zone *mempool; + if ((mempool = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE))) { + grn_obj cmd_buf; + zmq_pollitem_t items[] = { + { zmq_recv_sock, 0, ZMQ_POLLIN, 0} + }; + GRN_TEXT_INIT(&cmd_buf, 0); + zmq_setsockopt(zmq_recv_sock, ZMQ_SUBSCRIBE, "", 0); + while (loop) { + zmq_poll(items, 1, 10000); + if (items[0].revents & ZMQ_POLLIN) { + recv_handler(&ctx, zmq_recv_sock, mempool, &cmd_buf); + } + } + grn_obj_unlink(&ctx, &cmd_buf); + msgpack_zone_free(mempool); + } else { + print_error("cannot create msgpack zone."); + } + /* db_close */ + } else { + print_error("error in grn_db_open() on recv thread."); + } + grn_ctx_fin(&ctx); + } else { + print_error("error in grn_ctx_init() on recv thread."); + } + } else { + print_error("cannot create recv zmq_socket."); + } + } else { + print_error("cannot connect zmq_socket."); + } + return NULL; +} + +static int +serve_threads(int nthreads, int port, const char *db_path, void *zmq_ctx, + const char *send_endpoint, const char *recv_endpoint, + const char *log_base_path) +{ + int nfd; + uint32_t i; + if ((nfd = bind_socket(port)) < 0) { + print_error("cannot bind socket. please check port number with netstat."); + return -1; + } + + for (i = 0; i < nthreads; i++) { + memset(&threads[i], 0, sizeof(threads[i])); + threads[i].request_reopen_log_file = GRN_FALSE; + if (!(threads[i].base = event_init())) { + print_error("error in event_init() on thread %d.", i); + } else { + if (!(threads[i].httpd = evhttp_new(threads[i].base))) { + print_error("error in evhttp_new() on thread %d.", i); + } else { + int r; + if ((r = evhttp_accept_socket(threads[i].httpd, nfd))) { + print_error("error in evhttp_accept_socket() on thread %d.", i); + } else { + if (send_endpoint) { + if (!(threads[i].zmq_sock = zmq_socket(zmq_ctx, ZMQ_PUB))) { + print_error("cannot create zmq_socket."); + } else if (zmq_connect(threads[i].zmq_sock, send_endpoint)) { + print_error("cannot connect zmq_socket."); + zmq_close(threads[i].zmq_sock); + threads[i].zmq_sock = NULL; + } else { + uint64_t hwm = 1; + zmq_setsockopt(threads[i].zmq_sock, ZMQ_SNDHWM, &hwm, sizeof(uint64_t)); + } + } else { + threads[i].zmq_sock = NULL; + } + if (!(threads[i].ctx = grn_ctx_open(0))) { + print_error("error in grn_ctx_open() on thread %d.", i); + } else if (grn_ctx_use(threads[i].ctx, db)) { + print_error("error in grn_db_open() on thread %d.", i); + } else { + GRN_TEXT_INIT(&(threads[i].cmd_buf), 0); + GRN_TEXT_INIT(&(threads[i].pass_through_parameters), 0); + threads[i].log_base_path = log_base_path; + threads[i].thread_id = i; + evhttp_set_gencb(threads[i].httpd, generic_handler, &threads[i]); + evhttp_set_timeout(threads[i].httpd, 10); + { + struct timeval tv = {1, 0}; + evtimer_set(&(threads[i].pulse), timeout_handler, &threads[i]); + evtimer_add(&(threads[i].pulse), &tv); + } + if ((r = pthread_create(&(threads[i].thd), NULL, dispatch, threads[i].base))) { + print_error("error in pthread_create() on thread %d.", i); + } + } + } + } + } + } + + /* recv thread from learner */ + if (recv_endpoint) { + recv_thd_data rthd; + rthd.db_path = db_path; + rthd.recv_endpoint = recv_endpoint; + rthd.zmq_ctx = zmq_ctx; + + if (pthread_create(&(rthd.thd), NULL, recv_from_learner, &rthd)) { + print_error("error in pthread_create() on thread %d.", i); + } + if (pthread_join(rthd.thd, NULL)) { + print_error("error in pthread_join() on thread %d.", i); + } + } else { + while (loop) { sleep(1); } + } + + /* join all httpd thread */ + for (i = 0; i < nthreads; i++) { + if (threads[i].thd) { + if (pthread_join(threads[i].thd, NULL)) { + print_error("error in pthread_join() on thread %d.", i); + } + } + cleanup_httpd_thread(&(threads[i])); + } + return 0; +} + +static uint32_t +get_core_number(void) +{ +#ifdef ACTUALLY_GET_CORE_NUMBER +#ifdef _SC_NPROCESSORS_CONF + return sysconf(_SC_NPROCESSORS_CONF); +#else /* _SC_NPROCESSORS_CONF */ + int n_processors; + size_t length = sizeof(n_processors); + int mib[] = {CTL_HW, HW_NCPU}; + if (sysctl(mib, sizeof(mib) / sizeof(mib[0]), + &n_processors, &length, NULL, 0) == 0 && + length == sizeof(n_processors) && + 0 < n_processors) { + return n_processors; + } else { + return 1; + } +#endif /* _SC_NPROCESSORS_CONF */ +#endif /* ACTUALLY_GET_CORE_NUMBER */ + return 0; +} + +static void +usage(FILE *output) +{ + fprintf( + output, + "Usage: groonga-suggest-httpd [options...] db_path\n" + "db_path:\n" + " specify groonga database path which is used for suggestion.\n" + "\n" + "options:\n" + " -p, --port <port number> : http server port number\n" + " (default: %d)\n" + /* + " --address <ip/hostname> : server address to listen\n" + " (default: %s)\n" + */ + " -c <thread number> : number of server threads\n" + " (deprecated. use --n-threads)\n" + " -t, --n-threads <thread number> : number of server threads\n" + " (default: %d)\n" + " -s, --send-endpoint <send endpoint> : send endpoint\n" + " (ex. tcp://example.com:1234)\n" + " -r, --receive-endpoint <receive endpoint> : receive endpoint\n" + " (ex. tcp://example.com:1235)\n" + " -l, --log-base-path <path prefix> : log path prefix\n" + " --n-lines-per-log-file <lines number> : number of lines in a log file\n" + " use 0 for disabling this\n" + " (default: %d)\n" + " -d, --daemon : daemonize\n" + " --disable-max-fd-check : disable max FD check on start\n" + " -h, --help : show this message\n", + DEFAULT_PORT, default_max_threads, n_lines_per_log_file); +} + +int +main(int argc, char **argv) +{ + int port_no = DEFAULT_PORT; + const char *max_threads_string = NULL, *port_string = NULL; + const char *address; + const char *send_endpoint = NULL, *recv_endpoint = NULL, *log_base_path = NULL; + const char *n_lines_per_log_file_string = NULL; + int n_processed_args, flags = RUN_MODE_ENABLE_MAX_FD_CHECK; + run_mode mode = run_mode_none; + + if (!(default_max_threads = get_core_number())) { + default_max_threads = DEFAULT_MAX_THREADS; + } + + /* parse options */ + { + static grn_str_getopt_opt opts[] = { + {'c', NULL, NULL, 0, GETOPT_OP_NONE}, /* deprecated */ + {'t', "n-threads", NULL, 0, GETOPT_OP_NONE}, + {'h', "help", NULL, run_mode_usage, GETOPT_OP_UPDATE}, + {'p', "port", NULL, 0, GETOPT_OP_NONE}, + {'\0', "bind-address", NULL, 0, GETOPT_OP_NONE}, /* not supported yet */ + {'s', "send-endpoint", NULL, 0, GETOPT_OP_NONE}, + {'r', "receive-endpoint", NULL, 0, GETOPT_OP_NONE}, + {'l', "log-base-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "n-lines-per-log-file", NULL, 0, GETOPT_OP_NONE}, + {'d', "daemon", NULL, run_mode_daemon, GETOPT_OP_UPDATE}, + {'\0', "disable-max-fd-check", NULL, RUN_MODE_ENABLE_MAX_FD_CHECK, + GETOPT_OP_OFF}, + {'\0', NULL, NULL, 0, 0} + }; + opts[0].arg = &max_threads_string; + opts[1].arg = &max_threads_string; + opts[3].arg = &port_string; + opts[4].arg = &address; + opts[5].arg = &send_endpoint; + opts[6].arg = &recv_endpoint; + opts[7].arg = &log_base_path; + opts[8].arg = &n_lines_per_log_file_string; + + n_processed_args = grn_str_getopt(argc, argv, opts, &flags); + } + + /* main */ + mode = (flags & RUN_MODE_MASK); + if (n_processed_args < 0 || + (argc - n_processed_args) != 1 || + mode == run_mode_error) { + usage(stderr); + return EXIT_FAILURE; + } else if (mode == run_mode_usage) { + usage(stdout); + return EXIT_SUCCESS; + } else { + grn_ctx ctx; + void *zmq_ctx; + + if (max_threads_string) { + max_threads = atoi(max_threads_string); + if (max_threads > MAX_THREADS) { + print_error("too many threads. limit to %d.", MAX_THREADS); + max_threads = MAX_THREADS; + } + } else { + max_threads = default_max_threads; + } + + if (port_string) { + port_no = atoi(port_string); + } + + if (flags & RUN_MODE_ENABLE_MAX_FD_CHECK) { + /* check environment */ + struct rlimit rlim; + if (!getrlimit(RLIMIT_NOFILE, &rlim)) { + if (rlim.rlim_max < MIN_MAX_FDS) { + print_error("too small max fds. %d required.", MIN_MAX_FDS); + return -1; + } + rlim.rlim_cur = rlim.rlim_cur; + setrlimit(RLIMIT_NOFILE, &rlim); + } + } + + if (n_lines_per_log_file_string) { + int64_t n_lines; + n_lines = grn_atoll(n_lines_per_log_file_string, + n_lines_per_log_file_string + strlen(n_lines_per_log_file_string), + NULL); + if (n_lines < 0) { + print_error("--n-lines-per-log-file must be >= 0: <%s>", + n_lines_per_log_file_string); + return(EXIT_FAILURE); + } + if (n_lines > UINT32_MAX) { + print_error("--n-lines-per-log-file must be <= %ld: <%s>", + UINT32_MAX, n_lines_per_log_file_string); + return(EXIT_FAILURE); + } + n_lines_per_log_file = (uint32_t)n_lines; + } + + if (mode == run_mode_daemon) { + daemonize(); + } + + grn_init(); + grn_ctx_init(&ctx, 0); + if ((db = grn_db_open(&ctx, argv[n_processed_args]))) { + if ((zmq_ctx = zmq_init(1))) { + signal(SIGTERM, signal_handler); + signal(SIGINT, signal_handler); + signal(SIGQUIT, signal_handler); + signal(SIGUSR1, signal_reopen_log_file); + + serve_threads(max_threads, port_no, argv[n_processed_args], zmq_ctx, + send_endpoint, recv_endpoint, log_base_path); + zmq_term(zmq_ctx); + } else { + print_error("cannot create zmq context."); + } + grn_obj_close(&ctx, db); + } else { + print_error("cannot open db."); + } + grn_ctx_fin(&ctx); + grn_fin(); + } + return 0; +} diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c new file mode 100644 index 00000000..8109ae7f --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c @@ -0,0 +1,843 @@ +/* -*- c-basic-offset: 2 -*- */ +/* Copyright(C) 2010-2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +/* for grn_str_getopt() */ +#include <grn_str.h> +#include <grn_msgpack.h> + +#include "zmq_compatible.h" +#include <stdio.h> +#include <signal.h> +#include <unistd.h> +#include <pthread.h> +#include <groonga.h> +#include <inttypes.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <dirent.h> + +#include "util.h" + +#include <evhttp.h> + +#define DEFAULT_RECV_ENDPOINT "tcp://*:1234" +#define DEFAULT_SEND_ENDPOINT "tcp://*:1235" +#define SEND_WAIT 1000 /* 0.001sec */ + +#define CONST_STR_LEN(x) x, x ? sizeof(x) - 1 : 0 + +typedef enum { + RUN_MODE_NONE = 0x00, + RUN_MODE_USAGE = 0x01, + RUN_MODE_DAEMON = 0x02, + RUN_MODE_ERROR = 0x04 +} run_mode; + +#define RUN_MODE_MASK 0x007f + +typedef struct { + const char *db_path; + const char *send_endpoint; + pthread_t thd; + void *zmq_ctx; +} send_thd_data; + +static volatile sig_atomic_t loop = 1; + +static void +load_to_groonga(grn_ctx *ctx, + grn_obj *buf, + const char *query, uint32_t query_len, + const char *client_id, uint32_t client_id_len, + const char *learn_target_name, uint32_t learn_target_name_len, + uint64_t millisec, + int submit) +{ + GRN_BULK_REWIND(buf); + GRN_TEXT_PUTS(ctx, buf, "load --table event_"); + GRN_TEXT_PUT(ctx, buf, learn_target_name, learn_target_name_len); + GRN_TEXT_PUTS(ctx, buf, " --each 'suggest_preparer(_id,type,item,sequence,time,pair_"); + GRN_TEXT_PUT(ctx, buf, learn_target_name, learn_target_name_len); + GRN_TEXT_PUTS(ctx, buf, ")'"); + grn_ctx_send(ctx, GRN_TEXT_VALUE(buf), GRN_TEXT_LEN(buf), GRN_CTX_MORE); + grn_ctx_send(ctx, CONST_STR_LEN("["), GRN_CTX_MORE); + + GRN_BULK_REWIND(buf); + GRN_TEXT_PUTS(ctx, buf, "{\"item\":"); + grn_text_esc(ctx, buf, query, query_len); + GRN_TEXT_PUTS(ctx, buf, ",\"sequence\":"); + grn_text_esc(ctx, buf, client_id, client_id_len); + GRN_TEXT_PUTS(ctx, buf, ",\"time\":"); + grn_text_ftoa(ctx, buf, (double)millisec / 1000); + if (submit) { + GRN_TEXT_PUTS(ctx, buf, ",\"type\":\"submit\"}"); + } else { + GRN_TEXT_PUTS(ctx, buf, "}"); + } + /* printf("%.*s\n", GRN_TEXT_LEN(buf), GRN_TEXT_VALUE(buf)); */ + grn_ctx_send(ctx, GRN_TEXT_VALUE(buf), GRN_TEXT_LEN(buf), GRN_CTX_MORE); + + grn_ctx_send(ctx, CONST_STR_LEN("]"), 0); + + { + char *res; + int flags; + unsigned int res_len; + grn_ctx_recv(ctx, &res, &res_len, &flags); + } +} + +void +load_to_multi_targets(grn_ctx *ctx, + grn_obj *buf, + const char *query, uint32_t query_len, + const char *client_id, uint32_t client_id_len, + const char *learn_target_names, + uint32_t learn_target_names_len, + uint64_t millisec, + int submit) +{ + if (millisec && query && client_id && learn_target_names) { + unsigned int tn_len; + const char *tn, *tnp, *tne; + tn = tnp = learn_target_names; + tne = learn_target_names + learn_target_names_len; + while (tnp <= tne) { + if (tnp == tne || *tnp == '|') { + tn_len = tnp - tn; + + /* + printf("sec: %" PRIu64 " query %.*s client_id: %.*s target: %.*s\n", + millisec, + query_len, query, + client_id_len, client_id, + tn_len, tn); + */ + load_to_groonga(ctx, buf, query, query_len, client_id, client_id_len, + tn, tn_len, millisec, submit); + + tn = ++tnp; + } else { + tnp++; + } + } + } +} + +#define PACK_KEY_FROM_ID(id) do { \ + int _k_len; \ + char _k_buf[GRN_TABLE_MAX_KEY_SIZE]; \ + _k_len = grn_table_get_key(ctx, ref_table, (id), _k_buf, GRN_TABLE_MAX_KEY_SIZE); \ + msgpack_pack_str(&pk, _k_len); \ + msgpack_pack_str_body(&pk, _k_buf, _k_len); \ +} while (0) + +#define PACK_MAP_ITEM(col_name) do { \ + grn_obj _v; \ + msgpack_pack_str(&pk, sizeof(#col_name) - 1); \ + msgpack_pack_str_body(&pk, #col_name, sizeof(#col_name) - 1); \ + switch (col_##col_name->header.type) { \ + case GRN_COLUMN_FIX_SIZE: \ + GRN_VALUE_FIX_SIZE_INIT(&_v, 0, grn_obj_get_range(ctx, col_##col_name)); \ + break; \ + case GRN_COLUMN_VAR_SIZE: \ + if ((col_##col_name->header.flags & GRN_OBJ_COLUMN_TYPE_MASK) == GRN_OBJ_COLUMN_VECTOR) { \ + GRN_VALUE_FIX_SIZE_INIT(&_v, GRN_OBJ_VECTOR, grn_obj_get_range(ctx, col_##col_name)); \ + } else { \ + GRN_VALUE_VAR_SIZE_INIT(&_v, 0, grn_obj_get_range(ctx, col_##col_name)); \ + } \ + break; \ + } \ + grn_obj_get_value(ctx, col_##col_name, rec_id, &_v); \ + \ + switch (_v.header.type) { \ + case GRN_BULK: \ + switch (_v.header.domain) { \ + case GRN_DB_SHORT_TEXT: \ + msgpack_pack_str(&pk, GRN_TEXT_LEN(&_v)); \ + msgpack_pack_str_body(&pk, GRN_TEXT_VALUE(&_v), GRN_TEXT_LEN(&_v)); \ + break; \ + case GRN_DB_INT32: \ + msgpack_pack_int32(&pk, GRN_INT32_VALUE(&_v)); \ + break; \ + case GRN_DB_UINT32: \ + msgpack_pack_uint32(&pk, GRN_UINT32_VALUE(&_v)); \ + break; \ + case GRN_DB_TIME: \ + msgpack_pack_double(&pk, (double)GRN_TIME_VALUE(&_v) / GRN_TIME_USEC_PER_SEC); \ + break; \ + default: /* ref. to ShortText key */ \ + PACK_KEY_FROM_ID(GRN_RECORD_VALUE(&_v)); \ + } \ + break; \ + case GRN_UVECTOR: /* ref.s to ShortText key */ \ + { \ + grn_id *_idv = (grn_id *)GRN_BULK_HEAD(&_v), *_idve = (grn_id *)GRN_BULK_CURR(&_v); \ + msgpack_pack_array(&pk, _idve - _idv); \ + for (; _idv < _idve; _idv++) { \ + PACK_KEY_FROM_ID(*_idv); \ + } \ + } \ + break; \ + default: \ + print_error("invalid groonga object type(%d) for msgpack.", _v.header.type); \ + msgpack_pack_nil(&pk); \ + break; \ + } \ + grn_obj_close(ctx, &_v); \ +} while (0) + +static int +zmq_send_to_httpd(void *zmq_send_sock, void *data, size_t size) +{ + zmq_msg_t msg; + if (!zmq_msg_init_size(&msg, size)) { + memcpy((void *)zmq_msg_data(&msg), data, size); + if (zmq_msg_send(&msg, zmq_send_sock, 0) == -1) { + print_error("zmq_send() error"); + return -1; + } + zmq_msg_close(&msg); + } else { + print_error("zmq_msg_init_size() error"); + } + return 0; +} + +static void +send_handler(void *zmq_send_sock, grn_ctx *ctx) +{ + grn_table_cursor *cur; + if ((cur = grn_table_cursor_open(ctx, grn_ctx_db(ctx), NULL, 0, NULL, 0, + 0, -1, 0))) { + grn_id table_id; + while (loop && (table_id = grn_table_cursor_next(ctx, cur)) != GRN_ID_NIL) { + grn_obj *table; + if ((table = grn_ctx_at(ctx, table_id))) { + int name_len; + char name_buf[GRN_TABLE_MAX_KEY_SIZE]; + + name_len = grn_obj_name(ctx, table, name_buf, + GRN_TABLE_MAX_KEY_SIZE); + + if (name_len > 5) { + if (table->header.type == GRN_TABLE_PAT_KEY && + !memcmp(name_buf, CONST_STR_LEN("item_"))) { + /* ["_key","ShortText"],["last","Time"],["kana","kana"],["freq2","Int32"],["freq","Int32"],["co","pair_all"],["buzz","Int32"],["boost","Int32"] */ + grn_obj *ref_table; + grn_table_cursor *tc; + grn_obj *col_last, *col_kana, *col_freq, *col_freq2, + *col_buzz, *col_boost; + + col_kana = grn_obj_column(ctx, table, CONST_STR_LEN("kana")); + col_freq = grn_obj_column(ctx, table, CONST_STR_LEN("freq")); + col_last = grn_obj_column(ctx, table, CONST_STR_LEN("last")); + col_boost = grn_obj_column(ctx, table, CONST_STR_LEN("boost")); + col_freq2 = grn_obj_column(ctx, table, CONST_STR_LEN("freq2")); + col_buzz = grn_obj_column(ctx, table, CONST_STR_LEN("buzz")); + + ref_table = grn_ctx_at(ctx, grn_obj_get_range(ctx, col_kana)); + + if ((tc = grn_table_cursor_open(ctx, table, NULL, 0, NULL, + 0, 0, -1, 0))) { + grn_id rec_id; + while (loop && (rec_id = grn_table_cursor_next(ctx, tc)) + != GRN_ID_NIL) { + char *key; + size_t key_len; + msgpack_packer pk; + msgpack_sbuffer sbuf; + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_map(&pk, 8); + + /* ["_key","ShortText"],["last","Time"],["kana","kana"],["freq2","Int32"],["freq","Int32"],["co","pair_all"],["buzz","Int32"],["boost","Int32"] */ + msgpack_pack_str(&pk, 6); + msgpack_pack_str_body(&pk, "target", strlen("target")); + msgpack_pack_str(&pk, name_len); + msgpack_pack_str_body(&pk, name_buf, name_len); + + msgpack_pack_str(&pk, 4); + msgpack_pack_str_body(&pk, + GRN_COLUMN_NAME_KEY, + GRN_COLUMN_NAME_KEY_LEN); + key_len = grn_table_cursor_get_key(ctx, tc, (void **)&key); + msgpack_pack_str(&pk, key_len); + msgpack_pack_str_body(&pk, key, key_len); + + PACK_MAP_ITEM(last); + PACK_MAP_ITEM(kana); + PACK_MAP_ITEM(freq); + PACK_MAP_ITEM(freq2); + PACK_MAP_ITEM(buzz); + PACK_MAP_ITEM(boost); + + zmq_send_to_httpd(zmq_send_sock, sbuf.data, sbuf.size); + + usleep(SEND_WAIT); + + msgpack_sbuffer_destroy(&sbuf); + } + grn_table_cursor_close(ctx, tc); + } + } else if (table->header.type == GRN_TABLE_HASH_KEY && + !memcmp(name_buf, CONST_STR_LEN("pair_"))) { + grn_obj *ref_table; + grn_table_cursor *tc; + grn_obj *col_pre, *col_post, *col_freq0, *col_freq1, *col_freq2; + + col_pre = grn_obj_column(ctx, table, CONST_STR_LEN("pre")); + col_post = grn_obj_column(ctx, table, CONST_STR_LEN("post")); + col_freq0 = grn_obj_column(ctx, table, CONST_STR_LEN("freq0")); + col_freq1 = grn_obj_column(ctx, table, CONST_STR_LEN("freq1")); + col_freq2 = grn_obj_column(ctx, table, CONST_STR_LEN("freq2")); + + ref_table = grn_ctx_at(ctx, grn_obj_get_range(ctx, col_pre)); + + if ((tc = grn_table_cursor_open(ctx, table, NULL, 0, NULL, + 0, 0, -1, 0))) { + grn_id rec_id; + while (loop && (rec_id = grn_table_cursor_next(ctx, tc)) + != GRN_ID_NIL) { + uint64_t *key; + msgpack_packer pk; + msgpack_sbuffer sbuf; + + /* skip freq0 == 0 && freq1 == 0 && freq2 == 0 */ + { + grn_obj f; + grn_obj_get_value(ctx, col_freq0, rec_id, &f); + if (!GRN_INT32_VALUE(&f)) { + grn_obj_get_value(ctx, col_freq1, rec_id, &f); + if (!GRN_INT32_VALUE(&f)) { + grn_obj_get_value(ctx, col_freq2, rec_id, &f); + if (!GRN_INT32_VALUE(&f)) { continue; } + } + } + } + + /* make pair_* message */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_map(&pk, 7); + /* ["_key","UInt64"],["pre","item_all"],["post","item_all"],["freq2","Int32"],["freq1","Int32"],["freq0","Int32"] */ + + msgpack_pack_str(&pk, 6); + msgpack_pack_str_body(&pk, "target", strlen("target")); + msgpack_pack_str(&pk, name_len); + msgpack_pack_str_body(&pk, name_buf, name_len); + + msgpack_pack_str(&pk, 4); + msgpack_pack_str_body(&pk, + GRN_COLUMN_NAME_KEY, + GRN_COLUMN_NAME_KEY_LEN); + grn_table_cursor_get_key(ctx, tc, (void **)&key); + msgpack_pack_uint64(&pk, *key); + + PACK_MAP_ITEM(pre); + PACK_MAP_ITEM(post); + PACK_MAP_ITEM(freq0); + PACK_MAP_ITEM(freq1); + PACK_MAP_ITEM(freq2); + + zmq_send_to_httpd(zmq_send_sock, sbuf.data, sbuf.size); + + usleep(SEND_WAIT); + + msgpack_sbuffer_destroy(&sbuf); + } + grn_table_cursor_close(ctx, tc); + } + } + } + grn_obj_unlink(ctx, table); + } + } + grn_table_cursor_close(ctx, cur); + } +} + +static void * +send_to_httpd(void *arg) +{ + send_thd_data *thd = arg; + void *zmq_send_sock; + if ((zmq_send_sock = zmq_socket(thd->zmq_ctx, ZMQ_PUB))) { + if (!zmq_bind(zmq_send_sock, thd->send_endpoint)) { + grn_ctx ctx; + if (!(grn_ctx_init(&ctx, 0))) { + grn_obj *db; + if ((db = grn_db_open(&ctx, thd->db_path))) { + uint64_t hwm = 1; + zmq_setsockopt(zmq_send_sock, ZMQ_SNDHWM, &hwm, sizeof(uint64_t)); + while (loop) { + send_handler(zmq_send_sock, &ctx); + } + grn_obj_close(&ctx, db); + } else { + print_error("error in grn_db_open() on send thread."); + } + grn_ctx_fin(&ctx); + } else { + print_error("error in grn_ctx_init() on send thread."); + } + } else { + print_error("cannot bind zmq_socket."); + } + } else { + print_error("cannot create zmq_socket."); + } + return NULL; +} + +static void +handle_msg(msgpack_object *obj, grn_ctx *ctx, grn_obj *buf) +{ + int submit_flag = 0; + uint64_t millisec = 0; + const char *query = NULL, + *client_id = NULL, *learn_target_names = NULL; + uint32_t query_len = 0, client_id_len = 0, learn_target_names_len = 0; + if (obj->type == MSGPACK_OBJECT_MAP) { + int i; + for (i = 0; i < obj->via.map.size; i++) { + msgpack_object_kv *kv; + msgpack_object *key; + msgpack_object *value; + kv = &(obj->via.map.ptr[i]); + key = &(kv->key); + value = &(kv->val); + if (key->type == MSGPACK_OBJECT_STR && MSGPACK_OBJECT_STR_SIZE(key) > 0) { + switch (MSGPACK_OBJECT_STR_PTR(key)[0]) { + case 'i': + if (value->type == MSGPACK_OBJECT_STR) { + client_id_len = MSGPACK_OBJECT_STR_SIZE(value); + client_id = MSGPACK_OBJECT_STR_PTR(value); + } + break; + case 'q': + if (value->type == MSGPACK_OBJECT_STR) { + query_len = MSGPACK_OBJECT_STR_SIZE(value); + query = MSGPACK_OBJECT_STR_PTR(value); + } + break; + case 'l': + if (value->type == MSGPACK_OBJECT_STR) { + learn_target_names_len = MSGPACK_OBJECT_STR_SIZE(value); + learn_target_names = MSGPACK_OBJECT_STR_PTR(value); + } + break; + case 's': + if (kv->val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + millisec = kv->val.via.u64; + } + break; + case 't': + if (kv->val.type == MSGPACK_OBJECT_BOOLEAN) { + submit_flag = (kv->val.via.boolean ? 1 : 0); + } + break; + default: + break; + } + } + } + load_to_multi_targets(ctx, buf, query, query_len, + client_id, client_id_len, + learn_target_names, learn_target_names_len, + millisec, submit_flag); + } +} + +static void +recv_event_loop(msgpack_zone *mempool, void *zmq_sock, grn_ctx *ctx) +{ + grn_obj buf; + zmq_pollitem_t items[] = { + { zmq_sock, 0, ZMQ_POLLIN, 0} + }; + GRN_TEXT_INIT(&buf, 0); + while (loop) { + zmq_poll(items, 1, 10000); + if (items[0].revents & ZMQ_POLLIN) { /* always true */ + zmq_msg_t msg; + if (zmq_msg_init(&msg)) { + print_error("cannot init zmq message."); + } else { + if (zmq_msg_recv(&msg, zmq_sock, 0) == -1) { + print_error("cannot recv zmq message."); + } else { + msgpack_object obj; + msgpack_unpack_return ret; + ret = msgpack_unpack(zmq_msg_data(&msg), zmq_msg_size(&msg), NULL, mempool, &obj); + if (MSGPACK_UNPACK_SUCCESS == ret) { + /* msgpack_object_print(stdout, obj); */ + handle_msg(&obj, ctx, &buf); + } + msgpack_zone_clear(mempool); + } + zmq_msg_close(&msg); + } + } + } + grn_obj_unlink(ctx, &buf); +} + +struct _suggest_log_file { + FILE *fp; + char *path; + uint64_t line; + /* datas from one line */ + int submit; + char *query; + uint64_t millisec; + char *client_id; + char *learn_target_name; + /* link list */ + struct _suggest_log_file *next; +}; +typedef struct _suggest_log_file suggest_log_file; + +#if 0 +static void +print_log_file_list(suggest_log_file *list) +{ + while (list) { + printf("fp:%p millisec:%" PRIu64 " next:%p\n", + list->fp, list->millisec, list->next); + list = list->next; + } +} +#endif + +static void +free_log_line_data(suggest_log_file *l) +{ + if (l->query) { + free(l->query); + l->query = NULL; + } + if (l->client_id) { + free(l->client_id); + l->client_id = NULL; + } + if (l->learn_target_name) { + free(l->learn_target_name); + l->learn_target_name = NULL; + } +} + +#define MAX_LOG_LENGTH 0x2000 + +static void +read_log_line(suggest_log_file **list) +{ + suggest_log_file *t = *list; + char line_buf[MAX_LOG_LENGTH]; + while (1) { + free_log_line_data(t); + if (fgets(line_buf, MAX_LOG_LENGTH, t->fp)) { + char *eol; + t->line++; + if ((eol = strrchr(line_buf, '\n'))) { + const char *query, *types, *client_id, *learn_target_name; + struct evkeyvalq get_args; + *eol = '\0'; + evhttp_parse_query(line_buf, &get_args); + parse_keyval(NULL, + &get_args, &query, &types, &client_id, NULL, + &learn_target_name, NULL, &(t->millisec), NULL, NULL, NULL, + NULL); + if (query && client_id && learn_target_name && t->millisec) { + t->query = evhttp_decode_uri(query); + t->submit = (types && !strcmp(types, "submit")); + t->client_id = evhttp_decode_uri(client_id); + t->learn_target_name = evhttp_decode_uri(learn_target_name); + evhttp_clear_headers(&get_args); + break; + } + print_error("invalid line path:%s line:%" PRIu64, + t->path, t->line); + evhttp_clear_headers(&get_args); + } else { + /* read until new line */ + while (1) { + int c = fgetc(t->fp); + if (c == '\n' || c == EOF) { break; } + } + } + } else { + /* terminate reading log */ + fclose(t->fp); + free(t->path); + *list = t->next; + free(t); + break; + } + } +} + +/* re-sorting by list->millisec asc with moving a head item. */ +static void +sort_log_file_list(suggest_log_file **list) +{ + suggest_log_file *p, *target; + target = *list; + if (!target || !target->next || target->millisec < target->next->millisec) { + return; + } + *list = target->next; + for (p = *list; p; p = p->next) { + if (!p->next || target->millisec > p->next->millisec) { + target->next = p->next; + p->next = target; + return; + } + } +} + +#define PATH_SEPARATOR '/' + +static suggest_log_file * +gather_log_file(const char *dir_path, unsigned int dir_path_len) +{ + DIR *dir; + struct dirent *dirent; + char path[PATH_MAX + 1]; + suggest_log_file *list = NULL; + if (!(dir = opendir(dir_path))) { + print_error("cannot open log directory."); + return NULL; + } + memcpy(path, dir_path, dir_path_len); + path[dir_path_len] = PATH_SEPARATOR; + while ((dirent = readdir(dir))) { + struct stat fstat; + unsigned int d_namlen, path_len; + if (*(dirent->d_name) == '.' && ( + dirent->d_name[1] == '\0' || + (dirent->d_name[1] == '.' && dirent->d_name[2] == '\0'))) { + continue; + } + d_namlen = strlen(dirent->d_name); + path_len = dir_path_len + 1 + d_namlen; + if (dir_path_len + d_namlen >= PATH_MAX) { continue; } + memcpy(path + dir_path_len + 1, dirent->d_name, d_namlen); + path[path_len] = '\0'; + lstat(path, &fstat); + if (S_ISDIR(fstat.st_mode)) { + gather_log_file(path, path_len); + } else { + suggest_log_file *p = calloc(1, sizeof(suggest_log_file)); + if (!(p->fp = fopen(path, "r"))) { + free(p); + } else { + if (list) { + p->next = list; + } + p->path = strdup(path); + list = p; + read_log_line(&list); + sort_log_file_list(&list); + } + } + /* print_log_file_list(list); */ + } + return list; +} + +static void +load_log(grn_ctx *ctx, const char *log_dir_name) +{ + grn_obj buf; + suggest_log_file *list; + GRN_TEXT_INIT(&buf, 0); + list = gather_log_file(log_dir_name, strlen(log_dir_name)); + while (list) { + /* + printf("file:%s line:%" PRIu64 " query:%s millisec:%" PRIu64 "\n", + list->path, list->line, list->query, list->millisec); + */ + load_to_multi_targets(ctx, &buf, + list->query, strlen(list->query), + list->client_id, strlen(list->client_id), + list->learn_target_name, strlen(list->learn_target_name), + list->millisec, + list->submit); + read_log_line(&list); + sort_log_file_list(&list); + } + grn_obj_close(ctx, &buf); +} + +static void +usage(FILE *output) +{ + fprintf(output, + "Usage: groonga-suggest-learner [options...] db_path\n" + "options:\n" + " -r <recv endpoint>: recv endpoint (default: %s)\n" + " --receive-endpoint <recv endpoint>\n" + "\n" + " -s <send endpoint>: send endpoint (default: %s)\n" + " --send-endpoint <send endpoint>\n" + "\n" + " -l <log directory>: load from log files made on webserver.\n" + " --log-base-path <log directory>\n" + "\n" + " --log-path <path> : output logs to <path>\n" + " --log-level <level> : set log level to <level> (default: %d)\n" + " -d, --daemon : daemonize\n", + DEFAULT_RECV_ENDPOINT, DEFAULT_SEND_ENDPOINT, + GRN_LOG_DEFAULT_LEVEL); +} + +static void +signal_handler(int sig) +{ + loop = 0; +} + +int +main(int argc, char **argv) +{ + run_mode mode = RUN_MODE_NONE; + int n_processed_args; + const char *recv_endpoint = DEFAULT_RECV_ENDPOINT; + const char *send_endpoint = DEFAULT_SEND_ENDPOINT; + const char *log_base_path = NULL; + const char *db_path = NULL; + + /* parse options */ + { + int flags = mode; + const char *log_path = NULL; + const char *log_level = NULL; + static grn_str_getopt_opt opts[] = { + {'r', "receive-endpoint", NULL, 0, GETOPT_OP_NONE}, + {'s', "send-endpoint", NULL, 0, GETOPT_OP_NONE}, + {'l', "log-base-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "log-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "log-level", NULL, 0, GETOPT_OP_NONE}, + {'d', "daemon", NULL, RUN_MODE_DAEMON, GETOPT_OP_UPDATE}, + {'h', "help", NULL, RUN_MODE_USAGE, GETOPT_OP_UPDATE}, + {'\0', NULL, NULL, 0, 0} + }; + opts[0].arg = &recv_endpoint; + opts[1].arg = &send_endpoint; + opts[2].arg = &log_base_path; + opts[3].arg = &log_path; + opts[4].arg = &log_level; + + n_processed_args = grn_str_getopt(argc, argv, opts, &flags); + + if (log_path) { + grn_default_logger_set_path(log_path); + } + + if (log_level) { + const char * const end = log_level + strlen(log_level); + const char *rest = NULL; + const int value = grn_atoi(log_level, end, &rest); + if (end != rest || value < 0 || value > 9) { + fprintf(stderr, "invalid log level: <%s>\n", log_level); + return EXIT_FAILURE; + } + grn_default_logger_set_max_level(value); + } + + mode = (flags & RUN_MODE_MASK); + + if (mode & RUN_MODE_USAGE) { + usage(stdout); + return EXIT_SUCCESS; + } + + if ((n_processed_args < 0) || + (argc - n_processed_args) != 1) { + usage(stderr); + } + + db_path = argv[n_processed_args]; + } + + /* main */ + { + grn_ctx *ctx; + msgpack_zone *mempool; + + if (mode == RUN_MODE_DAEMON) { + daemonize(); + } + + grn_init(); + + ctx = grn_ctx_open(0); + if (!(grn_db_open(ctx, db_path))) { + print_error("cannot open database."); + } else { + if (log_base_path) { + /* loading log mode */ + load_log(ctx, log_base_path); + } else { + /* zeromq/msgpack recv mode */ + if (!(mempool = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE))) { + print_error("cannot create msgpack zone."); + } else { + void *zmq_ctx, *zmq_recv_sock; + if (!(zmq_ctx = zmq_init(1))) { + print_error("cannot create zmq context."); + } else { + if (!(zmq_recv_sock = zmq_socket(zmq_ctx, ZMQ_SUB))) { + print_error("cannot create zmq_socket."); + } else if (zmq_bind(zmq_recv_sock, recv_endpoint)) { + print_error("cannot bind zmq_socket."); + } else { + send_thd_data thd; + + signal(SIGTERM, signal_handler); + signal(SIGINT, signal_handler); + signal(SIGQUIT, signal_handler); + + zmq_setsockopt(zmq_recv_sock, ZMQ_SUBSCRIBE, "", 0); + thd.db_path = db_path; + thd.send_endpoint = send_endpoint; + thd.zmq_ctx = zmq_ctx; + + if (pthread_create(&(thd.thd), NULL, send_to_httpd, &thd)) { + print_error("error in pthread_create() for sending datas."); + } + recv_event_loop(mempool, zmq_recv_sock, ctx); + if (pthread_join(thd.thd, NULL)) { + print_error("error in pthread_join() for waiting completion of sending data."); + } + } + zmq_term(zmq_ctx); + } + msgpack_zone_free(mempool); + } + } + } + grn_obj_close(ctx, grn_ctx_db(ctx)); + grn_ctx_fin(ctx); + grn_fin(); + } + return 0; +} diff --git a/storage/mroonga/vendor/groonga/src/suggest/httpd_sources.am b/storage/mroonga/vendor/groonga/src/suggest/httpd_sources.am new file mode 100644 index 00000000..a09328d5 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/httpd_sources.am @@ -0,0 +1,3 @@ +groonga_suggest_httpd_SOURCES = \ + groonga_suggest_httpd.c \ + zmq_compatible.h diff --git a/storage/mroonga/vendor/groonga/src/suggest/learner_sources.am b/storage/mroonga/vendor/groonga/src/suggest/learner_sources.am new file mode 100644 index 00000000..03b49087 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/learner_sources.am @@ -0,0 +1,3 @@ +groonga_suggest_learner_SOURCES = \ + groonga_suggest_learner.c \ + zmq_compatible.h diff --git a/storage/mroonga/vendor/groonga/src/suggest/util.c b/storage/mroonga/vendor/groonga/src/suggest/util.c new file mode 100644 index 00000000..e455a257 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/util.c @@ -0,0 +1,215 @@ +/* -*- c-basic-offset: 2 -*- */ +/* Copyright(C) 2010- Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdint.h> +#include <stdarg.h> +#include <unistd.h> +#include <sys/wait.h> +#include <sys/queue.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include "util.h" + +#define DEFAULT_FREQUENCY_THRESHOLD 100 +#define DEFAULT_CONDITIONAL_PROBABILITY_THRESHOLD 0.2 + +int +print_error(const char *format, ...) +{ + int r; + va_list l; + + va_start(l, format); + vfprintf(stderr, format, l); + r = fprintf(stderr, "\n"); + fflush(stderr); + va_end(l); + + return r; +} + +int +daemonize(void) +{ + pid_t pid; + + switch (fork()) { + case 0: + break; + case -1: + print_error("fork failed."); + return -1; + default: + wait(NULL); + _exit(0); + } + switch ((pid = fork())) { + case 0: + break; + case -1: + perror("fork"); + return -1; + default: + fprintf(stderr, "%d\n", pid); + _exit(0); + } + { + int null_fd = open("/dev/null", O_RDWR, 0); + if (null_fd != -1) { + dup2(null_fd, 0); + dup2(null_fd, 1); + dup2(null_fd, 2); + if (null_fd > 2) { close(null_fd); } + } + } + return 1; +} + +static uint64_t +atouint64_t(const char *s) +{ + uint64_t r; + for (r = 0; *s; s++) { + r *= 10; + r += (*s - '0'); + } + return r; +} + +void +parse_keyval(grn_ctx *ctx, + struct evkeyvalq *get_args, + const char **query, const char **types, + const char **client_id, const char **target_name, + const char **learn_target_name, + const char **callback, + uint64_t *millisec, + int *frequency_threshold, + double *conditional_probability_threshold, + int *limit, + grn_obj *pass_through_parameters) +{ + struct evkeyval *get; + + if (query) { *query = NULL; } + if (types) { *types = NULL; } + if (client_id) { *client_id = NULL; } + if (target_name) { *target_name = NULL; } + if (learn_target_name) { *learn_target_name = NULL; } + if (callback) { *callback = NULL; } + if (millisec) { *millisec = 0; } + if (frequency_threshold) { + *frequency_threshold = DEFAULT_FREQUENCY_THRESHOLD; + } + if (conditional_probability_threshold) { + *conditional_probability_threshold = DEFAULT_CONDITIONAL_PROBABILITY_THRESHOLD; + } + if (limit) { *limit = -1; } + + TAILQ_FOREACH(get, get_args, next) { + grn_bool is_pass_through_parameter = GRN_FALSE; + size_t key_length; + + key_length = strlen(get->key); + switch (key_length) { + case 0: + break; + case 1: + switch(get->key[0]) { + case 'q': + if (query) { + *query = get->value; + } + break; + case 't': + /* TODO: check types */ + if (types) { + *types = get->value; + } + break; + case 'i': + if (client_id) { + *client_id = get->value; + } + break; + case 's': + if (millisec) { + *millisec = atouint64_t(get->value); + } + break; + case 'n': + /* TODO: check target_name */ + if (target_name) { + *target_name = get->value; + } + break; + case 'l': + if (learn_target_name) { + *learn_target_name = get->value; + } + break; + case 'h': + if (frequency_threshold) { + *frequency_threshold = atoi(get->value); + } + break; + case 'p': + if (conditional_probability_threshold) { + *conditional_probability_threshold = strtod(get->value, NULL); + } + break; + case 'm': + if (limit) { + *limit = atoi(get->value); + } + break; + default: + is_pass_through_parameter = GRN_TRUE; + break; + } + break; + default: + switch (get->key[0]) { + case 'c': + if (!strcmp(get->key, "callback")) { + if (callback) { + *callback = get->value; + } + } else { + is_pass_through_parameter = GRN_TRUE; + } + break; + default: + is_pass_through_parameter = GRN_TRUE; + } + } + + if (is_pass_through_parameter && pass_through_parameters) { + if (GRN_TEXT_LEN(pass_through_parameters) > 0) { + GRN_TEXT_PUTS(ctx, pass_through_parameters, "&"); + } + grn_text_urlenc(ctx, pass_through_parameters, get->key, strlen(get->key)); + GRN_TEXT_PUTS(ctx, pass_through_parameters, "="); + grn_text_urlenc(ctx, pass_through_parameters, + get->value, strlen(get->value)); + } + } +} diff --git a/storage/mroonga/vendor/groonga/src/suggest/util.h b/storage/mroonga/vendor/groonga/src/suggest/util.h new file mode 100644 index 00000000..eb36edfd --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/util.h @@ -0,0 +1,40 @@ +/* -*- c-basic-offset: 2 -*- */ +/* Copyright(C) 2010- Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ +#ifndef GRN_SUGGEST_UTIL_H +#define GRN_SUGGEST_UTIL_H + +#include <sys/queue.h> +#include <event.h> +#include <stdint.h> + +#include <groonga.h> + +int print_error(const char *format, ...); +int daemonize(void); +void parse_keyval(grn_ctx *ctx, + struct evkeyvalq *get_args, + const char **query, const char **types, + const char **client_id, const char **target_name, + const char **learn_target_name, + const char **callback, + uint64_t *millisec, + int *frequency_threshold, + double *conditional_probability_threshold, + int *limit, + grn_obj *pass_through_parameters); + +#endif /* GRN_SUGGEST_UTIL_H */ diff --git a/storage/mroonga/vendor/groonga/src/suggest/util_sources.am b/storage/mroonga/vendor/groonga/src/suggest/util_sources.am new file mode 100644 index 00000000..d4b74860 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/util_sources.am @@ -0,0 +1,3 @@ +libutil_la_SOURCES = \ + util.c \ + util.h diff --git a/storage/mroonga/vendor/groonga/src/suggest/zmq_compatible.h b/storage/mroonga/vendor/groonga/src/suggest/zmq_compatible.h new file mode 100644 index 00000000..e4897f68 --- /dev/null +++ b/storage/mroonga/vendor/groonga/src/suggest/zmq_compatible.h @@ -0,0 +1,33 @@ +/* -*- c-basic-offset: 2 -*- */ +/* Copyright(C) 2013 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*/ +#ifndef GRN_SUGGEST_ZMQ_COMPATIBLE_H +#define GRN_SUGGEST_ZMQ_COMPATIBLE_H + +#include <zmq.h> + +#ifndef ZMQ_SNDHWM +# define ZMQ_SNDHWM ZMQ_HWM +#endif + +#if ZMQ_VERSION_MAJOR == 2 +# define zmq_msg_send(message, socket, flags) \ + zmq_send((socket), (message), (flags)) +# define zmq_msg_recv(message, socket, flags) \ + zmq_recv((socket), (message), (flags)) +#endif + +#endif /* GRN_SUGGEST_ZMQ_COMPATIBLE_H */ |