Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 372 insertions, 0 deletions

diff --git a/src/plugins/fts-solr/solr-response.c b/src/plugins/fts-solr/solr-response.c
new file mode 100644
index 0000000..65a6a1f
--- /dev/null
+++ b/src/plugins/fts-solr/solr-response.c
@@ -0,0 +1,372 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "str.h"
+#include "istream.h"
+#include "solr-response.h"
+
+#include <expat.h>
+
+#define MAX_VALUE_LEN 2048
+
+enum solr_xml_response_state {
+	SOLR_XML_RESPONSE_STATE_ROOT,
+	SOLR_XML_RESPONSE_STATE_RESPONSE,
+	SOLR_XML_RESPONSE_STATE_RESULT,
+	SOLR_XML_RESPONSE_STATE_DOC,
+	SOLR_XML_RESPONSE_STATE_CONTENT
+};
+
+enum solr_xml_content_state {
+	SOLR_XML_CONTENT_STATE_NONE = 0,
+	SOLR_XML_CONTENT_STATE_UID,
+	SOLR_XML_CONTENT_STATE_SCORE,
+	SOLR_XML_CONTENT_STATE_MAILBOX,
+	SOLR_XML_CONTENT_STATE_NAMESPACE,
+	SOLR_XML_CONTENT_STATE_UIDVALIDITY,
+	SOLR_XML_CONTENT_STATE_ERROR
+};
+
+struct solr_response_parser {
+	XML_Parser xml_parser;
+	struct istream *input;
+
+	enum solr_xml_response_state state;
+	enum solr_xml_content_state content_state;
+	int depth;
+	string_t *buffer;
+
+	uint32_t uid, uidvalidity;
+	float score;
+	char *mailbox, *ns;
+
+	pool_t result_pool;
+	/* box_id -> solr_result */
+	HASH_TABLE(char *, struct solr_result *) mailboxes;
+	ARRAY(struct solr_result *) results;
+
+	bool xml_failed:1;
+};
+
+static int
+solr_xml_parse(struct solr_response_parser *parser,
+	       const void *data, size_t size, bool done)
+{
+	enum XML_Error err;
+	int line, col;
+
+	if (parser->xml_failed)
+		return -1;
+
+	if (XML_Parse(parser->xml_parser, data, size, done ? 1 : 0) != 0)
+		return 0;
+
+	err = XML_GetErrorCode(parser->xml_parser);
+	if (err != XML_ERROR_FINISHED) {
+		line = XML_GetCurrentLineNumber(parser->xml_parser);
+		col = XML_GetCurrentColumnNumber(parser->xml_parser);
+		i_error("fts_solr: Invalid XML input at %d:%d: %s "
+			"(near: %.*s)", line, col, XML_ErrorString(err),
+			(int)I_MIN(size, 128), (const char *)data);
+		parser->xml_failed = TRUE;
+		return -1;
+	}
+	return 0;
+}
+
+static const char *attrs_get_name(const char **attrs)
+{
+	for (; *attrs != NULL; attrs += 2) {
+		if (strcmp(attrs[0], "name") == 0)
+			return attrs[1];
+	}
+	return "";
+}
+
+static void
+solr_lookup_xml_start(void *context, const char *name, const char **attrs)
+{
+	struct solr_response_parser *parser = context;
+	const char *name_attr;
+
+	i_assert(parser->depth >= (int)parser->state);
+
+	parser->depth++;
+	if (parser->depth - 1 > (int)parser->state) {
+		/* skipping over unwanted elements */
+		return;
+	}
+
+	str_truncate(parser->buffer, 0);
+
+	/* response -> result -> doc */
+	switch (parser->state) {
+	case SOLR_XML_RESPONSE_STATE_ROOT:
+		if (strcmp(name, "response") == 0)
+			parser->state++;
+		break;
+	case SOLR_XML_RESPONSE_STATE_RESPONSE:
+		if (strcmp(name, "result") == 0)
+			parser->state++;
+		break;
+	case SOLR_XML_RESPONSE_STATE_RESULT:
+		if (strcmp(name, "doc") == 0) {
+			parser->state++;
+			parser->uid = 0;
+			parser->score = 0;
+			i_free_and_null(parser->mailbox);
+			i_free_and_null(parser->ns);
+			parser->uidvalidity = 0;
+		}
+		break;
+	case SOLR_XML_RESPONSE_STATE_DOC:
+		name_attr = attrs_get_name(attrs);
+		if (strcmp(name_attr, "uid") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_UID;
+		else if (strcmp(name_attr, "score") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_SCORE;
+		else if (strcmp(name_attr, "box") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_MAILBOX;
+		else if (strcmp(name_attr, "ns") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_NAMESPACE;
+		else if (strcmp(name_attr, "uidv") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_UIDVALIDITY;
+		else
+			break;
+		parser->state++;
+		break;
+	case SOLR_XML_RESPONSE_STATE_CONTENT:
+		break;
+	}
+}
+
+static struct solr_result *
+solr_result_get(struct solr_response_parser *parser, const char *box_id)
+{
+	struct solr_result *result;
+	char *box_id_dup;
+
+	result = hash_table_lookup(parser->mailboxes, box_id);
+	if (result != NULL)
+		return result;
+
+	box_id_dup = p_strdup(parser->result_pool, box_id);
+	result = p_new(parser->result_pool, struct solr_result, 1);
+	result->box_id = box_id_dup;
+	p_array_init(&result->uids, parser->result_pool, 32);
+	p_array_init(&result->scores, parser->result_pool, 32);
+	hash_table_insert(parser->mailboxes, box_id_dup, result);
+	array_push_back(&parser->results, &result);
+	return result;
+}
+
+static int solr_lookup_add_doc(struct solr_response_parser *parser)
+{
+	struct fts_score_map *score;
+	struct solr_result *result;
+	const char *box_id;
+
+	if (parser->uid == 0) {
+		i_error("fts_solr: uid missing from inside doc");
+		return -1;
+	}
+
+	if (parser->mailbox == NULL) {
+		/* looking up from a single mailbox only */
+		box_id = "";
+	} else if (parser->uidvalidity != 0) {
+		/* old style lookup */
+		string_t *str = t_str_new(64);
+		str_printfa(str, "%u\001", parser->uidvalidity);
+		str_append(str, parser->mailbox);
+		if (parser->ns != NULL)
+			str_printfa(str, "\001%s", parser->ns);
+		box_id = str_c(str);
+	} else {
+		/* new style lookup */
+		box_id = parser->mailbox;
+	}
+	result = solr_result_get(parser, box_id);
+
+	if (seq_range_array_add(&result->uids, parser->uid)) {
+		/* duplicate result */
+	} else if (parser->score != 0) {
+		score = array_append_space(&result->scores);
+		score->uid = parser->uid;
+		score->score = parser->score;
+	}
+	return 0;
+}
+
+static void solr_lookup_xml_end(void *context, const char *name ATTR_UNUSED)
+{
+	struct solr_response_parser *parser = context;
+	string_t *buf = parser->buffer;
+	int ret;
+
+	switch (parser->content_state) {
+	case SOLR_XML_CONTENT_STATE_NONE:
+		break;
+	case SOLR_XML_CONTENT_STATE_UID:
+		if (str_to_uint32(str_c(buf), &parser->uid) < 0 ||
+		    parser->uid == 0) {
+			i_error("fts_solr: received invalid uid '%s'",
+				str_c(buf));
+			parser->content_state = SOLR_XML_CONTENT_STATE_ERROR;
+		}
+		break;
+	case SOLR_XML_CONTENT_STATE_SCORE:
+		parser->score = strtod(str_c(buf), NULL);
+		break;
+	case SOLR_XML_CONTENT_STATE_MAILBOX:
+		parser->mailbox = i_strdup(str_c(buf));
+		break;
+	case SOLR_XML_CONTENT_STATE_NAMESPACE:
+		parser->ns = i_strdup(str_c(buf));
+		break;
+	case SOLR_XML_CONTENT_STATE_UIDVALIDITY:
+		if (str_to_uint32(str_c(buf), &parser->uidvalidity) < 0)
+			i_error("fts_solr: received invalid uidvalidity");
+		break;
+	case SOLR_XML_CONTENT_STATE_ERROR:
+		return;
+	}
+
+	i_assert(parser->depth >= (int)parser->state);
+
+	if (parser->state == SOLR_XML_RESPONSE_STATE_CONTENT &&
+	    parser->content_state == SOLR_XML_CONTENT_STATE_MAILBOX &&
+	    parser->mailbox == NULL) {
+		/* mailbox is namespace prefix */
+		parser->mailbox = i_strdup("");
+	}
+
+	if (parser->depth == (int)parser->state) {
+		ret = 0;
+		if (parser->state == SOLR_XML_RESPONSE_STATE_DOC) {
+			T_BEGIN {
+				ret = solr_lookup_add_doc(parser);
+			} T_END;
+		}
+		parser->state--;
+		if (ret < 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_ERROR;
+		else
+			parser->content_state = SOLR_XML_CONTENT_STATE_NONE;
+	}
+	parser->depth--;
+}
+
+static void solr_lookup_xml_data(void *context, const char *str, int len)
+{
+	struct solr_response_parser *parser = context;
+
+	switch (parser->content_state) {
+	case SOLR_XML_CONTENT_STATE_NONE:
+	case SOLR_XML_CONTENT_STATE_ERROR:
+		/* ignore element data */
+		return;
+	case SOLR_XML_CONTENT_STATE_UID:
+	case SOLR_XML_CONTENT_STATE_SCORE:
+	case SOLR_XML_CONTENT_STATE_MAILBOX:
+	case SOLR_XML_CONTENT_STATE_NAMESPACE:
+	case SOLR_XML_CONTENT_STATE_UIDVALIDITY:
+		break;
+	}
+
+	if (str_len(parser->buffer) + len > MAX_VALUE_LEN) {
+		i_error("fts_solr: XML element data length out of range");
+		parser->content_state = SOLR_XML_CONTENT_STATE_ERROR;
+		return;
+	}
+
+	str_append_data(parser->buffer, str, len);
+}
+
+struct solr_response_parser *
+solr_response_parser_init(pool_t result_pool, struct istream *input)
+{
+	struct solr_response_parser *parser;
+
+	parser = i_new(struct solr_response_parser, 1);
+
+	parser->xml_parser = XML_ParserCreate("UTF-8");
+	if (parser->xml_parser == NULL) {
+		i_fatal_status(FATAL_OUTOFMEM,
+			       "fts_solr: Failed to allocate XML parser");
+	}
+
+	parser->buffer = str_new(default_pool, 256);
+	hash_table_create(&parser->mailboxes, default_pool, 0,
+			  str_hash, strcmp);
+
+	parser->result_pool = result_pool;
+	pool_ref(result_pool);
+	p_array_init(&parser->results, result_pool, 32);
+
+	parser->input = input;
+	i_stream_ref(input);
+
+	parser->xml_failed = FALSE;
+	XML_SetElementHandler(parser->xml_parser,
+			      solr_lookup_xml_start, solr_lookup_xml_end);
+	XML_SetCharacterDataHandler(parser->xml_parser, solr_lookup_xml_data);
+	XML_SetUserData(parser->xml_parser, parser);
+
+	return parser;
+}
+
+void solr_response_parser_deinit(struct solr_response_parser **_parser)
+{
+	struct solr_response_parser *parser = *_parser;
+
+	*_parser = NULL;
+
+	if (parser == NULL)
+		return;
+
+	str_free(&parser->buffer);
+	hash_table_destroy(&parser->mailboxes);
+	XML_ParserFree(parser->xml_parser);
+	i_stream_unref(&parser->input);
+	pool_unref(&parser->result_pool);
+	i_free(parser);
+}
+
+int solr_response_parse(struct solr_response_parser *parser,
+			struct solr_result ***box_results_r)
+{
+	const unsigned char *data;
+	size_t size;
+	int stream_errno, ret;
+
+	i_assert(parser->input != NULL);
+	i_zero(box_results_r);
+
+	/* read payload */
+	while ((ret = i_stream_read_more(parser->input, &data, &size)) > 0) {
+		(void)solr_xml_parse(parser, data, size, FALSE);
+		i_stream_skip(parser->input, size);
+	}
+
+	if (ret == 0) {
+		/* we will be called again for more data */
+		return 0;
+	}
+
+	stream_errno = parser->input->stream_errno;
+	i_stream_unref(&parser->input);
+
+	if (parser->content_state == SOLR_XML_CONTENT_STATE_ERROR)
+		return -1;
+	if (stream_errno != 0)
+		return -1;
+
+	ret = solr_xml_parse(parser, "", 0, TRUE);
+
+	array_append_zero(&parser->results);
+	*box_results_r = array_front_modifiable(&parser->results);
+	return (ret == 0 ? 1 : -1);
+}