Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 470 insertions, 0 deletions

diff --git a/src/plugins/fts/doveadm-fts.c b/src/plugins/fts/doveadm-fts.c
new file mode 100644
index 0000000..1b902a1
--- /dev/null
+++ b/src/plugins/fts/doveadm-fts.c
@@ -0,0 +1,470 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "imap-util.h"
+#include "mail-namespace.h"
+#include "mail-search.h"
+#include "mailbox-list-iter.h"
+#include "fts-tokenizer.h"
+#include "fts-filter.h"
+#include "fts-language.h"
+#include "fts-storage.h"
+#include "fts-search-args.h"
+#include "fts-user.h"
+#include "doveadm-print.h"
+#include "doveadm-mail.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-fts.h"
+
+const char *doveadm_fts_plugin_version = DOVECOT_ABI_VERSION;
+
+struct fts_tokenize_cmd_context {
+	struct doveadm_mail_cmd_context ctx;
+	const char *language;
+	const char *tokens;
+};
+
+static int
+cmd_search_box(struct doveadm_mail_cmd_context *ctx,
+	       const struct mailbox_info *info)
+{
+	struct mailbox *box;
+	struct fts_backend *backend;
+	struct fts_result result;
+	int ret = 0;
+
+	backend = fts_list_backend(info->ns->list);
+	if (backend == NULL) {
+		i_error("fts not enabled for %s", info->vname);
+		ctx->exit_code = EX_CONFIG;
+		return -1;
+	}
+
+	i_zero(&result);
+	i_array_init(&result.definite_uids, 16);
+	i_array_init(&result.maybe_uids, 16);
+	i_array_init(&result.scores, 16);
+
+	box = mailbox_alloc(info->ns->list, info->vname, 0);
+	if (fts_backend_lookup(backend, box, ctx->search_args->args,
+				      FTS_LOOKUP_FLAG_AND_ARGS, &result) < 0) {
+		i_error("fts lookup failed");
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
+		ret = -1;
+	} else {
+		printf("%s: ", info->vname);
+		if (array_count(&result.definite_uids) == 0)
+			printf("no results\n");
+		else T_BEGIN {
+			string_t *str = t_str_new(128);
+			imap_write_seq_range(str, &result.definite_uids);
+			printf("%s\n", str_c(str));
+		} T_END;
+		if (array_count(&result.maybe_uids) > 0) T_BEGIN {
+			string_t *str = t_str_new(128);
+			imap_write_seq_range(str, &result.maybe_uids);
+			printf(" - maybe: %s\n", str_c(str));
+		} T_END;
+		fts_backend_lookup_done(backend);
+	}
+	mailbox_free(&box);
+	array_free(&result.definite_uids);
+	array_free(&result.maybe_uids);
+	array_free(&result.scores);
+	return ret;
+}
+
+static int
+cmd_fts_lookup_run(struct doveadm_mail_cmd_context *ctx,
+		   struct mail_user *user)
+{
+	const enum mailbox_list_iter_flags iter_flags =
+		MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+		MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+	struct doveadm_mailbox_list_iter *iter;
+	const struct mailbox_info *info;
+	int ret = 0;
+
+	iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args,
+					      iter_flags);
+	while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+		if (cmd_search_box(ctx, info) < 0)
+			ret = -1;
+	} T_END;
+	if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+		ret = -1;
+	return ret;
+}
+
+static void
+cmd_fts_lookup_init(struct doveadm_mail_cmd_context *ctx,
+		    const char *const args[])
+{
+	if (args[0] == NULL)
+		doveadm_mail_help_name("fts lookup");
+
+	ctx->search_args = doveadm_mail_build_search_args(args);
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_lookup_alloc(void)
+{
+	struct doveadm_mail_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+	ctx->v.run = cmd_fts_lookup_run;
+	ctx->v.init = cmd_fts_lookup_init;
+	return ctx;
+}
+
+static int
+cmd_fts_expand_run(struct doveadm_mail_cmd_context *ctx,
+		   struct mail_user *user)
+{
+	struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);
+	struct mailbox *box;
+	struct fts_backend *backend;
+	string_t *str = t_str_new(128);
+
+	backend = fts_list_backend(ns->list);
+	if (backend == NULL) {
+		i_error("fts not enabled for INBOX");
+		ctx->exit_code = EX_CONFIG;
+		return -1;
+	}
+
+	box = mailbox_alloc(ns->list, "INBOX", 0);
+	mail_search_args_init(ctx->search_args, box, FALSE, NULL);
+
+	if (fts_search_args_expand(backend, ctx->search_args) < 0)
+		i_fatal("Couldn't expand search args");
+	mail_search_args_to_cmdline(str, ctx->search_args->args);
+	printf("%s\n", str_c(str));
+	mailbox_free(&box);
+	return 0;
+}
+
+static void
+cmd_fts_expand_init(struct doveadm_mail_cmd_context *ctx,
+		    const char *const args[])
+{
+	if (args[0] == NULL)
+		doveadm_mail_help_name("fts expand");
+
+	ctx->search_args = doveadm_mail_build_search_args(args);
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_expand_alloc(void)
+{
+	struct doveadm_mail_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+	ctx->v.run = cmd_fts_expand_run;
+	ctx->v.init = cmd_fts_expand_init;
+	return ctx;
+}
+
+static int
+cmd_fts_tokenize_run(struct doveadm_mail_cmd_context *_ctx,
+		     struct mail_user *user)
+{
+	struct fts_tokenize_cmd_context *ctx =
+		(struct fts_tokenize_cmd_context *)_ctx;
+	struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);
+	struct fts_backend *backend;
+	struct fts_user_language *user_lang;
+	const struct fts_language *lang = NULL;
+	int ret, ret2;
+	bool final = FALSE;
+
+	backend = fts_list_backend(ns->list);
+	if (backend == NULL) {
+		i_error("fts not enabled for INBOX");
+		_ctx->exit_code = EX_CONFIG;
+		return -1;
+	}
+
+	if (ctx->language == NULL) {
+		struct fts_language_list *lang_list =
+			fts_user_get_language_list(user);
+		enum fts_language_result result;
+		const char *error;
+
+		result = fts_language_detect(lang_list,
+		    (const unsigned char *)ctx->tokens, strlen(ctx->tokens),
+                    &lang, &error);
+		if (lang == NULL)
+			lang = fts_language_list_get_first(lang_list);
+		switch (result) {
+		case FTS_LANGUAGE_RESULT_SHORT:
+			i_warning("Text too short, can't detect its language - assuming %s", lang->name);
+			break;
+		case FTS_LANGUAGE_RESULT_UNKNOWN:
+			i_warning("Can't detect its language - assuming %s", lang->name);
+			break;
+		case FTS_LANGUAGE_RESULT_OK:
+			break;
+		case FTS_LANGUAGE_RESULT_ERROR:
+			i_error("Language detection library initialization failed: %s", error);
+			_ctx->exit_code = EX_CONFIG;
+			return -1;
+		default:
+			i_unreached();
+		}
+	} else {
+		lang = fts_language_find(ctx->language);
+		if (lang == NULL) {
+			i_error("Unknown language: %s", ctx->language);
+			_ctx->exit_code = EX_USAGE;
+			return -1;
+		}
+	}
+	user_lang = fts_user_language_find(user, lang);
+	if (user_lang == NULL) {
+		i_error("Language not enabled for user: %s", ctx->language);
+		_ctx->exit_code = EX_USAGE;
+		return -1;
+	}
+
+	fts_tokenizer_reset(user_lang->index_tokenizer);
+	for (;;) {
+		const char *token, *error;
+
+		if (!final) {
+			ret = fts_tokenizer_next(user_lang->index_tokenizer,
+				(const unsigned char *)ctx->tokens, strlen(ctx->tokens),
+				&token, &error);
+		} else {
+			ret = fts_tokenizer_final(user_lang->index_tokenizer,
+						  &token, &error);
+		}
+		if (ret < 0)
+			break;
+		if (ret > 0 && user_lang->filter != NULL) {
+			ret2 = fts_filter_filter(user_lang->filter, &token, &error);
+			if (ret2 > 0)
+				doveadm_print(token);
+			else if (ret2 < 0)
+				i_error("Couldn't create indexable tokens: %s", error);
+		}
+		if (ret == 0) {
+			if (final)
+				break;
+			final = TRUE;
+		}
+	}
+	return 0;
+}
+
+static void
+cmd_fts_tokenize_init(struct doveadm_mail_cmd_context *_ctx,
+		      const char *const args[])
+{
+	struct fts_tokenize_cmd_context *ctx =
+		(struct fts_tokenize_cmd_context *)_ctx;
+
+	if (args[0] == NULL)
+		doveadm_mail_help_name("fts tokenize");
+
+	ctx->tokens = p_strdup(_ctx->pool, t_strarray_join(args, " "));
+
+	doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+	doveadm_print_header("token", "token", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+}
+
+static bool
+cmd_fts_tokenize_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+	struct fts_tokenize_cmd_context *ctx =
+		(struct fts_tokenize_cmd_context *)_ctx;
+
+	switch (c) {
+	case 'l':
+		ctx->language = p_strdup(_ctx->pool, optarg);
+		break;
+	default:
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_tokenize_alloc(void)
+{
+	struct fts_tokenize_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct fts_tokenize_cmd_context);
+	ctx->ctx.v.run = cmd_fts_tokenize_run;
+	ctx->ctx.v.init = cmd_fts_tokenize_init;
+	ctx->ctx.v.parse_arg = cmd_fts_tokenize_parse_arg;
+	ctx->ctx.getopt_args = "l";
+	return &ctx->ctx;
+}
+
+static int
+fts_namespace_find(struct mail_user *user, const char *ns_prefix,
+		   struct mail_namespace **ns_r)
+{
+	struct mail_namespace *ns;
+
+	if (ns_prefix == NULL)
+		ns = mail_namespace_find_inbox(user->namespaces);
+	else {
+		ns = mail_namespace_find_prefix(user->namespaces, ns_prefix);
+		if (ns == NULL) {
+			i_error("Namespace prefix not found: %s", ns_prefix);
+			return -1;
+		}
+	}
+
+	if (fts_list_backend(ns->list) == NULL) {
+		i_error("fts not enabled for user's namespace %s",
+			ns_prefix != NULL ? ns_prefix : "INBOX");
+		return -1;
+	}
+	*ns_r = ns;
+	return 0;
+}
+
+static int
+cmd_fts_optimize_run(struct doveadm_mail_cmd_context *ctx,
+		     struct mail_user *user)
+{
+	const char *ns_prefix = ctx->args[0];
+	struct mail_namespace *ns;
+	struct fts_backend *backend;
+
+	if (fts_namespace_find(user, ns_prefix, &ns) < 0) {
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_NOTFOUND);
+		return -1;
+	}
+	backend = fts_list_backend(ns->list);
+	if (fts_backend_optimize(backend) < 0) {
+		i_error("fts optimize failed");
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
+		return -1;
+	}
+	return 0;
+}
+
+static void
+cmd_fts_optimize_init(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
+		      const char *const args[])
+{
+	if (str_array_length(args) > 1)
+		doveadm_mail_help_name("fts optimize");
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_optimize_alloc(void)
+{
+	struct doveadm_mail_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+	ctx->v.run = cmd_fts_optimize_run;
+	ctx->v.init = cmd_fts_optimize_init;
+	return ctx;
+}
+
+static int
+cmd_fts_rescan_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user)
+{
+	const char *ns_prefix = ctx->args[0];
+	struct mail_namespace *ns;
+	struct fts_backend *backend;
+
+	if (fts_namespace_find(user, ns_prefix, &ns) < 0) {
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_NOTFOUND);
+		return -1;
+	}
+	backend = fts_list_backend(ns->list);
+	if (fts_backend_rescan(backend) < 0) {
+		i_error("fts rescan failed");
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
+		return -1;
+	}
+	return 0;
+}
+
+static void
+cmd_fts_rescan_init(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
+		    const char *const args[])
+{
+	if (str_array_length(args) > 1)
+		doveadm_mail_help_name("fts rescan");
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_rescan_alloc(void)
+{
+	struct doveadm_mail_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+	ctx->v.run = cmd_fts_rescan_run;
+	ctx->v.init = cmd_fts_rescan_init;
+	return ctx;
+}
+
+static struct doveadm_cmd_ver2 fts_commands[] = {
+{
+	.name = "fts lookup",
+	.mail_cmd = cmd_fts_lookup_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "fts expand",
+	.mail_cmd = cmd_fts_expand_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "fts tokenize",
+	.mail_cmd = cmd_fts_tokenize_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<text>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('l', "language", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "text", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "fts optimize",
+	.mail_cmd = cmd_fts_optimize_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[<namespace>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "namespace", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "fts rescan",
+	.mail_cmd = cmd_fts_rescan_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[<namespace>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "namespace", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+};
+
+void doveadm_fts_plugin_init(struct module *module ATTR_UNUSED)
+{
+	unsigned int i;
+
+	for (i = 0; i < N_ELEMENTS(fts_commands); i++)
+		doveadm_cmd_register_ver2(&fts_commands[i]);
+	doveadm_dump_fts_expunge_log_init();
+}
+
+void doveadm_fts_plugin_deinit(void)
+{
+}