path: root/src/plugins/fts/doveadm-fts.c

/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "str.h"
#include "imap-util.h"
#include "mail-namespace.h"
#include "mail-search.h"
#include "mailbox-list-iter.h"
#include "fts-tokenizer.h"
#include "fts-filter.h"
#include "fts-language.h"
#include "fts-storage.h"
#include "fts-search-args.h"
#include "fts-user.h"
#include "doveadm-print.h"
#include "doveadm-mail.h"
#include "doveadm-mailbox-list-iter.h"
#include "doveadm-fts.h"

const char *doveadm_fts_plugin_version = DOVECOT_ABI_VERSION;

struct fts_tokenize_cmd_context {
	struct doveadm_mail_cmd_context ctx;
	const char *language;
	const char *tokens;
};

static int
cmd_search_box(struct doveadm_mail_cmd_context *ctx,
	       const struct mailbox_info *info)
{
	struct mailbox *box;
	struct fts_backend *backend;
	struct fts_result result;
	int ret = 0;

	backend = fts_list_backend(info->ns->list);
	if (backend == NULL) {
		i_error("fts not enabled for %s", info->vname);
		ctx->exit_code = EX_CONFIG;
		return -1;
	}

	i_zero(&result);
	i_array_init(&result.definite_uids, 16);
	i_array_init(&result.maybe_uids, 16);
	i_array_init(&result.scores, 16);

	box = mailbox_alloc(info->ns->list, info->vname, 0);
	if (fts_backend_lookup(backend, box, ctx->search_args->args,
				      FTS_LOOKUP_FLAG_AND_ARGS, &result) < 0) {
		i_error("fts lookup failed");
		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
		ret = -1;
	} else {
		printf("%s: ", info->vname);
		if (array_count(&result.definite_uids) == 0)
			printf("no results\n");
		else T_BEGIN {
			string_t *str = t_str_new(128);
			imap_write_seq_range(str, &result.definite_uids);
			printf("%s\n", str_c(str));
		} T_END;
		if (array_count(&result.maybe_uids) > 0) T_BEGIN {
			string_t *str = t_str_new(128);
			imap_write_seq_range(str, &result.maybe_uids);
			printf(" - maybe: %s\n", str_c(str));
		} T_END;
		fts_backend_lookup_done(backend);
	}
	mailbox_free(&box);
	array_free(&result.definite_uids);
	array_free(&result.maybe_uids);
	array_free(&result.scores);
	return ret;
}

static int
cmd_fts_lookup_run(struct doveadm_mail_cmd_context *ctx,
		   struct mail_user *user)
{
	const enum mailbox_list_iter_flags iter_flags =
		MAILBOX_LIST_ITER_NO_AUTO_BOXES |
		MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
	struct doveadm_mailbox_list_iter *iter;
	const struct mailbox_info *info;
	int ret = 0;

	iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args,
					      iter_flags);
	while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
		if (cmd_search_box(ctx, info) < 0)
			ret = -1;
	} T_END;
	if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
		ret = -1;
	return ret;
}

static void
cmd_fts_lookup_init(struct doveadm_mail_cmd_context *ctx,
		    const char *const args[])
{
	if (args[0] == NULL)
		doveadm_mail_help_name("fts lookup");

	ctx->search_args = doveadm_mail_build_search_args(args);
}

static struct doveadm_mail_cmd_context *
cmd_fts_lookup_alloc(void)
{
	struct doveadm_mail_cmd_context *ctx;

	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
	ctx->v.run = cmd_fts_lookup_run;
	ctx->v.init = cmd_fts_lookup_init;
	return ctx;
}

static int
cmd_fts_expand_run(struct doveadm_mail_cmd_context *ctx,
		   struct mail_user *user)
{
	struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);
	struct mailbox *box;
	struct fts_backend *backend;
	string_t *str = t_str_new(128);

	backend = fts_list_backend(ns->list);
	if (backend == NULL) {
		i_error("fts not enabled for INBOX");
		ctx->exit_code = EX_CONFIG;
		return -1;
	}

	box = mailbox_alloc(ns->list, "INBOX", 0);
	mail_search_args_init(ctx->search_args, box, FALSE, NULL);

	if (fts_search_args_expand(backend, ctx->search_args) < 0)
		i_fatal("Couldn't expand search args");
	mail_search_args_to_cmdline(str, ctx->search_args->args);
	printf("%s\n", str_c(str));
	mailbox_free(&box);
	return 0;
}

static void
cmd_fts_expand_init(struct doveadm_mail_cmd_context *ctx,
		    const char *const args[])
{
	if (args[0] == NULL)
		doveadm_mail_help_name("fts expand");

	ctx->search_args = doveadm_mail_build_search_args(args);
}

static struct doveadm_mail_cmd_context *
cmd_fts_expand_alloc(void)
{
	struct doveadm_mail_cmd_context *ctx;

	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
	ctx->v.run = cmd_fts_expand_run;
	ctx->v.init = cmd_fts_expand_init;
	return ctx;
}

static int
cmd_fts_tokenize_run(struct doveadm_mail_cmd_context *_ctx,
		     struct mail_user *user)
{
	struct fts_tokenize_cmd_context *ctx =
		(struct fts_tokenize_cmd_context *)_ctx;
	struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);
	struct fts_backend *backend;
	struct fts_user_language *user_lang;
	const struct fts_language *lang = NULL;
	int ret, ret2;
	bool final = FALSE;

	backend = fts_list_backend(ns->list);
	if (backend == NULL) {
		i_error("fts not enabled for INBOX");
		_ctx->exit_code = EX_CONFIG;
		return -1;
	}

	if (ctx->language == NULL) {
		struct fts_language_list *lang_list =
			fts_user_get_language_list(user);
		enum fts_language_result result;
		const char *error;

		result = fts_language_detect(lang_list,
		    (const unsigned char *)ctx->tokens, strlen(ctx->tokens),
                    &lang, &error);
		if (lang == NULL)
			lang = fts_language_list_get_first(lang_list);
		switch (result) {
		case FTS_LANGUAGE_RESULT_SHORT:
			i_warning("Text too short, can't detect its language - assuming %s", lang->name);
			break;
		case FTS_LANGUAGE_RESULT_UNKNOWN:
			i_warning("Can't detect its language - assuming %s", lang->name);
			break;
		case FTS_LANGUAGE_RESULT_OK:
			break;
		case FTS_LANGUAGE_RESULT_ERROR:
			i_error("Language detection library initialization failed: %s", error);
			_ctx->exit_code = EX_CONFIG;
			return -1;
		default:
			i_unreached();
		}
	} else {
		lang = fts_language_find(ctx->language);
		if (lang == NULL) {
			i_error("Unknown language: %s", ctx->language);
			_ctx->exit_code = EX_USAGE;
			return -1;
		}
	}
	user_lang = fts_user_language_find(user, lang);
	if (user_lang == NULL) {
		i_error("Language not enabled for user: %s", ctx->language);
		_ctx->exit_code = EX_USAGE;
		return -1;
	}

	fts_tokenizer_reset(user_lang->index_tokenizer);
	for (;;) {
		const char *token, *error;

		if (!final) {
			ret = fts_tokenizer_next(user_lang->index_tokenizer,
				(const unsigned char *)ctx->tokens, strlen(ctx->tokens),
				&token, &error);
		} else {
			ret = fts_tokenizer_final(user_lang->index_tokenizer,
						  &token, &error);
		}
		if (ret < 0)
			break;
		if (ret > 0 && user_lang->filter != NULL) {
			ret2 = fts_filter_filter(user_lang->filter, &token, &error);
			if (ret2 > 0)
				doveadm_print(token);
			else if (ret2 < 0)
				i_error("Couldn't create indexable tokens: %s", error);
		}
		if (ret == 0) {
			if (final)
				break;
			final = TRUE;
		}
	}
	return 0;
}

static void
cmd_fts_tokenize_init(struct doveadm_mail_cmd_context *_ctx,
		      const char *const args[])
{
	struct fts_tokenize_cmd_context *ctx =
		(struct fts_tokenize_cmd_context *)_ctx;

	if (args[0] == NULL)
		doveadm_mail_help_name("fts tokenize");

	ctx->tokens = p_strdup(_ctx->pool, t_strarray_join(args, " "));

	doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
	doveadm_print_header("token", "token", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
}

static bool
cmd_fts_tokenize_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
{
	struct fts_tokenize_cmd_context *ctx =
		(struct fts_tokenize_cmd_context *)_ctx;

	switch (c) {
	case 'l':
		ctx->language = p_strdup(_ctx->pool, optarg);
		break;
	default:
		return FALSE;
	}
	return TRUE;
}

static struct doveadm_mail_cmd_context *
cmd_fts_tokenize_alloc(void)
{
	struct fts_tokenize_cmd_context *ctx;

	ctx = doveadm_mail_cmd_alloc(struct fts_tokenize_cmd_context);
	ctx->ctx.v.run = cmd_fts_tokenize_run;
	ctx->ctx.v.init = cmd_fts_tokenize_init;
	ctx->ctx.v.parse_arg = cmd_fts_tokenize_parse_arg;
	ctx->ctx.getopt_args = "l";
	return &ctx->ctx;
}

static int
fts_namespace_find(struct mail_user *user, const char *ns_prefix,
		   struct mail_namespace **ns_r)
{
	struct mail_namespace *ns;

	if (ns_prefix == NULL)
		ns = mail_namespace_find_inbox(user->namespaces);
	else {
		ns = mail_namespace_find_prefix(user->namespaces, ns_prefix);
		if (ns == NULL) {
			i_error("Namespace prefix not found: %s", ns_prefix);
			return -1;
		}
	}

	if (fts_list_backend(ns->list) == NULL) {
		i_error("fts not enabled for user's namespace %s",
			ns_prefix != NULL ? ns_prefix : "INBOX");
		return -1;
	}
	*ns_r = ns;
	return 0;
}

static int
cmd_fts_optimize_run(struct doveadm_mail_cmd_context *ctx,
		     struct mail_user *user)
{
	const char *ns_prefix = ctx->args[0];
	struct mail_namespace *ns;
	struct fts_backend *backend;

	if (fts_namespace_find(user, ns_prefix, &ns) < 0) {
		doveadm_mail_failed_error(ctx, MAIL_ERROR_NOTFOUND);
		return -1;
	}
	backend = fts_list_backend(ns->list);
	if (fts_backend_optimize(backend) < 0) {
		i_error("fts optimize failed");
		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
		return -1;
	}
	return 0;
}

static void
cmd_fts_optimize_init(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
		      const char *const args[])
{
	if (str_array_length(args) > 1)
		doveadm_mail_help_name("fts optimize");
}

static struct doveadm_mail_cmd_context *
cmd_fts_optimize_alloc(void)
{
	struct doveadm_mail_cmd_context *ctx;

	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
	ctx->v.run = cmd_fts_optimize_run;
	ctx->v.init = cmd_fts_optimize_init;
	return ctx;
}

static int
cmd_fts_rescan_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user)
{
	const char *ns_prefix = ctx->args[0];
	struct mail_namespace *ns;
	struct fts_backend *backend;

	if (fts_namespace_find(user, ns_prefix, &ns) < 0) {
		doveadm_mail_failed_error(ctx, MAIL_ERROR_NOTFOUND);
		return -1;
	}
	backend = fts_list_backend(ns->list);
	if (fts_backend_rescan(backend) < 0) {
		i_error("fts rescan failed");
		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
		return -1;
	}
	return 0;
}

static void
cmd_fts_rescan_init(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
		    const char *const args[])
{
	if (str_array_length(args) > 1)
		doveadm_mail_help_name("fts rescan");
}

static struct doveadm_mail_cmd_context *
cmd_fts_rescan_alloc(void)
{
	struct doveadm_mail_cmd_context *ctx;

	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
	ctx->v.run = cmd_fts_rescan_run;
	ctx->v.init = cmd_fts_rescan_init;
	return ctx;
}

static struct doveadm_cmd_ver2 fts_commands[] = {
{
	.name = "fts lookup",
	.mail_cmd = cmd_fts_lookup_alloc,
	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>",
DOVEADM_CMD_PARAMS_START
DOVEADM_CMD_MAIL_COMMON
DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
DOVEADM_CMD_PARAMS_END
},
{
	.name = "fts expand",
	.mail_cmd = cmd_fts_expand_alloc,
	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>",
DOVEADM_CMD_PARAMS_START
DOVEADM_CMD_MAIL_COMMON
DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
DOVEADM_CMD_PARAMS_END
},
{
	.name = "fts tokenize",
	.mail_cmd = cmd_fts_tokenize_alloc,
	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<text>",
DOVEADM_CMD_PARAMS_START
DOVEADM_CMD_MAIL_COMMON
DOVEADM_CMD_PARAM('l', "language", CMD_PARAM_STR, 0)
DOVEADM_CMD_PARAM('\0', "text", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
DOVEADM_CMD_PARAMS_END
},
{
	.name = "fts optimize",
	.mail_cmd = cmd_fts_optimize_alloc,
	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[<namespace>]",
DOVEADM_CMD_PARAMS_START
DOVEADM_CMD_MAIL_COMMON
DOVEADM_CMD_PARAM('\0', "namespace", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
DOVEADM_CMD_PARAMS_END
},
{
	.name = "fts rescan",
	.mail_cmd = cmd_fts_rescan_alloc,
	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[<namespace>]",
DOVEADM_CMD_PARAMS_START
DOVEADM_CMD_MAIL_COMMON
DOVEADM_CMD_PARAM('\0', "namespace", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
DOVEADM_CMD_PARAMS_END
},
};

void doveadm_fts_plugin_init(struct module *module ATTR_UNUSED)
{
	unsigned int i;

	for (i = 0; i < N_ELEMENTS(fts_commands); i++)
		doveadm_cmd_register_ver2(&fts_commands[i]);
	doveadm_dump_fts_expunge_log_init();
}

void doveadm_fts_plugin_deinit(void)
{
}