fts-storage.c revision a7061727e8f1378228f110b23d816329d39ce82b
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "lib.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "array.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "mail-search.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "mail-storage-private.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "mailbox-list-private.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "../virtual/virtual-storage.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "fts-api-private.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "fts-build.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "fts-search-serialize.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "fts-plugin.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "fts-storage.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include <stdlib.h>
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#define FTS_CONTEXT(obj) \
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen MODULE_CONTEXT(obj, fts_storage_module)
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#define FTS_MAIL_CONTEXT(obj) \
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen MODULE_CONTEXT(obj, fts_mail_module)
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#define FTS_LIST_CONTEXT(obj) \
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen MODULE_CONTEXT(obj, fts_mailbox_list_module)
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainenstruct fts_mailbox_list {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen union mailbox_list_module_context module_ctx;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen struct fts_backend *backend;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen};
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstruct fts_mailbox {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen union mailbox_module_context module_ctx;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_backend_update_context *sync_update_ctx;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen};
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstruct fts_transaction_context {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen union mailbox_transaction_module_context module_ctx;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_scores *scores;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen};
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstruct fts_mail {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen union mail_module_context module_ctx;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen char score[30];
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen};
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_storage_module,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen &mail_storage_module_register);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen &mailbox_list_module_register);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic void fts_scores_unref(struct fts_scores **_scores)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_scores *scores = *_scores;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen *_scores = NULL;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (--scores->refcount == 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen array_free(&scores->score_map);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_free(scores);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic bool fts_try_build_init(struct mail_search_context *ctx,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_search_context *fctx)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (fts_backend_is_updating(fctx->backend)) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* this process is already building the indexes */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return FALSE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fctx->build_initialized = TRUE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen switch (fts_build_init(fctx->backend, ctx->transaction->box, FALSE,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen &fctx->build_ctx)) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen case -1:
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen break;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen case 0:
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* the index was up to date */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_search_lookup(fctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen break;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen case 1:
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* hide "searching" notifications while building index */
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen ctx->progress_hidden = TRUE;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen break;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return TRUE;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen}
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainenstatic bool fts_want_build_args(const struct mail_search_arg *args)
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen{
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen /* we want to update index only when searching from message body.
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen it's not worth the wait for searching only from headers, which
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen could be in cache file already */
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen for (; args != NULL; args = args->next) {
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen switch (args->type) {
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen case SEARCH_OR:
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen case SEARCH_SUB:
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen case SEARCH_INTHREAD:
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (fts_want_build_args(args->value.subargs))
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return TRUE;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen break;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen case SEARCH_BODY:
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen case SEARCH_TEXT:
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return TRUE;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen default:
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen break;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return FALSE;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen}
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainenstatic struct mail_search_context *
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainenfts_mailbox_search_init(struct mailbox_transaction_context *t,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct mail_search_args *args,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen const enum mail_sort_type *sort_program,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen enum mail_fetch_field wanted_fields,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct mailbox_header_lookup_ctx *wanted_headers)
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen{
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(t->box->list);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct mail_search_context *ctx;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct fts_search_context *fctx;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen ctx = fbox->module_ctx.super.search_init(t, args, sort_program,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen wanted_fields, wanted_headers);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (!fts_backend_can_lookup(flist->backend, args->args))
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return ctx;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx = i_new(struct fts_search_context, 1);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx->box = t->box;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx->backend = flist->backend;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx->t = t;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx->args = args;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx->result_pool = pool_alloconly_create("fts results", 1024*64);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
bed3fcccce6a15dfd7491e82df10acd5f5c0bc9bTimo Sirainen i_array_init(&fctx->levels, 8);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx->scores = i_new(struct fts_scores, 1);
e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainen fctx->scores->refcount = 1;
e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainen i_array_init(&fctx->scores->score_map, 64);
e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainen MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen fctx->virtual_mailbox =
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* transaction contains the last search's scores. they can be
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen queried later with mail_get_special() */
02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen if (ft->scores != NULL)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_scores_unref(&ft->scores);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ft->scores = fctx->scores;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ft->scores->refcount++;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (fts_want_build_args(args->args))
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen (void)fts_try_build_init(ctx, fctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen else {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fctx->build_initialized = TRUE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_search_lookup(fctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return ctx;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic bool fts_mailbox_build_continue(struct mail_search_context *ctx)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen int ret;
02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (fctx == NULL)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return TRUE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!fctx->build_initialized) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* we're still waiting for this process (but another command)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen to finish building the indexes */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!fts_try_build_init(ctx, fctx))
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return FALSE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (fctx->build_ctx != NULL) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* this command is still building the indexes */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ret = fts_build_more(fctx->build_ctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ret == 0)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return FALSE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ctx->progress_hidden = FALSE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (fts_build_deinit(&fctx->build_ctx) < 0)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ret = -1;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ret > 0)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_search_lookup(fctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return TRUE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic bool
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenfts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail **mail_r, bool *tryagain_r)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!fts_mailbox_build_continue(ctx)) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen *tryagain_r = TRUE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return FALSE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return fbox->module_ctx.super.
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen search_next_nonblock(ctx, mail_r, tryagain_r);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic void
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenfts_search_apply_results_level(struct mail_search_context *ctx,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_search_arg *args, unsigned int *idx)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const struct fts_search_level *level;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen level = array_idx(&fctx->levels, *idx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (array_is_created(&level->definite_seqs) &&
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen seq_range_exists(&level->definite_seqs, ctx->seq))
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_search_deserialize_add_matches(args, level->args_matches);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen else if (!array_is_created(&level->maybe_seqs) ||
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen !seq_range_exists(&level->maybe_seqs, ctx->seq))
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_search_deserialize_add_nonmatches(args, level->args_matches);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen for (; args != NULL; args = args->next) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen continue;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen *idx += 1;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_search_apply_results_level(ctx, args->value.subargs, idx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen unsigned int idx;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (fctx == NULL || !fctx->fts_lookup_success) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* fts lookup not done for this search */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return fbox->module_ctx.super.search_next_update_seq(ctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* restore original [non]matches */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_search_deserialize(ctx->args->args, fctx->orig_matches);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!fbox->module_ctx.super.search_next_update_seq(ctx))
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return FALSE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ctx->seq >= fctx->first_unindexed_seq) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* we've not indexed this far */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return TRUE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* apply [non]matches based on the FTS lookup results */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen idx = 0;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fts_search_apply_results_level(ctx, ctx->args->args, &idx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return TRUE;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic int fts_mailbox_search_deinit(struct mail_search_context *ctx)
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (fctx != NULL) {
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen if (fctx->build_ctx != NULL) {
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen /* the search was cancelled */
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen (void)fts_build_deinit(&fctx->build_ctx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_free(&fctx->orig_matches);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen array_free(&fctx->levels);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen pool_unref(&fctx->result_pool);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen fts_scores_unref(&fctx->scores);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen i_free(fctx);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen }
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen return fbox->module_ctx.super.search_deinit(ctx);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen}
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainenstatic int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score)
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen{
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen return *uid < score->uid ? -1 :
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen (*uid > score->uid ? 1 : 0);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen}
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainenstatic int fts_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen const char **value_r)
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen{
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
3a017aa592823edf0363d77f13458d569637915eTimo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
3a017aa592823edf0363d77f13458d569637915eTimo Sirainen const struct fts_score_map *scores;
3a017aa592823edf0363d77f13458d569637915eTimo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (field != MAIL_FETCH_SEARCH_RELEVANCY || ft->scores == NULL)
3a017aa592823edf0363d77f13458d569637915eTimo Sirainen scores = NULL;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen else {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen scores = array_bsearch(&ft->scores->score_map, &_mail->uid,
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen fts_score_cmp);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (scores != NULL) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_assert(scores->uid == _mail->uid);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_snprintf(fmail->score, sizeof(fmail->score),
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen "%f", scores->score);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen *value_r = fmail->score;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen }
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return fmail->module_ctx.super.get_special(_mail, field, value_r);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainenvoid fts_mail_allocated(struct mail *_mail)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_vfuncs *v = mail->vlast;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
bddd52cb7f3e5a894c080f60750aa76b5aeaf103Timo Sirainen struct fts_mail *fmail;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (fbox == NULL)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fmail = p_new(mail->pool, struct fts_mail, 1);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen fmail->module_ctx.super = *v;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail->vlast = &fmail->module_ctx.super;
b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen
b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen v->get_special = fts_mail_get_special;
b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen MODULE_CONTEXT_SET(mail, fts_mail_module, fmail);
b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen}
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic struct mailbox_transaction_context *
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenfts_transaction_begin(struct mailbox *box,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen enum mailbox_transaction_flags flags)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mailbox_transaction_context *t;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_transaction_context *ft;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ft = i_new(struct fts_transaction_context, 1);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen t = fbox->module_ctx.super.transaction_begin(box, flags);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen MODULE_CONTEXT_SET(t, fts_storage_module, ft);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return t;
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen}
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainenstatic void fts_transaction_rollback(struct mailbox_transaction_context *t)
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
if (ft->scores != NULL)
fts_scores_unref(&ft->scores);
i_free(ft);
fbox->module_ctx.super.transaction_rollback(t);
}
static int
fts_transaction_commit(struct mailbox_transaction_context *t,
struct mail_transaction_commit_changes *changes_r)
{
struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
if (ft->scores != NULL)
fts_scores_unref(&ft->scores);
i_free(ft);
return fbox->module_ctx.super.transaction_commit(t, changes_r);
}
static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
enum mailbox_sync_type sync_type)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
struct fts_mailbox *fbox = FTS_CONTEXT(box);
if (fbox->module_ctx.super.sync_notify != NULL)
fbox->module_ctx.super.sync_notify(box, uid, sync_type);
if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE) {
if (uid == 0 && fbox->sync_update_ctx != NULL) {
/* this sync is finished */
(void)fts_backend_update_deinit(&fbox->sync_update_ctx);
}
return;
}
if (fbox->sync_update_ctx == NULL) {
if (fts_backend_is_updating(flist->backend)) {
/* FIXME: maildir workaround - we could get here
because we're building an index, which doesn't find
some mail and starts syncing the mailbox.. */
return;
}
fbox->sync_update_ctx = fts_backend_update_init(flist->backend);
fts_backend_update_set_mailbox(fbox->sync_update_ctx, box);
}
fts_backend_update_expunge(fbox->sync_update_ctx, uid);
}
static int fts_update(struct mailbox *box)
{
struct fts_storage_build_context *build_ctx;
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
int ret = 0;
if ((ret = fts_build_init(flist->backend, box,
TRUE, &build_ctx)) <= 0) {
if (box->storage->set->mail_debug)
i_debug("%s: FTS index is up to date", box->vname);
return ret;
}
if (box->storage->set->mail_debug)
i_debug("%s: Updating FTS index", box->vname);
while ((ret = fts_build_more(build_ctx)) == 0) ;
if (fts_build_deinit(&build_ctx) < 0)
ret = -1;
return ret < 0 ? -1 : 0;
}
static int fts_sync_deinit(struct mailbox_sync_context *ctx,
struct mailbox_sync_status *status_r)
{
struct mailbox *box = ctx->box;
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
bool precache, force_resync;
int ret = 0;
precache = (ctx->flags & MAILBOX_SYNC_FLAG_PRECACHE) != 0;
force_resync = (ctx->flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0;
if (fbox->module_ctx.super.sync_deinit(ctx, status_r) < 0)
return -1;
ctx = NULL;
flist->backend->syncing = TRUE;
if (force_resync) {
if (fts_backend_optimize(flist->backend) < 0) {
mail_storage_set_critical(box->storage,
"FTS optimize for mailbox %s failed",
box->vname);
ret = -1;
}
}
if (precache) {
if (fts_update(box) < 0) {
mail_storage_set_critical(box->storage,
"FTS index update for mailbox %s failed",
box->vname);
ret = -1;
}
}
flist->backend->syncing = FALSE;
return ret;
}
void fts_mailbox_allocated(struct mailbox *box)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
struct mailbox_vfuncs *v = box->vlast;
struct fts_mailbox *fbox;
if (flist == NULL)
return;
fbox = p_new(box->pool, struct fts_mailbox, 1);
fbox->module_ctx.super = *v;
box->vlast = &fbox->module_ctx.super;
v->search_init = fts_mailbox_search_init;
v->search_next_nonblock = fts_mailbox_search_next_nonblock;
v->search_next_update_seq = fts_mailbox_search_next_update_seq;
v->search_deinit = fts_mailbox_search_deinit;
v->transaction_begin = fts_transaction_begin;
v->transaction_rollback = fts_transaction_rollback;
v->transaction_commit = fts_transaction_commit;
v->sync_notify = fts_mailbox_sync_notify;
v->sync_deinit = fts_sync_deinit;
MODULE_CONTEXT_SET(box, fts_storage_module, fbox);
}
static void fts_mailbox_list_deinit(struct mailbox_list *list)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list);
fts_backend_deinit(&flist->backend);
flist->module_ctx.super.deinit(list);
}
void fts_mailbox_list_created(struct mailbox_list *list)
{
struct fts_backend *backend;
const char *name, *path, *error;
name = mail_user_plugin_getenv(list->ns->user, "fts");
if (name == NULL) {
if (list->mail_set->mail_debug)
i_debug("fts: No fts setting - plugin disabled");
return;
}
path = mailbox_list_get_path(list, NULL,
MAILBOX_LIST_PATH_TYPE_INDEX);
if (*path == '\0') {
if (list->mail_set->mail_debug) {
i_debug("fts: Indexes disabled for namespace '%s'",
list->ns->prefix);
}
return;
}
if (fts_backend_init(name, list->ns, &error, &backend) < 0) {
i_error("fts: Failed to initialize backend '%s': %s",
name, error);
} else {
struct fts_mailbox_list *flist;
struct mailbox_list_vfuncs *v = list->vlast;
flist = p_new(list->pool, struct fts_mailbox_list, 1);
flist->module_ctx.super = *v;
flist->backend = backend;
list->vlast = &flist->module_ctx.super;
v->deinit = fts_mailbox_list_deinit;
MODULE_CONTEXT_SET(list, fts_mailbox_list_module, flist);
}
}
struct fts_backend *fts_mailbox_backend(struct mailbox *box)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
return flist->backend;
}
struct fts_backend *fts_list_backend(struct mailbox_list *list)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list);
return flist == NULL ? NULL : flist->backend;
}