plugins/fts/fts-storage.c

	fts-storage.c revision fe3f637ef0b9544a567b54f3743cf8e3649d8d0f
bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2006-2015 Dovecot authors, see the included COPYING file */
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "lib.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "array.h"
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen#include "net.h"
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen#include "str.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "strescape.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "write-full.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "mail-search-build.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "mail-storage-private.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "mailbox-list-private.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "../virtual/virtual-storage.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "fts-api-private.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "fts-tokenizer.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "fts-indexer.h"
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen#include "fts-build-mail.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "fts-search-serialize.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include "fts-plugin.h"
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi#include "fts-storage.h"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#include <stdlib.h>
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#define FTS_CONTEXT(obj) \
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    MODULE_CONTEXT(obj, fts_storage_module)
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#define FTS_MAIL_CONTEXT(obj) \
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    MODULE_CONTEXT(obj, fts_mail_module)
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#define FTS_LIST_CONTEXT(obj) \
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    MODULE_CONTEXT(obj, fts_mailbox_list_module)
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#define INDEXER_SOCKET_NAME "indexer"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainenstruct fts_mailbox_list {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    union mailbox_list_module_context module_ctx;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    struct fts_backend *backend;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    struct fts_backend_update_context *update_ctx;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    unsigned int update_ctx_refcount;
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen};
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainenstruct fts_mailbox {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    union mailbox_module_context module_ctx;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    struct fts_backend_update_context *sync_update_ctx;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen};
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainenstruct fts_transaction_context {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    union mailbox_transaction_module_context module_ctx;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    struct fts_scores *scores;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    uint32_t next_index_seq;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    uint32_t highest_virtual_uid;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    unsigned int precached:1;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    unsigned int mails_saved:1;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    unsigned int failed:1;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen};
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainenstruct fts_mail {
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    union mail_module_context module_ctx;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    char score[30];
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    unsigned int virtual_mail:1;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen};
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_storage_module,
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen                  &mail_storage_module_register);
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register);
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module,
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen                  &mailbox_list_module_register);
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainenstatic int fts_mailbox_get_last_cached_seq(struct mailbox *box, uint32_t *seq_r)
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen{
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    uint32_t seq1, seq2, last_uid;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    if (fts_backend_get_last_uid(flist->backend, box, &last_uid) < 0) {
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen        mail_storage_set_internal_error(box->storage);
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen        return -1;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    }
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    if (last_uid == 0)
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen        *seq_r = 0;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    else {
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen        mailbox_get_seq_range(box, 1, last_uid, &seq1, &seq2);
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen        *seq_r = seq2;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    }
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen    return 0;
7a60e1dc9e93ef3f7c7fe1af6385a0bfa1e31bc3Timo Sirainen}
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainenstatic int
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainenfts_mailbox_get_status(struct mailbox *box, enum mailbox_status_items items,
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen               struct mailbox_status *status_r)
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen{
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    struct fts_mailbox *fbox = FTS_CONTEXT(box);
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    uint32_t seq;
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    if (fbox->module_ctx.super.get_status(box, items, status_r) < 0)
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen        return -1;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    if ((items & STATUS_LAST_CACHED_SEQ) != 0) {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        if (fts_mailbox_get_last_cached_seq(box, &seq) < 0)
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen            return -1;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        /* use whichever is smaller */
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        if (status_r->last_cached_seq > seq)
02aedbc20af0160091670233383d228f10b168afTimo Sirainen            status_r->last_cached_seq = seq;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    }
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    return 0;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen}
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainenstatic void fts_scores_unref(struct fts_scores **_scores)
02aedbc20af0160091670233383d228f10b168afTimo Sirainen{
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    struct fts_scores *scores = *_scores;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    *_scores = NULL;
cbcc083b53cc2e263f039edd4e4c7cb74830651eTimo Sirainen    if (--scores->refcount == 0) {
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen        array_free(&scores->score_map);
adad68df4602d80639d7f28e4c7550fbafbd8d60Timo Sirainen        i_free(scores);
cbcc083b53cc2e263f039edd4e4c7cb74830651eTimo Sirainen    }
cbcc083b53cc2e263f039edd4e4c7cb74830651eTimo Sirainen}
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainenstatic void fts_try_build_init(struct mail_search_context *ctx,
02aedbc20af0160091670233383d228f10b168afTimo Sirainen                   struct fts_search_context *fctx)
ac1118842c3d80285e32d2cd53bda3e95e5be217Timo Sirainen{
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    int ret;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    i_assert(!fts_backend_is_updating(fctx->backend));
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    ret = fts_indexer_init(fctx->backend, ctx->transaction->box,
02aedbc20af0160091670233383d228f10b168afTimo Sirainen                   &fctx->indexer_ctx);
350d6194f7336f3c89d641a01b31c7417d67b08aTimo Sirainen    if (ret < 0)
350d6194f7336f3c89d641a01b31c7417d67b08aTimo Sirainen        return;
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    if (ret == 0) {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        /* the index was up to date */
ac1118842c3d80285e32d2cd53bda3e95e5be217Timo Sirainen        fts_search_lookup(fctx);
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi    } else {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        /* hide "searching" notifications while building index */
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        ctx->progress_hidden = TRUE;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    }
02aedbc20af0160091670233383d228f10b168afTimo Sirainen}
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
ac1118842c3d80285e32d2cd53bda3e95e5be217Timo Sirainenstatic bool fts_want_build_args(const struct mail_search_arg *args)
02aedbc20af0160091670233383d228f10b168afTimo Sirainen{
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    /* we want to update index only when searching from message body.
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen       it's not worth the wait for searching only from headers, which
02aedbc20af0160091670233383d228f10b168afTimo Sirainen       could be in cache file already */
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen    for (; args != NULL; args = args->next) {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        switch (args->type) {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        case SEARCH_OR:
ac1118842c3d80285e32d2cd53bda3e95e5be217Timo Sirainen        case SEARCH_SUB:
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        case SEARCH_INTHREAD:
02aedbc20af0160091670233383d228f10b168afTimo Sirainen            if (fts_want_build_args(args->value.subargs))
02aedbc20af0160091670233383d228f10b168afTimo Sirainen                return TRUE;
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi            break;
1f58bee818868d7261224766b66182c8eaf31ba8Aki Tuomi        case SEARCH_BODY:
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi        case SEARCH_TEXT:
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen            return TRUE;
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen        default:
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen            break;
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen        }
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen    }
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen    return FALSE;
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen}
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainenstatic bool fts_args_have_fuzzy(const struct mail_search_arg *args)
9fc9965693110c250c6d6aa36ab60a77c934cc49Timo Sirainen{
9fc9965693110c250c6d6aa36ab60a77c934cc49Timo Sirainen    for (; args != NULL; args = args->next) {
9fc9965693110c250c6d6aa36ab60a77c934cc49Timo Sirainen        if (args->fuzzy)
9fc9965693110c250c6d6aa36ab60a77c934cc49Timo Sirainen            return TRUE;
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen        switch (args->type) {
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        case SEARCH_OR:
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        case SEARCH_SUB:
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        case SEARCH_INTHREAD:
02aedbc20af0160091670233383d228f10b168afTimo Sirainen            if (fts_args_have_fuzzy(args->value.subargs))
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi                return TRUE;
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi            break;
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi        default:
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi            break;
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi        }
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi    }
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi    return FALSE;
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi}
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomistatic struct mail_search_context *
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomifts_mailbox_search_init(struct mailbox_transaction_context *t,
9fc9965693110c250c6d6aa36ab60a77c934cc49Timo Sirainen            struct mail_search_args *args,
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi            const enum mail_sort_type *sort_program,
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi            enum mail_fetch_field wanted_fields,
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi            struct mailbox_header_lookup_ctx *wanted_headers)
75af6e5ca2c4685e2316d27364ac7b00def7fed7Timo Sirainen{
9fc9965693110c250c6d6aa36ab60a77c934cc49Timo Sirainen    struct fts_transaction_context *ft = FTS_CONTEXT(t);
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi    struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(t->box->list);
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    struct mail_search_context *ctx;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    struct fts_search_context *fctx;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    ctx = fbox->module_ctx.super.search_init(t, args, sort_program,
02aedbc20af0160091670233383d228f10b168afTimo Sirainen                         wanted_fields, wanted_headers);
02aedbc20af0160091670233383d228f10b168afTimo Sirainen
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    if (!fts_backend_can_lookup(flist->backend, args->args))
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi        return ctx;
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    fctx = i_new(struct fts_search_context, 1);
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    fctx->box = t->box;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    fctx->backend = flist->backend;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    fctx->t = t;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    fctx->args = args;
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    fctx->result_pool = pool_alloconly_create("fts results", 1024*64);
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
02aedbc20af0160091670233383d228f10b168afTimo Sirainen    fctx->virtual_mailbox =
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
76537b1991e7815c7a867a997f7fa2b3c17412d4Aki Tuomi    fctx->enforced =
02aedbc20af0160091670233383d228f10b168afTimo Sirainen        mail_user_plugin_getenv(t->box->storage->user,
                    "fts_enforced") != NULL;
    i_array_init(&fctx->levels, 8);
    fctx->scores = i_new(struct fts_scores, 1);
    fctx->scores->refcount = 1;
    i_array_init(&fctx->scores->score_map, 64);
    MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);

    /* FIXME: we'll assume that all the args are fuzzy. not good,
       but would require much more work to fix it. */
    if (!fts_args_have_fuzzy(args->args) &&
        mail_user_plugin_getenv(t->box->storage->user,
                    "fts_no_autofuzzy") != NULL)
        fctx->flags |= FTS_LOOKUP_FLAG_NO_AUTO_FUZZY;
    /* transaction contains the last search's scores. they can be
       queried later with mail_get_special() */
    if (ft->scores != NULL)
        fts_scores_unref(&ft->scores);
    ft->scores = fctx->scores;
    ft->scores->refcount++;

    if (fctx->enforced || fts_want_build_args(args->args))
        fts_try_build_init(ctx, fctx);
    else
        fts_search_lookup(fctx);
    return ctx;
}

static bool fts_mailbox_build_continue(struct mail_search_context *ctx)
{
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);
    int ret;

    ret = fts_indexer_more(fctx->indexer_ctx);
    if (ret == 0)
        return FALSE;

    /* indexing finished */
    ctx->progress_hidden = FALSE;
    if (fts_indexer_deinit(&fctx->indexer_ctx) < 0)
        ret = -1;
    if (ret > 0)
        fts_search_lookup(fctx);
    if (ret < 0) {
        /* if indexing timed out, it probably means that
           the mailbox is still being indexed, but it's a large
           mailbox and it takes a while. in this situation
           we'll simply abort the search.

           if indexing failed for any other reason, just
           fallback to searching the slow way. */
        fctx->indexing_timed_out =
            mailbox_get_last_mail_error(fctx->box) == MAIL_ERROR_INUSE;
    }
    return TRUE;
}

static bool
fts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
                 struct mail **mail_r, bool *tryagain_r)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);
    struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);

    if (fctx == NULL && ft->failed) {
        /* precaching already failed - stop now instead of potentially
           going through the same failure for all the mails */
        return FALSE;
    }

    if (fctx != NULL && fctx->indexer_ctx != NULL) {
        /* this command is still building the indexes */
        if (!fts_mailbox_build_continue(ctx)) {
            *tryagain_r = TRUE;
            return FALSE;
        }
        if (fctx->indexing_timed_out) {
            *tryagain_r = FALSE;
            return FALSE;
        }
    }
    if (fctx != NULL && !fctx->fts_lookup_success && fctx->enforced)
        return FALSE;

    return fbox->module_ctx.super.
        search_next_nonblock(ctx, mail_r, tryagain_r);
}

static void
fts_search_apply_results_level(struct mail_search_context *ctx,
                   struct mail_search_arg *args, unsigned int *idx)
{
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);
    const struct fts_search_level *level;

    level = array_idx(&fctx->levels, *idx);

    if (array_is_created(&level->definite_seqs) &&
        seq_range_exists(&level->definite_seqs, ctx->seq))
        fts_search_deserialize_add_matches(args, level->args_matches);
    else if (!array_is_created(&level->maybe_seqs) ||
         !seq_range_exists(&level->maybe_seqs, ctx->seq))
        fts_search_deserialize_add_nonmatches(args, level->args_matches);

    for (; args != NULL; args = args->next) {
        if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
            continue;

        *idx += 1;
        fts_search_apply_results_level(ctx, args->value.subargs, idx);
    }
}

static bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);
    unsigned int idx;

    if (fctx == NULL || !fctx->fts_lookup_success) {
        /* fts lookup not done for this search */
        if (fctx != NULL && fctx->indexing_timed_out)
            return FALSE;
        return fbox->module_ctx.super.search_next_update_seq(ctx);
    }

    /* restore original [non]matches */
    fts_search_deserialize(ctx->args->args, fctx->orig_matches);

    if (!fbox->module_ctx.super.search_next_update_seq(ctx))
        return FALSE;

    if (ctx->seq >= fctx->first_unindexed_seq) {
        /* we've not indexed this far */
        return TRUE;
    }

    /* apply [non]matches based on the FTS lookup results */
    idx = 0;
    fts_search_apply_results_level(ctx, ctx->args->args, &idx);
    return TRUE;
}

static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
    struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);
    int ret = 0;

    if (fctx != NULL) {
        if (fctx->indexer_ctx != NULL) {
            if (fts_indexer_deinit(&fctx->indexer_ctx) < 0)
                ft->failed = TRUE;
        }
        if (fctx->indexing_timed_out)
            ret = -1;
        if (!fctx->fts_lookup_success && fctx->enforced) {
            /* FTS lookup failed and we didn't want to fallback to
               opening all the mails and searching manually */
            mail_storage_set_internal_error(ctx->transaction->box->storage);
            ret = -1;
        }

        buffer_free(&fctx->orig_matches);
        array_free(&fctx->levels);
        pool_unref(&fctx->result_pool);
        fts_scores_unref(&fctx->scores);
        i_free(fctx);
    } else {
        if (ft->failed)
            ret = -1;
    }
    if (fbox->module_ctx.super.search_deinit(ctx) < 0)
        ret = -1;
    return ret;
}

static int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score)
{
    return *uid < score->uid ? -1 :
        (*uid > score->uid ? 1 : 0);
}

static int fts_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
                const char **value_r)
{
    struct mail_private *mail = (struct mail_private *)_mail;
    struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
    struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
    const struct fts_score_map *scores;

    if (field != MAIL_FETCH_SEARCH_RELEVANCY || ft->scores == NULL)
        scores = NULL;
    else {
        scores = array_bsearch(&ft->scores->score_map, &_mail->uid,
                       fts_score_cmp);
    }
    if (scores != NULL) {
        i_assert(scores->uid == _mail->uid);
        (void)i_snprintf(fmail->score, sizeof(fmail->score),
                 "%f", scores->score);

        *value_r = fmail->score;
        return 0;
    }

    return fmail->module_ctx.super.get_special(_mail, field, value_r);
}

static int
fts_mail_precache_range(struct mailbox_transaction_context *trans,
            struct fts_backend_update_context *update_ctx,
            uint32_t seq1, uint32_t seq2)
{
    struct mail_search_args *search_args;
    struct mail_search_context *ctx;
    struct mail *mail;
    int ret = 0;

    search_args = mail_search_build_init();
    mail_search_build_add_seqset(search_args, seq1, seq2);
    ctx = mailbox_search_init(trans, search_args, NULL,
                  MAIL_FETCH_STREAM_HEADER |
                  MAIL_FETCH_STREAM_BODY, NULL);
    mail_search_args_unref(&search_args);

    while (mailbox_search_next(ctx, &mail)) {
        if (fts_build_mail(update_ctx, mail) < 0) {
            mail_storage_set_internal_error(trans->box->storage);
            ret = -1;
            break;
        }
        mail_precache(mail);
    }
    if (mailbox_search_deinit(&ctx) < 0)
        ret = -1;
    return ret;
}

static int fts_mail_precache_init(struct mail *_mail)
{
    struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(_mail->box->list);
    uint32_t last_seq;

    if (fts_mailbox_get_last_cached_seq(_mail->box, &last_seq) < 0)
        return -1;

    ft->precached = TRUE;
    ft->next_index_seq = last_seq + 1;
    if (flist->update_ctx == NULL)
        flist->update_ctx = fts_backend_update_init(flist->backend);
    flist->update_ctx_refcount++;
    return 0;
}

static void fts_mail_index(struct mail *_mail)
{
    struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(_mail->box->list);

    if (ft->failed)
        return;

    if (!ft->precached) {
        if (fts_mail_precache_init(_mail) < 0) {
            ft->failed = TRUE;
            return;
        }
    }
    if (ft->next_index_seq < _mail->seq) {
        /* most likely a virtual mailbox. we'll first need to
           index all mails up to the current one. */
        fts_backend_update_set_mailbox(flist->update_ctx, _mail->box);
        if (fts_mail_precache_range(_mail->transaction,
                        flist->update_ctx,
                        ft->next_index_seq,
                        _mail->seq-1) < 0) {
            ft->failed = TRUE;
            return;
        }
    }

    if (ft->next_index_seq == _mail->seq) {
        fts_backend_update_set_mailbox(flist->update_ctx, _mail->box);
        if (fts_build_mail(flist->update_ctx, _mail) < 0) {
            mail_storage_set_internal_error(_mail->box->storage);
            ft->failed = TRUE;
        }
        ft->next_index_seq = _mail->seq + 1;
    }
}

static void fts_mail_precache(struct mail *_mail)
{
    struct mail_private *mail = (struct mail_private *)_mail;
    struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
    struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);

    fmail->module_ctx.super.precache(_mail);
    if (fmail->virtual_mail) {
        if (ft->highest_virtual_uid < _mail->uid)
            ft->highest_virtual_uid = _mail->uid;
    } else T_BEGIN {
        fts_mail_index(_mail);
    } T_END;
}

void fts_mail_allocated(struct mail *_mail)
{
    struct mail_private *mail = (struct mail_private *)_mail;
    struct mail_vfuncs *v = mail->vlast;
    struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
    struct fts_mail *fmail;

    if (fbox == NULL)
        return;

    fmail = p_new(mail->pool, struct fts_mail, 1);
    fmail->module_ctx.super = *v;
    mail->vlast = &fmail->module_ctx.super;
    fmail->virtual_mail =
        strcmp(_mail->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;

    v->get_special = fts_mail_get_special;
    v->precache = fts_mail_precache;
    MODULE_CONTEXT_SET(mail, fts_mail_module, fmail);
}

static struct mailbox_transaction_context *
fts_transaction_begin(struct mailbox *box,
              enum mailbox_transaction_flags flags)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct mailbox_transaction_context *t;
    struct fts_transaction_context *ft;

    ft = i_new(struct fts_transaction_context, 1);

    t = fbox->module_ctx.super.transaction_begin(box, flags);
    MODULE_CONTEXT_SET(t, fts_storage_module, ft);
    return t;
}

static int fts_transaction_end(struct mailbox_transaction_context *t)
{
    struct fts_transaction_context *ft = FTS_CONTEXT(t);
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(t->box->list);
    int ret = ft->failed ? -1 : 0;

    if (ft->precached) {
        i_assert(flist->update_ctx_refcount > 0);
        if (--flist->update_ctx_refcount == 0) {
            if (fts_backend_update_deinit(&flist->update_ctx) < 0)
                ret = -1;
        }
    } else if (ft->highest_virtual_uid > 0) {
        if (fts_index_set_last_uid(t->box, ft->highest_virtual_uid) < 0)
            ret = -1;
    }
    if (ft->scores != NULL)
        fts_scores_unref(&ft->scores);
    i_free(ft);
    return ret;
}

static void fts_transaction_rollback(struct mailbox_transaction_context *t)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(t->box);

    (void)fts_transaction_end(t);
    fbox->module_ctx.super.transaction_rollback(t);
}

static void fts_queue_index(struct mailbox *box)
{
    struct mail_user *user = box->storage->user;
    string_t *str = t_str_new(256);
    const char *path, *value;
    unsigned int max_recent_msgs;
    int fd;

    path = t_strconcat(user->set->base_dir, "/"INDEXER_SOCKET_NAME, NULL);
    fd = net_connect_unix(path);
    if (fd == -1) {
        i_error("net_connect_unix(%s) failed: %m", path);
        return;
    }

    value = mail_user_plugin_getenv(user, "fts_autoindex_max_recent_msgs");
    if (value == NULL || str_to_uint(value, &max_recent_msgs) < 0)
        max_recent_msgs = 0;

    str_append(str, INDEXER_HANDSHAKE);
    str_append(str, "APPEND\t0\t");
    str_append_tabescaped(str, user->username);
    str_append_c(str, '\t');
    str_append_tabescaped(str, box->vname);
    str_printfa(str, "\t%u\n", max_recent_msgs);
    if (write_full(fd, str_data(str), str_len(str)) < 0)
        i_error("write(%s) failed: %m", path);
    i_close_fd(&fd);
}

static int
fts_transaction_commit(struct mailbox_transaction_context *t,
               struct mail_transaction_commit_changes *changes_r)
{
    struct fts_transaction_context *ft = FTS_CONTEXT(t);
    struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
    struct mailbox *box = t->box;
    bool autoindex;
    int ret = 0;

    autoindex = ft->mails_saved &&
        mail_user_plugin_getenv(box->storage->user,
                    "fts_autoindex") != NULL;

    if (fts_transaction_end(t) < 0) {
        mail_storage_set_error(t->box->storage, MAIL_ERROR_TEMP,
                       "FTS transaction commit failed");
        ret = -1;
    }
    if (fbox->module_ctx.super.transaction_commit(t, changes_r) < 0)
        ret = -1;
    if (ret < 0)
        return -1;

    if (autoindex)
        fts_queue_index(box);
    return 0;
}

static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
                    enum mailbox_sync_type sync_type)
{
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
    struct fts_mailbox *fbox = FTS_CONTEXT(box);

    if (fbox->module_ctx.super.sync_notify != NULL)
        fbox->module_ctx.super.sync_notify(box, uid, sync_type);

    if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE) {
        if (uid == 0 && fbox->sync_update_ctx != NULL) {
            /* this sync is finished */
            (void)fts_backend_update_deinit(&fbox->sync_update_ctx);
        }
        return;
    }

    if (fbox->sync_update_ctx == NULL) {
        if (fts_backend_is_updating(flist->backend)) {
            /* FIXME: maildir workaround - we could get here
               because we're building an index, which doesn't find
               some mail and starts syncing the mailbox.. */
            return;
        }
        fbox->sync_update_ctx = fts_backend_update_init(flist->backend);
        fts_backend_update_set_mailbox(fbox->sync_update_ctx, box);
    }
    fts_backend_update_expunge(fbox->sync_update_ctx, uid);
}

static int fts_sync_deinit(struct mailbox_sync_context *ctx,
               struct mailbox_sync_status *status_r)
{
    struct mailbox *box = ctx->box;
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
    bool optimize;
    int ret = 0;

    optimize = (ctx->flags & (MAILBOX_SYNC_FLAG_FORCE_RESYNC |
                  MAILBOX_SYNC_FLAG_OPTIMIZE)) != 0;
    if (fbox->module_ctx.super.sync_deinit(ctx, status_r) < 0)
        return -1;
    ctx = NULL;

    if (optimize) {
        if (fts_backend_optimize(flist->backend) < 0) {
            mail_storage_set_critical(box->storage,
                "FTS optimize for mailbox %s failed",
                box->vname);
            ret = -1;
        }
    }
    return ret;
}

static int fts_save_finish(struct mail_save_context *ctx)
{
    struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);

    if (fbox->module_ctx.super.save_finish(ctx) < 0)
        return -1;
    ft->mails_saved = TRUE;
    return 0;
}

static int fts_copy(struct mail_save_context *ctx, struct mail *mail)
{
    struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);

    if (fbox->module_ctx.super.copy(ctx, mail) < 0)
        return -1;
    ft->mails_saved = TRUE;
    return 0;
}

void fts_mailbox_allocated(struct mailbox *box)
{
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
    struct mailbox_vfuncs *v = box->vlast;
    struct fts_mailbox *fbox;

    if (flist == NULL)
        return;

    fbox = p_new(box->pool, struct fts_mailbox, 1);
    fbox->module_ctx.super = *v;
    box->vlast = &fbox->module_ctx.super;

    v->get_status = fts_mailbox_get_status;
    v->search_init = fts_mailbox_search_init;
    v->search_next_nonblock = fts_mailbox_search_next_nonblock;
    v->search_next_update_seq = fts_mailbox_search_next_update_seq;
    v->search_deinit = fts_mailbox_search_deinit;
    v->transaction_begin = fts_transaction_begin;
    v->transaction_rollback = fts_transaction_rollback;
    v->transaction_commit = fts_transaction_commit;
    v->sync_notify = fts_mailbox_sync_notify;
    v->sync_deinit = fts_sync_deinit;
    v->save_finish = fts_save_finish;
    v->copy = fts_copy;

    MODULE_CONTEXT_SET(box, fts_storage_module, fbox);
}

static void fts_mailbox_list_deinit(struct mailbox_list *list)
{
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list);

    fts_backend_deinit(&flist->backend);
    flist->module_ctx.super.deinit(list);
}


static void
fts_mailbox_list_init(struct mailbox_list *list, const char *name)
{
    struct fts_backend *backend;
    const char *path, *error;

    if (!mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_INDEX, &path)) {
        if (list->mail_set->mail_debug) {
            i_debug("fts: Indexes disabled for namespace '%s'",
                list->ns->prefix);
        }
        return;
    }

    if (fts_backend_init(name, list->ns, &error, &backend) < 0) {
        i_error("fts: Failed to initialize backend '%s': %s",
            name, error);
    } else {
        struct fts_mailbox_list *flist;
        struct mailbox_list_vfuncs *v = list->vlast;

        if ((backend->flags & FTS_BACKEND_FLAG_FUZZY_SEARCH) != 0)
            list->ns->user->fuzzy_search = TRUE;

        flist = p_new(list->pool, struct fts_mailbox_list, 1);
        flist->module_ctx.super = *v;
        flist->backend = backend;
        list->vlast = &flist->module_ctx.super;
        v->deinit = fts_mailbox_list_deinit;
        MODULE_CONTEXT_SET(list, fts_mailbox_list_module, flist);
    }
}

void fts_mail_namespaces_added(struct mail_namespace *namespaces)
{
    struct mail_namespace *ns;
    const char *name;

    name = mail_user_plugin_getenv(namespaces->user, "fts");
    if (name == NULL) {
        if (namespaces->user->mail_debug)
            i_debug("fts: No fts setting - plugin disabled");
        return;
    }

    for (ns = namespaces; ns != NULL; ns = ns->next)
        fts_mailbox_list_init(ns->list, name);
}

struct fts_backend *fts_mailbox_backend(struct mailbox *box)
{
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);

    return flist->backend;
}

struct fts_backend *fts_list_backend(struct mailbox_list *list)
{
    struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list);

    return flist == NULL ? NULL : flist->backend;
}