plugins/fts/fts-storage.c

	fts-storage.c revision a1761856683b4bf745eb4e32cefabeb851efb301
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen/* Copyright (C) 2006 Timo Sirainen */
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "lib.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "array.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "str.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "istream.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "message-parser.h"
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen#include "message-decoder.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "mail-search.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "mail-storage-private.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "fts-api-private.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "fts-plugin.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include <stdlib.h>
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#define FTS_CONTEXT(obj) \
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    *((void **)array_idx_modifiable(&(obj)->module_contexts, \
660b4d36110c44b1e4b4b45a78c22d1569ccdb54Timo Sirainen                    fts_storage_module_id))
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainenstruct fts_mailbox {
283ccfe110ed62e48f36e0d84e47da8cae5106beTimo Sirainen    struct mailbox_vfuncs super;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    struct fts_backend *backend;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    const char *env;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    unsigned int backend_failed:1;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen};
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenstruct fts_search_context {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    ARRAY_TYPE(seq_range) result;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    unsigned int result_pos;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    unsigned int locked:1;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen};
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainenstruct fts_transaction_context {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    bool expunges;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen};
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainenstruct fts_mail {
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    struct mail_vfuncs super;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen};
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainenstatic unsigned int fts_storage_module_id = 0;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainenstatic bool fts_storage_module_id_set = FALSE;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainenstatic int fts_mailbox_close(struct mailbox *box)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen{
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    struct fts_mailbox *fbox = FTS_CONTEXT(box);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    int ret;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    if (fbox->backend != NULL)
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        fts_backend_deinit(fbox->backend);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    ret = fbox->super.close(box);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    i_free(fbox);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    return ret;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen}
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainenstatic int uid_range_to_seq(struct mailbox *box,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen                ARRAY_TYPE(seq_range) *uid_range,
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen                ARRAY_TYPE(seq_range) *seq_range)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen{
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    const struct seq_range *range;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    struct seq_range new_range;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    unsigned int i, count;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    range = array_get(uid_range, &count);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    i_array_init(seq_range, count);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    for (i = 0; i < count; i++) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        if (mailbox_get_uids(box, range[i].seq1, range[i].seq2,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen                     &new_range.seq1, &new_range.seq2) < 0) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen            array_free(seq_range);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen            return -1;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        }
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        if (new_range.seq1 != 0)
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen            array_append(seq_range, &new_range, 1);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    }
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    return 0;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen}
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainenstruct fts_storage_build_context {
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    struct fts_backend_build_context *build;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    uint32_t uid;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    string_t *headers;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    bool save_part;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen};
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainenstatic int fts_build_mail_flush(struct fts_storage_build_context *ctx)
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen{
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    if (str_len(ctx->headers) == 0)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen        return 1;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    if (fts_backend_build_more(ctx->build, ctx->uid, str_data(ctx->headers),
36d2b3dc8766ef336a289a51075ca2f3236ef1efTimo Sirainen                   str_len(ctx->headers)) < 0)
36d2b3dc8766ef336a289a51075ca2f3236ef1efTimo Sirainen        return -1;
36d2b3dc8766ef336a289a51075ca2f3236ef1efTimo Sirainen
36d2b3dc8766ef336a289a51075ca2f3236ef1efTimo Sirainen    str_truncate(ctx->headers, 0);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    return 1;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen}
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainenstatic bool fts_build_update_save_part(struct fts_storage_build_context *ctx,
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen                       const struct message_block *block)
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen{
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    /* we'll index only text/xxx and message/rfc822 parts for now */
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen    if ((block->part->flags &
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen         (MESSAGE_PART_FLAG_TEXT |
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen          MESSAGE_PART_FLAG_MESSAGE_RFC822)) == 0)
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        return FALSE;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
283ccfe110ed62e48f36e0d84e47da8cae5106beTimo Sirainen    ctx->save_part = TRUE;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    return TRUE;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen}
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainenstatic int fts_build_mail_header(struct fts_storage_build_context *ctx,
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen                 const struct message_block *block)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen{
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen    const struct message_header_line *hdr = block->hdr;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    /* hdr->full_value is always set because we get the block from
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen       message_decoder */
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    str_append(ctx->headers, hdr->name);
2a4e8f370c566ffd360922227fc73d0ee36abee7Timo Sirainen    str_append_n(ctx->headers, hdr->middle, hdr->middle_len);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    str_append_n(ctx->headers, hdr->full_value, hdr->full_value_len);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    if (!hdr->no_newline)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen        str_append_c(ctx->headers, '\n');
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    if (!ctx->save_part) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        if (strcasecmp(hdr->name, "Content-Type") == 0) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen            if (!fts_build_update_save_part(ctx, block))
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen                return 0;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        }
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        return 1;
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen    }
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen
c8b29548f94160e0fe2f02e66d899d9e6cb495b3Timo Sirainen    return fts_build_mail_flush(ctx);
c8b29548f94160e0fe2f02e66d899d9e6cb495b3Timo Sirainen}
c8b29548f94160e0fe2f02e66d899d9e6cb495b3Timo Sirainen
c8b29548f94160e0fe2f02e66d899d9e6cb495b3Timo Sirainenstatic int
c8b29548f94160e0fe2f02e66d899d9e6cb495b3Timo Sirainenfts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
c8b29548f94160e0fe2f02e66d899d9e6cb495b3Timo Sirainen{
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen    struct istream *input;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    struct message_parser_ctx *parser;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    struct message_decoder_context *decoder;
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen    struct message_block raw_block, block;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    struct message_part *prev_part, *skip_part;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    int ret;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    ctx->uid = mail->uid;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    input = mail_get_stream(mail, NULL, NULL);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    if (input == NULL)
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        return -1;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen    prev_part = skip_part = NULL;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    parser = message_parser_init(pool_datastack_create(), input);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen    decoder = message_decoder_init();
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen    for (;;) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        ret = message_parser_parse_next_block(parser, &raw_block);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        i_assert(ret != 0);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen        if (ret < 0) {
            if (input->stream_errno == 0)
                ret = 0;
            break;
        }
        if (raw_block.part == skip_part)
            continue;

        if (!message_decoder_decode_next_block(decoder, &raw_block,
                               &block))
            continue;

        if (block.part != prev_part &&
            (block.hdr != NULL || block.size != 0)) {
            str_truncate(ctx->headers, 0);
            ctx->save_part = FALSE;
            prev_part = block.part;
            skip_part = NULL;
        }

        if (block.hdr != NULL) {
            ret = fts_build_mail_header(ctx, &block);
            if (ret < 0)
                break;
            if (ret == 0)
                skip_part = raw_block.part;
        } else if (block.size == 0) {
            /* end of headers */
            if (fts_build_update_save_part(ctx, &block)) {
                ret = fts_build_mail_flush(ctx);
                if (ret < 0)
                    break;
            }
        } else {
            if (fts_backend_build_more(ctx->build, mail->uid,
                           block.data,
                           block.size) < 0) {
                ret = -1;
                break;
            }
        }
    }
    (void)message_parser_deinit(&parser);
    message_decoder_deinit(&decoder);
    return ret;
}

static int fts_build_new(struct mailbox_transaction_context *t)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
    struct fts_storage_build_context ctx;
    struct mail_search_context *search_ctx;
    struct mail_search_seqset seqset;
    struct mail_search_arg search_arg;
    struct mail *mail;
    uint32_t last_uid, last_uid_locked;
    int ret = 0;

    if (fts_backend_get_last_uid(fbox->backend, &last_uid) < 0)
        return -1;

    memset(&seqset, 0, sizeof(seqset));
    if (mailbox_get_uids(t->box, last_uid+1, (uint32_t)-1,
                 &seqset.seq1, &seqset.seq2) < 0)
        return -1;
    if (seqset.seq1 == 0) {
        /* no new messages */
        return 0;
    }

    memset(&ctx, 0, sizeof(ctx));
    ctx.build = fts_backend_build_init(fbox->backend, &last_uid_locked);
    if (last_uid != last_uid_locked) {
        /* changed, need to get again the sequences */
        i_assert(last_uid < last_uid_locked);

        last_uid = last_uid_locked;
        if (mailbox_get_uids(t->box, last_uid+1, (uint32_t)-1,
                     &seqset.seq1, &seqset.seq2) < 0) {
            (void)fts_backend_build_deinit(ctx.build);
            return -1;
        }
        if (seqset.seq1 == 0) {
            /* no new messages */
            (void)fts_backend_build_deinit(ctx.build);
            return 0;
        }
    }

    memset(&search_arg, 0, sizeof(search_arg));
    search_arg.type = SEARCH_SEQSET;
    search_arg.value.seqset = &seqset;

    ctx.headers = str_new(default_pool, 512);
    mail = mail_alloc(t, 0, NULL);
    search_ctx = mailbox_search_init(t, NULL, &search_arg, NULL);
    while (mailbox_search_next(search_ctx, mail) > 0) {
        if (fts_build_mail(&ctx, mail) < 0) {
            ret = -1;
            break;
        }
    }
    if (mailbox_search_deinit(&search_ctx) < 0)
        ret = -1;
    mail_free(&mail);

    if (fts_backend_build_deinit(ctx.build) < 0)
        ret = -1;
    str_free(&ctx.headers);
    return ret;
}

static struct mail_search_context *
fts_mailbox_search_init(struct mailbox_transaction_context *t,
            const char *charset, struct mail_search_arg *args,
            const enum mail_sort_type *sort_program)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
    struct mail_search_context *ctx;
    struct fts_search_context *fctx;
    ARRAY_TYPE(seq_range) uid_result;

    ctx = fbox->super.search_init(t, charset, args, sort_program);

    fctx = i_new(struct fts_search_context, 1);
    array_idx_set(&ctx->module_contexts, fts_storage_module_id, &fctx);

    if (fbox->backend == NULL)
        return ctx;

    /* FIXME: handle AND/OR. Maybe also header lookups? */
    while (args != NULL &&
           args->type != SEARCH_BODY &&
           args->type != SEARCH_TEXT)
        args = args->next;

    if (args != NULL) {
        if (fts_build_new(t) < 0)
            return ctx;

        if (fts_backend_lock(fbox->backend) <= 0)
            return ctx;
        fctx->locked = TRUE;

        i_array_init(&uid_result, 64);
        if (fts_backend_lookup(fbox->backend, args->value.str,
                       &uid_result) < 0) {
            /* failed, fallback to reading everything */
            array_free(&uid_result);
        }

        if (fbox->backend->definite_lookups) {
            args->match_always = TRUE;
            args->result = 1;
        }
        args = args->next;
        while (args != NULL) {
            if (args->type == SEARCH_BODY ||
                args->type == SEARCH_TEXT) {
                if (fbox->backend->definite_lookups) {
                    args->match_always = TRUE;
                    args->result = 1;
                }
                if (fts_backend_filter(fbox->backend,
                               args->value.str,
                               &uid_result) < 0) {
                    /* failed, but we already have limited
                       the search, so just ignore this */
                    break;
                }
            }
            args = args->next;
        }

        if (array_is_created(&uid_result)) {
            (void)uid_range_to_seq(t->box, &uid_result,
                           &fctx->result);
            array_free(&uid_result);
        }
    }
    return ctx;
}

static int fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);
    struct seq_range *range;
    unsigned int count;
    uint32_t wanted_seq;
    int ret;

    if (!array_is_created(&fctx->result))
        return fbox->super.search_next_update_seq(ctx);

    do {
        range = array_get_modifiable(&fctx->result, &count);
        while (fctx->result_pos < count &&
               ctx->seq > range[fctx->result_pos].seq2)
            fctx->result_pos++;

        if (fctx->result_pos == count)
            return 0;

        if (ctx->seq > range[fctx->result_pos].seq1)
            range[fctx->result_pos].seq1 = ctx->seq+1;
        else {
            ctx->seq = range[fctx->result_pos].seq1 - 1;

            if (fctx->result_pos < count &&
                ctx->seq + 1 == range[fctx->result_pos].seq2)
                fctx->result_pos++;
            else
                range[fctx->result_pos].seq1++;
        }

        wanted_seq = ctx->seq + 1;
        ret = fbox->super.search_next_update_seq(ctx);
    } while (ret > 0 && wanted_seq != ctx->seq);

    return ret;
}

static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);

    if (fctx->locked)
        fts_backend_unlock(fbox->backend);

    if (array_is_created(&fctx->result))
        array_free(&fctx->result);
    i_free(fctx);
    return fbox->super.search_deinit(ctx);
}

static int fts_mail_expunge(struct mail *_mail)
{
    struct mail_private *mail = (struct mail_private *)_mail;
    struct fts_mail *fmail = FTS_CONTEXT(mail);
    struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
    struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);

    if (fmail->super.expunge(_mail) < 0)
        return -1;

    ft->expunges = TRUE;
    fts_backend_expunge(fbox->backend, _mail);
    return 0;
}

static struct mail *
fts_mail_alloc(struct mailbox_transaction_context *t,
           enum mail_fetch_field wanted_fields,
           struct mailbox_header_lookup_ctx *wanted_headers)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
    struct fts_mail *fmail;
    struct mail *_mail;
    struct mail_private *mail;

    _mail = fbox->super.mail_alloc(t, wanted_fields, wanted_headers);
    if (fbox->backend != NULL) {
        mail = (struct mail_private *)_mail;

        fmail = p_new(mail->pool, struct fts_mail, 1);
        fmail->super = mail->v;

        mail->v.expunge = fts_mail_expunge;
        array_idx_set(&mail->module_contexts,
                  fts_storage_module_id, &fmail);
    }
    return _mail;
}

static struct mailbox_transaction_context *
fts_transaction_begin(struct mailbox *box,
              enum mailbox_transaction_flags flags)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct mailbox_transaction_context *t;
    struct fts_transaction_context *ft;

    ft = i_new(struct fts_transaction_context, 1);

    /* the backend creation is delayed until the first transaction is
       started. at that point the mailbox has been synced at least once. */
    if (fbox->backend == NULL && !fbox->backend_failed) {
        fbox->backend = fts_backend_init(fbox->env, box);
        if (fbox->backend == NULL)
            fbox->backend_failed = TRUE;
    }

    t = fbox->super.transaction_begin(box, flags);
    array_idx_set(&t->module_contexts, fts_storage_module_id, &ft);
    return t;
}

static void fts_transaction_rollback(struct mailbox_transaction_context *t)
{
    struct mailbox *box = t->box;
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct fts_transaction_context *ft = FTS_CONTEXT(t);

    fbox->super.transaction_rollback(t);
    if (ft->expunges)
        fts_backend_expunge_finish(fbox->backend, box, FALSE);
    i_free(ft);
}

static int fts_transaction_commit(struct mailbox_transaction_context *t,
                  enum mailbox_sync_flags flags)
{
    struct mailbox *box = t->box;
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct fts_transaction_context *ft = FTS_CONTEXT(t);
    int ret;

    ret = fbox->super.transaction_commit(t, flags);
    if (ft->expunges)
        fts_backend_expunge_finish(fbox->backend, box, ret == 0);
    i_free(ft);
    return ret;
}

void fts_mailbox_opened(struct mailbox *box)
{
    struct fts_mailbox *fbox;
    const char *env;

    if (fts_next_hook_mailbox_opened != NULL)
        fts_next_hook_mailbox_opened(box);

    env = getenv("FTS");
    if (env == NULL)
        return;

    fbox = i_new(struct fts_mailbox, 1);
    fbox->env = env;
    fbox->super = box->v;
    box->v.close = fts_mailbox_close;
    box->v.search_init = fts_mailbox_search_init;
    box->v.search_next_update_seq = fts_mailbox_search_next_update_seq;
    box->v.search_deinit = fts_mailbox_search_deinit;
    box->v.mail_alloc = fts_mail_alloc;
    box->v.transaction_begin = fts_transaction_begin;
    box->v.transaction_rollback = fts_transaction_rollback;
    box->v.transaction_commit = fts_transaction_commit;

    if (!fts_storage_module_id_set) {
        fts_storage_module_id = mail_storage_module_id++;
        fts_storage_module_id_set = TRUE;
    }

    array_idx_set(&box->module_contexts, fts_storage_module_id, &fbox);
}