plugins/fts/fts-storage.c

	fts-storage.c revision f0bf9880c05266114a9b85bf95b72f6cdf83b901
76b43e4417bab52e913da39b5f5bc2a130d3f149Timo Sirainen/* Copyright (C) 2006 Timo Sirainen */
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "lib.h"
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen#include "array.h"
ff487c974815bdaa2d05a3b834f4c2c841f4cc34Timo Sirainen#include "str.h"
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen#include "istream.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "message-parser.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "message-decoder.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "mail-search.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "mail-storage-private.h"
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen#include "fts-api-private.h"
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen#include "fts-plugin.h"
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen#include <stdlib.h>
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen#define FTS_CONTEXT(obj) \
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    *((void **)array_idx_modifiable(&(obj)->module_contexts, \
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                    fts_storage_module_id))
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen#define FTS_SEARCH_NONBLOCK_COUNT 10
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainenstruct fts_mailbox {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct mailbox_vfuncs super;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct fts_backend *backend_exact;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct fts_backend *backend_fast;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    const char *env;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    unsigned int backend_set:1;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen};
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainenstruct fts_search_context {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    ARRAY_TYPE(seq_range) result;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    unsigned int result_pos;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct mail_search_arg *args, *best_arg;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct fts_backend *backend;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct fts_storage_build_context *build_ctx;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    unsigned int locked:1;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen};
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainenstruct fts_storage_build_context {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct mail_search_context *search_ctx;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct mail_search_seqset seqset;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct mail_search_arg search_arg;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct mail *mail;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct fts_backend_build_context *build;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen    uint32_t uid;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    string_t *headers;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    bool save_part;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen};
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen
a8e132559a7ebe54c8269d79ce29fa3338c76199Timo Sirainenstruct fts_transaction_context {
7d97b8b0e47ad995915373b5c2a57f622a393352Timo Sirainen    bool expunges;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen};
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenstruct fts_mail {
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    struct mail_vfuncs super;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen};
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenstatic unsigned int fts_storage_module_id = 0;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenstatic bool fts_storage_module_id_set = FALSE;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenstatic int fts_mailbox_close(struct mailbox *box)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen{
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    struct fts_mailbox *fbox = FTS_CONTEXT(box);
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    int ret;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    if (fbox->backend_exact != NULL)
b4f2560c29dacd066ba89e782d95ceed7ac473a3Timo Sirainen        fts_backend_deinit(fbox->backend_exact);
b4f2560c29dacd066ba89e782d95ceed7ac473a3Timo Sirainen    if (fbox->backend_fast != NULL)
b4f2560c29dacd066ba89e782d95ceed7ac473a3Timo Sirainen        fts_backend_deinit(fbox->backend_fast);
b4f2560c29dacd066ba89e782d95ceed7ac473a3Timo Sirainen
b4f2560c29dacd066ba89e782d95ceed7ac473a3Timo Sirainen    ret = fbox->super.close(box);
b4f2560c29dacd066ba89e782d95ceed7ac473a3Timo Sirainen    i_free(fbox);
b4f2560c29dacd066ba89e782d95ceed7ac473a3Timo Sirainen    return ret;
b4f2560c29dacd066ba89e782d95ceed7ac473a3Timo Sirainen}
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenstatic int uid_range_to_seq(struct mailbox *box,
ac713658d206e8d001fef7c0e36945793f2eb942Timo Sirainen                ARRAY_TYPE(seq_range) *uid_range,
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen                ARRAY_TYPE(seq_range) *seq_range)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen{
ac713658d206e8d001fef7c0e36945793f2eb942Timo Sirainen    const struct seq_range *range;
ac713658d206e8d001fef7c0e36945793f2eb942Timo Sirainen    struct seq_range new_range;
ac713658d206e8d001fef7c0e36945793f2eb942Timo Sirainen    unsigned int i, count;
ac713658d206e8d001fef7c0e36945793f2eb942Timo Sirainen
ac713658d206e8d001fef7c0e36945793f2eb942Timo Sirainen    range = array_get(uid_range, &count);
ac713658d206e8d001fef7c0e36945793f2eb942Timo Sirainen    i_array_init(seq_range, count);
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    for (i = 0; i < count; i++) {
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen        if (mailbox_get_uids(box, range[i].seq1, range[i].seq2,
517d1e7142d57299c733b30423e35e7e1f8d01d6Timo Sirainen                     &new_range.seq1, &new_range.seq2) < 0) {
517d1e7142d57299c733b30423e35e7e1f8d01d6Timo Sirainen            array_free(seq_range);
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen            return -1;
d368bfd671ae6d04a69eb7f418521d49b8bbf77aTimo Sirainen        }
d368bfd671ae6d04a69eb7f418521d49b8bbf77aTimo Sirainen
d368bfd671ae6d04a69eb7f418521d49b8bbf77aTimo Sirainen        if (new_range.seq1 != 0)
d368bfd671ae6d04a69eb7f418521d49b8bbf77aTimo Sirainen            array_append(seq_range, &new_range, 1);
d368bfd671ae6d04a69eb7f418521d49b8bbf77aTimo Sirainen    }
d368bfd671ae6d04a69eb7f418521d49b8bbf77aTimo Sirainen    return 0;
7d97b8b0e47ad995915373b5c2a57f622a393352Timo Sirainen}
7d97b8b0e47ad995915373b5c2a57f622a393352Timo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenstatic int fts_build_mail_flush(struct fts_storage_build_context *ctx)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen{
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    if (str_len(ctx->headers) == 0)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen        return 1;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen    if (fts_backend_build_more(ctx->build, ctx->uid, str_data(ctx->headers),
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen                   str_len(ctx->headers)) < 0)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen        return -1;
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen    str_truncate(ctx->headers, 0);
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    return 1;
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen}
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainenstatic bool fts_build_update_save_part(struct fts_storage_build_context *ctx,
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen                       const struct message_block *block)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen{
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    /* we'll index only text/xxx and message/rfc822 parts for now */
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    if ((block->part->flags &
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen         (MESSAGE_PART_FLAG_TEXT |
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen          MESSAGE_PART_FLAG_MESSAGE_RFC822)) == 0)
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        return FALSE;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen    ctx->save_part = TRUE;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    return TRUE;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen}
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainenstatic int fts_build_mail_header(struct fts_storage_build_context *ctx,
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen                 const struct message_block *block)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen{
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen    const struct message_header_line *hdr = block->hdr;
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen
157bce86d0a01477bb8ebd0d380e6b2297f326f7Timo Sirainen    /* hdr->full_value is always set because we get the block from
8eb94c5190ba09bb6f6f068eec7bf96750f08d1dTimo Sirainen       message_decoder */
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    str_append(ctx->headers, hdr->name);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    str_append_n(ctx->headers, hdr->middle, hdr->middle_len);
8eb94c5190ba09bb6f6f068eec7bf96750f08d1dTimo Sirainen    str_append_n(ctx->headers, hdr->full_value, hdr->full_value_len);
965ed6ea3fc8f7637bd0d159d2fdb283a191ce34Timo Sirainen    if (!hdr->no_newline)
91e4199476cb2add8143c18583fa57e1decfea88Timo Sirainen        str_append_c(ctx->headers, '\n');
0727e38ac12efb8963a339daf56255e2be1f29fcTimo Sirainen
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen    if (!ctx->save_part) {
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen        if (strcasecmp(hdr->name, "Content-Type") == 0) {
8eb94c5190ba09bb6f6f068eec7bf96750f08d1dTimo Sirainen            if (!fts_build_update_save_part(ctx, block))
8eb94c5190ba09bb6f6f068eec7bf96750f08d1dTimo Sirainen                return 0;
08aea01ef9a9d20703e0fcf8618e6195c0037a44Timo Sirainen        }
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        return 1;
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen    }
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen    return fts_build_mail_flush(ctx);
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen}
747e77e3ab073a8e9e69c7a3e71b4593c5655d03Timo Sirainen
747e77e3ab073a8e9e69c7a3e71b4593c5655d03Timo Sirainenstatic int fts_build_mail(struct fts_storage_build_context *ctx)
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen{
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    struct istream *input;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    struct message_parser_ctx *parser;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    struct message_decoder_context *decoder;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    struct message_block raw_block, block;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    struct message_part *prev_part, *skip_part;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    int ret;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    ctx->uid = ctx->mail->uid;
157bce86d0a01477bb8ebd0d380e6b2297f326f7Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    input = mail_get_stream(ctx->mail, NULL, NULL);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    if (input == NULL)
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen        return -1;
08aea01ef9a9d20703e0fcf8618e6195c0037a44Timo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    prev_part = skip_part = NULL;
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    parser = message_parser_init(pool_datastack_create(), input);
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen    decoder = message_decoder_init();
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen    for (;;) {
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen        ret = message_parser_parse_next_block(parser, &raw_block);
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen        i_assert(ret != 0);
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen        if (ret < 0) {
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen            if (input->stream_errno == 0)
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen                ret = 0;
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen            break;
08aea01ef9a9d20703e0fcf8618e6195c0037a44Timo Sirainen        }
08aea01ef9a9d20703e0fcf8618e6195c0037a44Timo Sirainen        if (raw_block.part == skip_part)
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen            continue;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen        if (!message_decoder_decode_next_block(decoder, &raw_block,
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen                               &block))
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen            continue;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        if (block.part != prev_part &&
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            (block.hdr != NULL || block.size != 0)) {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            str_truncate(ctx->headers, 0);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            ctx->save_part = FALSE;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            prev_part = block.part;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            skip_part = NULL;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        }
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        if (block.hdr != NULL) {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            ret = fts_build_mail_header(ctx, &block);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            if (ret < 0)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                break;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            if (ret == 0)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                skip_part = raw_block.part;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        } else if (block.size == 0) {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            /* end of headers */
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            if (fts_build_update_save_part(ctx, &block)) {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                ret = fts_build_mail_flush(ctx);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                if (ret < 0)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                    break;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            }
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        } else {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            if (fts_backend_build_more(ctx->build, ctx->mail->uid,
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                           block.data,
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                           block.size) < 0) {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                ret = -1;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen                break;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen            }
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        }
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    }
    (void)message_parser_deinit(&parser);
    message_decoder_deinit(&decoder);
    return ret;
}

static int fts_build_init(struct fts_search_context *fctx,
              struct mailbox_transaction_context *t)
{
    struct fts_backend *backend = fctx->backend;
    struct fts_storage_build_context *ctx;
    struct fts_backend_build_context *build;
    struct mail_search_seqset seqset;
    uint32_t last_uid, last_uid_locked;

    if (fts_backend_get_last_uid(backend, &last_uid) < 0)
        return -1;

    if (last_uid == 0 && fctx->best_arg->type == SEARCH_HEADER) {
        /* index doesn't exist. we're not creating it just for
           header lookups. */
        return -1;
    }

    memset(&seqset, 0, sizeof(seqset));
    if (mailbox_get_uids(t->box, last_uid+1, (uint32_t)-1,
                 &seqset.seq1, &seqset.seq2) < 0)
        return -1;
    if (seqset.seq1 == 0) {
        /* no new messages */
        return 0;
    }

    build = fts_backend_build_init(backend, &last_uid_locked);
    if (last_uid != last_uid_locked) {
        /* changed, need to get again the sequences */
        i_assert(last_uid < last_uid_locked);

        last_uid = last_uid_locked;
        if (mailbox_get_uids(t->box, last_uid+1, (uint32_t)-1,
                     &seqset.seq1, &seqset.seq2) < 0) {
            (void)fts_backend_build_deinit(build);
            return -1;
        }
        if (seqset.seq1 == 0) {
            /* no new messages */
            (void)fts_backend_build_deinit(build);
            return 0;
        }
    }

    ctx = i_new(struct fts_storage_build_context, 1);
    ctx->build = build;
    ctx->seqset = seqset;
    ctx->search_arg.type = SEARCH_SEQSET;
    ctx->search_arg.value.seqset = &ctx->seqset;

    ctx->headers = str_new(default_pool, 512);
    ctx->mail = mail_alloc(t, 0, NULL);
    ctx->search_ctx = mailbox_search_init(t, NULL, &ctx->search_arg, NULL);

    fctx->build_ctx = ctx;
    return 0;
}

static int fts_build_deinit(struct fts_storage_build_context *ctx)
{
    int ret = 0;

    if (mailbox_search_deinit(&ctx->search_ctx) < 0)
        ret = -1;
    mail_free(&ctx->mail);

    if (fts_backend_build_deinit(ctx->build) < 0)
        ret = -1;
    str_free(&ctx->headers);
    i_free(ctx);
    return ret;
}

static int fts_build_more(struct fts_storage_build_context *ctx)
{
    unsigned int count = 0;
    int ret;

    while (mailbox_search_next(ctx->search_ctx, ctx->mail) > 0) {
        t_push();
        ret = fts_build_mail(ctx);
        t_pop();

        if (ret < 0)
            return -1;

        if (++count == FTS_SEARCH_NONBLOCK_COUNT)
            return 0;
    }
    return 1;
}

static void fts_search_filter_args(struct fts_search_context *fctx,
                   struct mail_search_arg *args,
                   ARRAY_TYPE(seq_range) *uid_result)
{
    const char *key;

    for (; args != NULL; args = args->next) {
        switch (args->type) {
        case SEARCH_BODY_FAST:
        case SEARCH_TEXT_FAST:
            if ((fctx->backend->flags &
                 FTS_BACKEND_FLAG_EXACT_LOOKUPS) == 0)
                break;
            /* fall through */
        case SEARCH_BODY:
        case SEARCH_TEXT:
        case SEARCH_HEADER:
            if (args == fctx->best_arg) {
                /* already handled this one */
                break;
            }

            key = args->value.str;
            if (*key == '\0') {
                i_assert(args->type == SEARCH_HEADER);

                /* we're only checking the existence
                   of the header. */
                key = args->hdr_field_name;
            }

            if (fts_backend_filter(fctx->backend, key,
                           uid_result) < 0) {
                /* failed, but we already have limited
                   the search, so just ignore this */
                break;
            }
            if (args->type != SEARCH_HEADER &&
                (fctx->backend->flags &
                 FTS_BACKEND_FLAG_DEFINITE_LOOKUPS) != 0) {
                args->match_always = TRUE;
                args->result = 1;
            }
            break;
        case SEARCH_OR:
        case SEARCH_SUB:
            fts_search_filter_args(fctx, args->value.subargs,
                           uid_result);
            break;
        default:
            break;
        }
    }
}

static void fts_search_init(struct mailbox *box,
                struct fts_search_context *fctx)
{
    struct fts_backend *backend = fctx->backend;
    const char *key;
    ARRAY_TYPE(seq_range) uid_result;

    if (fts_backend_lock(backend) <= 0)
        return;
    fctx->locked = TRUE;

    key = fctx->best_arg->value.str;
    if (*key == '\0') {
        i_assert(fctx->best_arg->type == SEARCH_HEADER);

        /* we're only checking the existence
           of the header. */
        key = fctx->best_arg->hdr_field_name;
    }

    i_array_init(&uid_result, 64);
    if (fts_backend_lookup(backend, key, &uid_result) < 0) {
        /* failed, fallback to reading everything */
        array_free(&uid_result);
        return;
    }

    if ((backend->flags & FTS_BACKEND_FLAG_DEFINITE_LOOKUPS) != 0) {
        fctx->best_arg->match_always = TRUE;
        fctx->best_arg->result = 1;
    }

    fts_search_filter_args(fctx, fctx->args, &uid_result);

    (void)uid_range_to_seq(box, &uid_result, &fctx->result);
    array_free(&uid_result);
}

static bool arg_is_better(const struct mail_search_arg *new_arg,
              const struct mail_search_arg *old_arg)
{
    if (old_arg == NULL)
        return TRUE;

    /* prefer not to use headers. they have a larger possibility of
       having lots of identical strings */
    if (old_arg->type == SEARCH_HEADER)
        return TRUE;
    else if (new_arg->type == SEARCH_HEADER)
        return FALSE;

    return strlen(new_arg->value.str) > strlen(old_arg->value.str);
}

static void fts_search_args_check(struct mail_search_arg *args,
                  bool *have_fast_r, bool *have_exact_r,
                  struct mail_search_arg **best_fast_arg,
                  struct mail_search_arg **best_exact_arg)
{
    for (; args != NULL; args = args->next) {
        switch (args->type) {
        case SEARCH_BODY_FAST:
        case SEARCH_TEXT_FAST:
            if (*args->value.str == '\0') {
                /* this matches everything */
                args->match_always = TRUE;
                args->result = 1;
                break;
            }
            if (arg_is_better(args, *best_fast_arg)) {
                *best_fast_arg = args;
                *have_fast_r = TRUE;
            }
            break;
        case SEARCH_BODY:
        case SEARCH_TEXT:
            if (*args->value.str == '\0') {
                /* this matches everything */
                args->match_always = TRUE;
                args->result = 1;
                break;
            }
        case SEARCH_HEADER:
            if (arg_is_better(args, *best_exact_arg)) {
                *best_exact_arg = args;
                *have_exact_r = TRUE;
            }
            break;
        case SEARCH_OR:
        case SEARCH_SUB:
            fts_search_args_check(args->value.subargs,
                          have_fast_r, have_exact_r,
                          best_fast_arg, best_exact_arg);
            break;
        default:
            break;
        }
    }
}

static struct mail_search_context *
fts_mailbox_search_init(struct mailbox_transaction_context *t,
            const char *charset, struct mail_search_arg *args,
            const enum mail_sort_type *sort_program)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
    struct mail_search_context *ctx;
    struct fts_search_context *fctx;
    struct mail_search_arg *best_fast_arg, *best_exact_arg;
    bool have_fast, have_exact;

    ctx = fbox->super.search_init(t, charset, args, sort_program);

    fctx = i_new(struct fts_search_context, 1);
    fctx->args = args;
    array_idx_set(&ctx->module_contexts, fts_storage_module_id, &fctx);

    if (fbox->backend_exact == NULL && fbox->backend_fast == NULL)
        return ctx;

    have_fast = have_exact = FALSE;
    best_fast_arg = best_exact_arg = NULL;
    fts_search_args_check(args, &have_fast, &have_exact,
                  &best_fast_arg, &best_exact_arg);
    if (have_fast && fbox->backend_fast != NULL) {
        /* use fast backend whenever possible */
        fctx->backend = fbox->backend_fast;
        fctx->best_arg = best_fast_arg;
    } else if (have_exact || have_fast) {
        fctx->backend = fbox->backend_exact;
        fctx->best_arg = arg_is_better(best_exact_arg, best_fast_arg) ?
            best_exact_arg : best_fast_arg;
    }

    if (fctx->backend != NULL) {
        if (fts_build_init(fctx, t) < 0)
            fctx->backend = NULL;
        else if (fctx->build_ctx == NULL) {
            /* the index was up to date */
            fts_search_init(t->box, fctx);
        }
    }

    return ctx;
}

static int fts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
                        struct mail *mail, bool *tryagain_r)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);
    int ret;

    if (fctx->build_ctx != NULL) {
        /* still building the index */
        ret = fts_build_more(fctx->build_ctx);
        if (ret == 0) {
            *tryagain_r = TRUE;
            return 0;
        }

        /* finished / error */
        fts_build_deinit(fctx->build_ctx);
        fctx->build_ctx = NULL;

        if (ret == 0)
            fts_search_init(ctx->transaction->box, fctx);
    }
    return fbox->super.search_next_nonblock(ctx, mail, tryagain_r);

}

static int fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);
    struct seq_range *range;
    unsigned int count;
    uint32_t wanted_seq;
    int ret;

    if (!array_is_created(&fctx->result))
        return fbox->super.search_next_update_seq(ctx);

    do {
        range = array_get_modifiable(&fctx->result, &count);
        while (fctx->result_pos < count &&
               ctx->seq > range[fctx->result_pos].seq2)
            fctx->result_pos++;

        if (fctx->result_pos == count)
            return 0;

        if (ctx->seq > range[fctx->result_pos].seq1)
            range[fctx->result_pos].seq1 = ctx->seq+1;
        else {
            ctx->seq = range[fctx->result_pos].seq1 - 1;

            if (fctx->result_pos < count &&
                ctx->seq + 1 == range[fctx->result_pos].seq2)
                fctx->result_pos++;
            else
                range[fctx->result_pos].seq1++;
        }

        wanted_seq = ctx->seq + 1;
        ret = fbox->super.search_next_update_seq(ctx);
    } while (ret > 0 && wanted_seq != ctx->seq);

    return ret;
}

static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
    struct fts_search_context *fctx = FTS_CONTEXT(ctx);

    if (fctx->build_ctx != NULL) {
        /* the search was cancelled */
        fts_build_deinit(fctx->build_ctx);
    }

    if (fctx->locked)
        fts_backend_unlock(fctx->backend);

    if (array_is_created(&fctx->result))
        array_free(&fctx->result);
    i_free(fctx);
    return fbox->super.search_deinit(ctx);
}

static int fts_mail_expunge(struct mail *_mail)
{
    struct mail_private *mail = (struct mail_private *)_mail;
    struct fts_mail *fmail = FTS_CONTEXT(mail);
    struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
    struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);

    if (fmail->super.expunge(_mail) < 0)
        return -1;

    ft->expunges = TRUE;
    if (fbox->backend_exact != NULL)
        fts_backend_expunge(fbox->backend_exact, _mail);
    if (fbox->backend_fast != NULL)
        fts_backend_expunge(fbox->backend_fast, _mail);
    return 0;
}

static struct mail *
fts_mail_alloc(struct mailbox_transaction_context *t,
           enum mail_fetch_field wanted_fields,
           struct mailbox_header_lookup_ctx *wanted_headers)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
    struct fts_mail *fmail;
    struct mail *_mail;
    struct mail_private *mail;

    _mail = fbox->super.mail_alloc(t, wanted_fields, wanted_headers);
    if (fbox->backend_exact != NULL || fbox->backend_fast != NULL) {
        mail = (struct mail_private *)_mail;

        fmail = p_new(mail->pool, struct fts_mail, 1);
        fmail->super = mail->v;

        mail->v.expunge = fts_mail_expunge;
        array_idx_set(&mail->module_contexts,
                  fts_storage_module_id, &fmail);
    }
    return _mail;
}

static void fts_box_backends_init(struct mailbox *box)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct fts_backend *backend;
    const char *const *tmp;

    for (tmp = t_strsplit(fbox->env, ", "); *tmp != NULL; tmp++) {
        backend = fts_backend_init(*tmp, box);
        if (backend == NULL)
            continue;

        if ((backend->flags &
             FTS_BACKEND_FLAG_EXACT_LOOKUPS) != 0) {
            if (fbox->backend_exact != NULL) {
                i_fatal("fts: duplicate exact backend: %s",
                    *tmp);
            }
            fbox->backend_exact = backend;
        } else {
            if (fbox->backend_fast != NULL) {
                i_fatal("fts: duplicate fast backend: %s",
                    *tmp);
            }
            fbox->backend_fast = backend;
        }
    }
}

static struct mailbox_transaction_context *
fts_transaction_begin(struct mailbox *box,
              enum mailbox_transaction_flags flags)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct mailbox_transaction_context *t;
    struct fts_transaction_context *ft;

    ft = i_new(struct fts_transaction_context, 1);

    /* the backend creation is delayed until the first transaction is
       started. at that point the mailbox has been synced at least once. */
    if (!fbox->backend_set) {
        fts_box_backends_init(box);
        fbox->backend_set = TRUE;
    }

    t = fbox->super.transaction_begin(box, flags);
    array_idx_set(&t->module_contexts, fts_storage_module_id, &ft);
    return t;
}

static void
fts_transaction_finish(struct mailbox *box, struct fts_transaction_context *ft,
               bool committed)
{
    struct fts_mailbox *fbox = FTS_CONTEXT(box);

    if (ft->expunges) {
        if (fbox->backend_fast != NULL) {
            fts_backend_expunge_finish(fbox->backend_fast,
                           box, committed);
        }
    }
    i_free(ft);
}

static void fts_transaction_rollback(struct mailbox_transaction_context *t)
{
    struct mailbox *box = t->box;
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct fts_transaction_context *ft = FTS_CONTEXT(t);

    fbox->super.transaction_rollback(t);
    fts_transaction_finish(box, ft, FALSE);
}

static int fts_transaction_commit(struct mailbox_transaction_context *t,
                  enum mailbox_sync_flags flags)
{
    struct mailbox *box = t->box;
    struct fts_mailbox *fbox = FTS_CONTEXT(box);
    struct fts_transaction_context *ft = FTS_CONTEXT(t);
    int ret;

    ret = fbox->super.transaction_commit(t, flags);
    fts_transaction_finish(box, ft, ret == 0);
    return ret;
}

void fts_mailbox_opened(struct mailbox *box)
{
    struct fts_mailbox *fbox;
    const char *env;

    if (fts_next_hook_mailbox_opened != NULL)
        fts_next_hook_mailbox_opened(box);

    env = getenv("FTS");
    if (env == NULL)
        return;

    fbox = i_new(struct fts_mailbox, 1);
    fbox->env = env;
    fbox->super = box->v;
    box->v.close = fts_mailbox_close;
    box->v.search_init = fts_mailbox_search_init;
    box->v.search_next_nonblock = fts_mailbox_search_next_nonblock;
    box->v.search_next_update_seq = fts_mailbox_search_next_update_seq;
    box->v.search_deinit = fts_mailbox_search_deinit;
    box->v.mail_alloc = fts_mail_alloc;
    box->v.transaction_begin = fts_transaction_begin;
    box->v.transaction_rollback = fts_transaction_rollback;
    box->v.transaction_commit = fts_transaction_commit;

    if (!fts_storage_module_id_set) {
        fts_storage_module_id = mail_storage_module_id++;
        fts_storage_module_id_set = TRUE;
    }

    array_idx_set(&box->module_contexts, fts_storage_module_id, &fbox);
}