fts-storage.c revision bc93929cdd9000ca560a5f42a27f50ab307f1efb
7cb128dc4cae2a03a742f63ba7afee23c78e3af0Phil Carmody/* Copyright (C) 2006 Timo Sirainen */
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "lib.h"
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "ioloop.h"
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "array.h"
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "str.h"
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "istream.h"
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "message-parser.h"
72f67d7d7681934bb3d4baaea1d88f79521dee3dTimo Sirainen#include "message-decoder.h"
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "mail-search.h"
1358e2c58ce29231485a5cfa454756d429ad3d2cTimo Sirainen#include "mail-storage-private.h"
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "fts-api-private.h"
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include "fts-plugin.h"
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#include <stdlib.h>
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#define FTS_CONTEXT(obj) \
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen *((void **)array_idx_modifiable(&(obj)->module_contexts, \
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen fts_storage_module_id))
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#define FTS_SEARCH_NONBLOCK_COUNT 10
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen#define FTS_BUILD_NOTIFY_INTERVAL_SECS 10
50fe5e053d565b40b0ed2cc7a64b6b98971c0305Timo Sirainen
633a3da9d3e9a5befd3405f6651043a6bdd327cbTimo Sirainenstruct fts_mailbox {
50fe5e053d565b40b0ed2cc7a64b6b98971c0305Timo Sirainen struct mailbox_vfuncs super;
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen struct fts_backend *backend_exact;
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen struct fts_backend *backend_fast;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen const char *env;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen unsigned int backend_set:1;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen};
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainenstruct fts_search_context {
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen ARRAY_TYPE(seq_range) result;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen unsigned int result_pos;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct mail_search_arg *args, *best_arg;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct fts_backend *backend;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct fts_storage_build_context *build_ctx;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct mailbox_transaction_context *t;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen unsigned int build_initialized:1;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen unsigned int locked:1;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen};
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainenstruct fts_storage_build_context {
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct mail_search_context *search_ctx;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct mail_search_seqset seqset;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct mail_search_arg search_arg;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct mail *mail;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct fts_backend_build_context *build;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct timeval search_start_time, last_notify;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen uint32_t uid;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen string_t *headers;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen bool save_part;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen};
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainenstruct fts_transaction_context {
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen bool expunges;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen};
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainenstruct fts_mail {
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct mail_vfuncs super;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen};
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainenstatic unsigned int fts_storage_module_id = 0;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainenstatic bool fts_storage_module_id_set = FALSE;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainenstatic int fts_mailbox_close(struct mailbox *box)
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen{
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen int ret;
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen if (fbox->backend_exact != NULL)
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen fts_backend_deinit(fbox->backend_exact);
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen if (fbox->backend_fast != NULL)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen fts_backend_deinit(fbox->backend_fast);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen ret = fbox->super.close(box);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen i_free(fbox);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return ret;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen}
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainenstatic int uid_range_to_seq(struct mailbox *box,
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen ARRAY_TYPE(seq_range) *uid_range,
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen ARRAY_TYPE(seq_range) *seq_range)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen{
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen const struct seq_range *range;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen struct seq_range new_range;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen unsigned int i, count;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen range = array_get(uid_range, &count);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen i_array_init(seq_range, count);
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen for (i = 0; i < count; i++) {
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (mailbox_get_uids(box, range[i].seq1, range[i].seq2,
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen &new_range.seq1, &new_range.seq2) < 0) {
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen array_free(seq_range);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return -1;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen }
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (new_range.seq1 != 0)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen array_append(seq_range, &new_range, 1);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen }
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return 0;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen}
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainenstatic int fts_build_mail_flush(struct fts_storage_build_context *ctx)
ca7e608c21bdfd7b6a2e7fcb2678ade4c1c1972cTimo Sirainen{
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (str_len(ctx->headers) == 0)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return 1;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (fts_backend_build_more(ctx->build, ctx->uid, str_data(ctx->headers),
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen str_len(ctx->headers), TRUE) < 0)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return -1;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen str_truncate(ctx->headers, 0);
3ba70a3fa363850cc513d3cf28d1916e2469b90fTimo Sirainen return 1;
009217abb57a24a4076092e8e4e165545747839eStephan Bosch}
009217abb57a24a4076092e8e4e165545747839eStephan Bosch
009217abb57a24a4076092e8e4e165545747839eStephan Boschstatic bool fts_build_update_save_part(struct fts_storage_build_context *ctx,
009217abb57a24a4076092e8e4e165545747839eStephan Bosch const struct message_block *block)
009217abb57a24a4076092e8e4e165545747839eStephan Bosch{
009217abb57a24a4076092e8e4e165545747839eStephan Bosch /* we'll index only text/xxx and message/rfc822 parts for now */
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if ((block->part->flags &
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen (MESSAGE_PART_FLAG_TEXT |
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen MESSAGE_PART_FLAG_MESSAGE_RFC822)) == 0)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return FALSE;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen ctx->save_part = TRUE;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return TRUE;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen}
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainenstatic int fts_build_mail_header(struct fts_storage_build_context *ctx,
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen const struct message_block *block)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen{
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen const struct message_header_line *hdr = block->hdr;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen /* hdr->full_value is always set because we get the block from
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen message_decoder */
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen str_append(ctx->headers, hdr->name);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen str_append_n(ctx->headers, hdr->middle, hdr->middle_len);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen str_append_n(ctx->headers, hdr->full_value, hdr->full_value_len);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (!hdr->no_newline)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen str_append_c(ctx->headers, '\n');
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (!ctx->save_part) {
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (strcasecmp(hdr->name, "Content-Type") == 0) {
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (!fts_build_update_save_part(ctx, block))
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return 0;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen }
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return 1;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen }
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return fts_build_mail_flush(ctx);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen}
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainenstatic int fts_build_mail(struct fts_storage_build_context *ctx)
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen{
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen struct istream *input;
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen struct message_parser_ctx *parser;
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen struct message_decoder_context *decoder;
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen struct message_block raw_block, block;
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen struct message_part *prev_part, *skip_part;
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen int ret;
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen ctx->uid = ctx->mail->uid;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen input = mail_get_stream(ctx->mail, NULL, NULL);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (input == NULL)
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return -1;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen prev_part = skip_part = NULL;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen parser = message_parser_init(pool_datastack_create(), input);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen decoder = message_decoder_init();
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen for (;;) {
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen ret = message_parser_parse_next_block(parser, &raw_block);
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen i_assert(ret != 0);
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen if (ret < 0) {
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen if (input->stream_errno == 0)
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen ret = 0;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen break;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen }
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen if (raw_block.part == skip_part)
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen continue;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen if (!message_decoder_decode_next_block(decoder, &raw_block,
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen &block))
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen continue;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen if (block.part != prev_part &&
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen (block.hdr != NULL || block.size != 0)) {
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen str_truncate(ctx->headers, 0);
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen ctx->save_part = FALSE;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen prev_part = block.part;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen skip_part = NULL;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen }
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen if (block.hdr != NULL) {
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen ret = fts_build_mail_header(ctx, &block);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen if (ret < 0)
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen break;
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen if (ret == 0)
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen skip_part = raw_block.part;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen } else if (block.size == 0) {
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen /* end of headers */
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen if (fts_build_update_save_part(ctx, &block)) {
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen ret = fts_build_mail_flush(ctx);
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen if (ret < 0)
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen break;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen }
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen } else {
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen if (fts_backend_build_more(ctx->build, ctx->mail->uid,
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen block.data, block.size,
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen FALSE) < 0) {
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ret = -1;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen break;
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen }
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen }
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen }
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen (void)message_parser_deinit(&parser);
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen message_decoder_deinit(&decoder);
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen return ret;
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen}
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainenstatic int fts_build_init(struct fts_search_context *fctx)
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen{
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen struct mailbox_transaction_context *t = fctx->t;
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen struct fts_backend *backend = fctx->backend;
50fe5e053d565b40b0ed2cc7a64b6b98971c0305Timo Sirainen struct fts_storage_build_context *ctx;
50fe5e053d565b40b0ed2cc7a64b6b98971c0305Timo Sirainen struct fts_backend_build_context *build;
50fe5e053d565b40b0ed2cc7a64b6b98971c0305Timo Sirainen struct mail_search_seqset seqset;
50fe5e053d565b40b0ed2cc7a64b6b98971c0305Timo Sirainen uint32_t last_uid, last_uid_locked;
50fe5e053d565b40b0ed2cc7a64b6b98971c0305Timo Sirainen
6899af0c40f5e1ef80bd18961cea3a192134a4a4Timo Sirainen if (fts_backend_get_last_uid(backend, &last_uid) < 0)
6899af0c40f5e1ef80bd18961cea3a192134a4a4Timo Sirainen return -1;
6899af0c40f5e1ef80bd18961cea3a192134a4a4Timo Sirainen
6899af0c40f5e1ef80bd18961cea3a192134a4a4Timo Sirainen if (last_uid == 0 && fctx->best_arg->type == SEARCH_HEADER) {
50fe5e053d565b40b0ed2cc7a64b6b98971c0305Timo Sirainen /* index doesn't exist. we're not creating it just for
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen header lookups. */
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen return -1;
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen }
6899af0c40f5e1ef80bd18961cea3a192134a4a4Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen memset(&seqset, 0, sizeof(seqset));
396999508030c8498cb538eb8943e6b61394994cTimo Sirainen if (mailbox_get_uids(t->box, last_uid+1, (uint32_t)-1,
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen &seqset.seq1, &seqset.seq2) < 0)
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen return -1;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen if (seqset.seq1 == 0) {
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen /* no new messages */
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen return 0;
50e20db49f29917fe9adcf1b56b11badf28bd0e4Timo Sirainen }
57fc30eb8669d1a564372a2ca9842da4f118663fTimo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen build = fts_backend_build_init(backend, &last_uid_locked);
009217abb57a24a4076092e8e4e165545747839eStephan Bosch if (last_uid != last_uid_locked) {
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen /* changed, need to get again the sequences */
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen i_assert(last_uid < last_uid_locked);
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen last_uid = last_uid_locked;
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen if (mailbox_get_uids(t->box, last_uid+1, (uint32_t)-1,
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen &seqset.seq1, &seqset.seq2) < 0) {
7a5e2e937f0f388465c2938645ee971b4fd01bf1Timo Sirainen (void)fts_backend_build_deinit(build);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen return -1;
4b94ae3dcbace4781b64e87aea00ec0bc03a0d8aTimo Sirainen }
4b94ae3dcbace4781b64e87aea00ec0bc03a0d8aTimo Sirainen if (seqset.seq1 == 0) {
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen /* no new messages */
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen (void)fts_backend_build_deinit(build);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen return 0;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen }
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen }
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ctx = i_new(struct fts_storage_build_context, 1);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ctx->build = build;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ctx->seqset = seqset;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ctx->search_arg.type = SEARCH_SEQSET;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen ctx->search_arg.value.seqset = &ctx->seqset;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen ctx->headers = str_new(default_pool, 512);
56439f2879ad690d9ac637e2b90b612760d2f219Timo Sirainen ctx->mail = mail_alloc(t, 0, NULL);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ctx->search_ctx = mailbox_search_init(t, NULL, &ctx->search_arg, NULL);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen fctx->build_ctx = ctx;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen return 0;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen}
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainenstatic int fts_build_deinit(struct fts_storage_build_context *ctx)
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen{
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen struct mailbox *box = ctx->mail->transaction->box;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen int ret = 0;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen if (mailbox_search_deinit(&ctx->search_ctx) < 0)
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ret = -1;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen mail_free(&ctx->mail);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen if (fts_backend_build_deinit(ctx->build) < 0)
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ret = -1;
c84c0d4cb7f16aedbcb3d5db6b2d41851cc572c9Timo Sirainen
c84c0d4cb7f16aedbcb3d5db6b2d41851cc572c9Timo Sirainen if (ioloop_time - ctx->search_start_time.tv_sec >=
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen FTS_BUILD_NOTIFY_INTERVAL_SECS) {
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen /* we notified at least once */
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen box->storage->callbacks->
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen notify_ok(box, "Mailbox indexing finished",
6851806b0d2e134d52757147e3fc242557ce8b4dTimo Sirainen box->storage->callback_context);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen }
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen str_free(&ctx->headers);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen i_free(ctx);
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen return ret;
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen}
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainenstatic void fts_build_notify(struct fts_storage_build_context *ctx)
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen{
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen struct mailbox *box = ctx->mail->transaction->box;
ff561236f26a203378f5bc1682dda406c78822ccTimo Sirainen const char *text;
ff561236f26a203378f5bc1682dda406c78822ccTimo Sirainen float percentage;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen unsigned int msecs, secs;
396999508030c8498cb538eb8943e6b61394994cTimo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen if (ctx->last_notify.tv_sec == 0) {
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen /* set the search time in here, in case a plugin
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen already spent some time indexing the mailbox */
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen ctx->search_start_time = ioloop_timeval;
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen } else if (box->storage->callbacks->notify_ok != NULL) {
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen percentage = (ctx->mail->seq - ctx->seqset.seq1) * 100.0 /
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen (ctx->seqset.seq2 - ctx->seqset.seq1);
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen msecs = (ioloop_timeval.tv_sec -
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen ctx->search_start_time.tv_sec) * 1000 +
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen (ioloop_timeval.tv_usec -
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen ctx->search_start_time.tv_usec) / 1000;
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen secs = (msecs / (percentage / 100.0) - msecs) / 1000;
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen
4b94ae3dcbace4781b64e87aea00ec0bc03a0d8aTimo Sirainen t_push();
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen text = t_strdup_printf("Indexed %d%% of the mailbox, "
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen "ETA %d:%02d", (int)percentage,
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen secs/60, secs%60);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen box->storage->callbacks->
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen notify_ok(box, text, box->storage->callback_context);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen t_pop();
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen }
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen ctx->last_notify = ioloop_timeval;
d1d425c3ed541dde4ae11e88291a0044e6c55f88Timo Sirainen}
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
d84c270231a617298088f597474a73f4a14921aeTimo Sirainenstatic int fts_build_more(struct fts_storage_build_context *ctx)
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen{
d84c270231a617298088f597474a73f4a14921aeTimo Sirainen unsigned int count = 0;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen int ret;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen if (ioloop_time - ctx->last_notify.tv_sec >=
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen FTS_BUILD_NOTIFY_INTERVAL_SECS)
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen fts_build_notify(ctx);
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen while (mailbox_search_next(ctx->search_ctx, ctx->mail) > 0) {
633a3da9d3e9a5befd3405f6651043a6bdd327cbTimo Sirainen t_push();
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen ret = fts_build_mail(ctx);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen t_pop();
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen if (ret < 0)
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen return -1;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
acb0236a7a1242f567cec83480c1efc9fc352f63Timo Sirainen if (++count == FTS_SEARCH_NONBLOCK_COUNT)
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen return 0;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen }
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
3b3a70990ab21c97140fcc2d1e3eb84c17d90388Timo Sirainen return 1;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen}
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainenstatic void fts_search_filter_args(struct fts_search_context *fctx,
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen struct mail_search_arg *args,
33bd898e7756b289e65f43133312d9637afc1371Timo Sirainen ARRAY_TYPE(seq_range) *uid_result)
33bd898e7756b289e65f43133312d9637afc1371Timo Sirainen{
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen const char *key;
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen enum fts_lookup_flags flags;
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen for (; args != NULL; args = args->next) {
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen switch (args->type) {
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen case SEARCH_BODY_FAST:
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen case SEARCH_TEXT_FAST:
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen if ((fctx->backend->flags &
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen FTS_BACKEND_FLAG_EXACT_LOOKUPS) == 0)
194755bdfb97c07ca8b9df071099f68947b971e3Timo Sirainen break;
633a3da9d3e9a5befd3405f6651043a6bdd327cbTimo Sirainen /* fall through */
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen case SEARCH_BODY:
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen case SEARCH_TEXT:
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen case SEARCH_HEADER:
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen if (args == fctx->best_arg) {
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen /* already handled this one */
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen break;
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen }
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen key = args->value.str;
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen if (*key == '\0') {
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen i_assert(args->type == SEARCH_HEADER);
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen /* we're only checking the existence
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen of the header. */
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen key = args->hdr_field_name;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen }
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen flags = FTS_LOOKUP_FLAG_BODY;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen if (args->type == SEARCH_TEXT_FAST ||
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen args->type == SEARCH_TEXT)
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen flags |= FTS_LOOKUP_FLAG_HEADERS;
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen if (fts_backend_filter(fctx->backend, flags, key,
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen uid_result) < 0) {
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen /* failed, but we already have limited
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen the search, so just ignore this */
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen break;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen }
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen if (args->type != SEARCH_HEADER &&
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen (fctx->backend->flags &
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen FTS_BACKEND_FLAG_DEFINITE_LOOKUPS) != 0) {
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen args->match_always = TRUE;
c0787d6ab19f4a17ec08699d0bbc77f13a9b02a9Timo Sirainen args->result = 1;
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen }
6ed1e82824590b514201d9db84ba96bdfc832dd5Timo Sirainen break;
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen case SEARCH_OR:
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen case SEARCH_SUB:
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen fts_search_filter_args(fctx, args->value.subargs,
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen uid_result);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen break;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen default:
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen break;
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen }
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen }
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen}
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
633a3da9d3e9a5befd3405f6651043a6bdd327cbTimo Sirainenstatic void fts_search_init(struct mailbox *box,
633a3da9d3e9a5befd3405f6651043a6bdd327cbTimo Sirainen struct fts_search_context *fctx)
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen{
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen struct fts_backend *backend = fctx->backend;
33bd898e7756b289e65f43133312d9637afc1371Timo Sirainen enum fts_lookup_flags flags;
33bd898e7756b289e65f43133312d9637afc1371Timo Sirainen const char *key;
33bd898e7756b289e65f43133312d9637afc1371Timo Sirainen ARRAY_TYPE(seq_range) uid_result;
33bd898e7756b289e65f43133312d9637afc1371Timo Sirainen
33bd898e7756b289e65f43133312d9637afc1371Timo Sirainen if (fts_backend_lock(backend) <= 0)
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen return;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen fctx->locked = TRUE;
df6551ce47053de2c366f490bef60803207beaa4Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen key = fctx->best_arg->value.str;
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen if (*key == '\0') {
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen i_assert(fctx->best_arg->type == SEARCH_HEADER);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen /* we're only checking the existence
4383941ed43d004f34f77334a06ad14e6a305607Timo Sirainen of the header. */
4383941ed43d004f34f77334a06ad14e6a305607Timo Sirainen flags = FTS_LOOKUP_FLAG_HEADERS;
4383941ed43d004f34f77334a06ad14e6a305607Timo Sirainen key = fctx->best_arg->hdr_field_name;
4383941ed43d004f34f77334a06ad14e6a305607Timo Sirainen } else {
4383941ed43d004f34f77334a06ad14e6a305607Timo Sirainen flags = FTS_LOOKUP_FLAG_BODY;
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen if (fctx->best_arg->type == SEARCH_TEXT_FAST ||
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen fctx->best_arg->type == SEARCH_TEXT)
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen flags |= FTS_LOOKUP_FLAG_HEADERS;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen }
637ec4c33b4715737a41f7e58c9b6d1f693c27e2Timo Sirainen
e9e5e84ffb2ce2e606a24ce6d930580367562ff0Timo Sirainen i_array_init(&uid_result, 64);
if (fts_backend_lookup(backend, flags, key, &uid_result) < 0) {
/* failed, fallback to reading everything */
array_free(&uid_result);
return;
}
if ((backend->flags & FTS_BACKEND_FLAG_DEFINITE_LOOKUPS) != 0) {
fctx->best_arg->match_always = TRUE;
fctx->best_arg->result = 1;
}
fts_search_filter_args(fctx, fctx->args, &uid_result);
(void)uid_range_to_seq(box, &uid_result, &fctx->result);
array_free(&uid_result);
}
static bool arg_is_better(const struct mail_search_arg *new_arg,
const struct mail_search_arg *old_arg)
{
if (old_arg == NULL)
return TRUE;
/* prefer not to use headers. they have a larger possibility of
having lots of identical strings */
if (old_arg->type == SEARCH_HEADER)
return TRUE;
else if (new_arg->type == SEARCH_HEADER)
return FALSE;
return strlen(new_arg->value.str) > strlen(old_arg->value.str);
}
static void fts_search_args_check(struct mail_search_arg *args,
bool *have_fast_r, bool *have_exact_r,
struct mail_search_arg **best_fast_arg,
struct mail_search_arg **best_exact_arg)
{
for (; args != NULL; args = args->next) {
switch (args->type) {
case SEARCH_BODY_FAST:
case SEARCH_TEXT_FAST:
if (*args->value.str == '\0') {
/* this matches everything */
args->match_always = TRUE;
args->result = 1;
break;
}
if (arg_is_better(args, *best_fast_arg)) {
*best_fast_arg = args;
*have_fast_r = TRUE;
}
break;
case SEARCH_BODY:
case SEARCH_TEXT:
if (*args->value.str == '\0') {
/* this matches everything */
args->match_always = TRUE;
args->result = 1;
break;
}
case SEARCH_HEADER:
if (arg_is_better(args, *best_exact_arg)) {
*best_exact_arg = args;
*have_exact_r = TRUE;
}
break;
case SEARCH_OR:
case SEARCH_SUB:
fts_search_args_check(args->value.subargs,
have_fast_r, have_exact_r,
best_fast_arg, best_exact_arg);
break;
default:
break;
}
}
}
static bool fts_try_build_init(struct fts_search_context *fctx)
{
if (fctx->backend == NULL) {
fctx->build_initialized = TRUE;
return TRUE;
}
if (fts_backend_is_building(fctx->backend))
return FALSE;
fctx->build_initialized = TRUE;
if (fts_build_init(fctx) < 0)
fctx->backend = NULL;
else if (fctx->build_ctx == NULL) {
/* the index was up to date */
fts_search_init(fctx->t->box, fctx);
}
return TRUE;
}
static struct mail_search_context *
fts_mailbox_search_init(struct mailbox_transaction_context *t,
const char *charset, struct mail_search_arg *args,
const enum mail_sort_type *sort_program)
{
struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
struct mail_search_context *ctx;
struct fts_search_context *fctx;
struct mail_search_arg *best_fast_arg, *best_exact_arg;
bool have_fast, have_exact;
ctx = fbox->super.search_init(t, charset, args, sort_program);
fctx = i_new(struct fts_search_context, 1);
fctx->t = t;
fctx->args = args;
array_idx_set(&ctx->module_contexts, fts_storage_module_id, &fctx);
if (fbox->backend_exact == NULL && fbox->backend_fast == NULL)
return ctx;
have_fast = have_exact = FALSE;
best_fast_arg = best_exact_arg = NULL;
fts_search_args_check(args, &have_fast, &have_exact,
&best_fast_arg, &best_exact_arg);
if (have_fast && fbox->backend_fast != NULL) {
/* use fast backend whenever possible */
fctx->backend = fbox->backend_fast;
fctx->best_arg = best_fast_arg;
} else if (have_exact || have_fast) {
fctx->backend = fbox->backend_exact;
fctx->best_arg = arg_is_better(best_exact_arg, best_fast_arg) ?
best_exact_arg : best_fast_arg;
}
fts_try_build_init(fctx);
return ctx;
}
static int fts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
struct mail *mail, bool *tryagain_r)
{
struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
struct fts_search_context *fctx = FTS_CONTEXT(ctx);
int ret;
if (!fctx->build_initialized) {
if (!fts_try_build_init(fctx)) {
*tryagain_r = TRUE;
return 0;
}
}
if (fctx->build_ctx != NULL) {
/* still building the index */
ret = fts_build_more(fctx->build_ctx);
if (ret == 0) {
*tryagain_r = TRUE;
return 0;
}
/* finished / error */
fts_build_deinit(fctx->build_ctx);
fctx->build_ctx = NULL;
if (ret > 0)
fts_search_init(ctx->transaction->box, fctx);
}
return fbox->super.search_next_nonblock(ctx, mail, tryagain_r);
}
static int fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
{
struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
struct fts_search_context *fctx = FTS_CONTEXT(ctx);
struct seq_range *range;
unsigned int count;
uint32_t wanted_seq;
int ret;
if (!array_is_created(&fctx->result))
return fbox->super.search_next_update_seq(ctx);
do {
range = array_get_modifiable(&fctx->result, &count);
while (fctx->result_pos < count &&
ctx->seq > range[fctx->result_pos].seq2)
fctx->result_pos++;
if (fctx->result_pos == count)
return 0;
if (ctx->seq > range[fctx->result_pos].seq1)
range[fctx->result_pos].seq1 = ctx->seq+1;
else {
ctx->seq = range[fctx->result_pos].seq1 - 1;
if (fctx->result_pos < count &&
ctx->seq + 1 == range[fctx->result_pos].seq2)
fctx->result_pos++;
else
range[fctx->result_pos].seq1++;
}
wanted_seq = ctx->seq + 1;
ret = fbox->super.search_next_update_seq(ctx);
} while (ret > 0 && wanted_seq != ctx->seq);
return ret;
}
static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
{
struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
struct fts_search_context *fctx = FTS_CONTEXT(ctx);
if (fctx->build_ctx != NULL) {
/* the search was cancelled */
fts_build_deinit(fctx->build_ctx);
}
if (fctx->locked)
fts_backend_unlock(fctx->backend);
if (array_is_created(&fctx->result))
array_free(&fctx->result);
i_free(fctx);
return fbox->super.search_deinit(ctx);
}
static int fts_mail_expunge(struct mail *_mail)
{
struct mail_private *mail = (struct mail_private *)_mail;
struct fts_mail *fmail = FTS_CONTEXT(mail);
struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
if (fmail->super.expunge(_mail) < 0)
return -1;
ft->expunges = TRUE;
if (fbox->backend_exact != NULL)
fts_backend_expunge(fbox->backend_exact, _mail);
if (fbox->backend_fast != NULL)
fts_backend_expunge(fbox->backend_fast, _mail);
return 0;
}
static struct mail *
fts_mail_alloc(struct mailbox_transaction_context *t,
enum mail_fetch_field wanted_fields,
struct mailbox_header_lookup_ctx *wanted_headers)
{
struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
struct fts_mail *fmail;
struct mail *_mail;
struct mail_private *mail;
_mail = fbox->super.mail_alloc(t, wanted_fields, wanted_headers);
if (fbox->backend_exact != NULL || fbox->backend_fast != NULL) {
mail = (struct mail_private *)_mail;
fmail = p_new(mail->pool, struct fts_mail, 1);
fmail->super = mail->v;
mail->v.expunge = fts_mail_expunge;
array_idx_set(&mail->module_contexts,
fts_storage_module_id, &fmail);
}
return _mail;
}
static void fts_box_backends_init(struct mailbox *box)
{
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct fts_backend *backend;
const char *const *tmp;
for (tmp = t_strsplit(fbox->env, ", "); *tmp != NULL; tmp++) {
backend = fts_backend_init(*tmp, box);
if (backend == NULL)
continue;
if ((backend->flags &
FTS_BACKEND_FLAG_EXACT_LOOKUPS) != 0) {
if (fbox->backend_exact != NULL) {
i_fatal("fts: duplicate exact backend: %s",
*tmp);
}
fbox->backend_exact = backend;
} else {
if (fbox->backend_fast != NULL) {
i_fatal("fts: duplicate fast backend: %s",
*tmp);
}
fbox->backend_fast = backend;
}
}
}
static struct mailbox_transaction_context *
fts_transaction_begin(struct mailbox *box,
enum mailbox_transaction_flags flags)
{
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct mailbox_transaction_context *t;
struct fts_transaction_context *ft;
ft = i_new(struct fts_transaction_context, 1);
/* the backend creation is delayed until the first transaction is
started. at that point the mailbox has been synced at least once. */
if (!fbox->backend_set) {
fts_box_backends_init(box);
fbox->backend_set = TRUE;
}
t = fbox->super.transaction_begin(box, flags);
array_idx_set(&t->module_contexts, fts_storage_module_id, &ft);
return t;
}
static void
fts_transaction_finish(struct mailbox *box, struct fts_transaction_context *ft,
bool committed)
{
struct fts_mailbox *fbox = FTS_CONTEXT(box);
if (ft->expunges) {
if (fbox->backend_fast != NULL) {
fts_backend_expunge_finish(fbox->backend_fast,
box, committed);
}
}
i_free(ft);
}
static void fts_transaction_rollback(struct mailbox_transaction_context *t)
{
struct mailbox *box = t->box;
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
fbox->super.transaction_rollback(t);
fts_transaction_finish(box, ft, FALSE);
}
static int fts_transaction_commit(struct mailbox_transaction_context *t,
enum mailbox_sync_flags flags)
{
struct mailbox *box = t->box;
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
int ret;
ret = fbox->super.transaction_commit(t, flags);
fts_transaction_finish(box, ft, ret == 0);
return ret;
}
void fts_mailbox_opened(struct mailbox *box)
{
struct fts_mailbox *fbox;
const char *env;
if (fts_next_hook_mailbox_opened != NULL)
fts_next_hook_mailbox_opened(box);
env = getenv("FTS");
if (env == NULL)
return;
fbox = i_new(struct fts_mailbox, 1);
fbox->env = env;
fbox->super = box->v;
box->v.close = fts_mailbox_close;
box->v.search_init = fts_mailbox_search_init;
box->v.search_next_nonblock = fts_mailbox_search_next_nonblock;
box->v.search_next_update_seq = fts_mailbox_search_next_update_seq;
box->v.search_deinit = fts_mailbox_search_deinit;
box->v.mail_alloc = fts_mail_alloc;
box->v.transaction_begin = fts_transaction_begin;
box->v.transaction_rollback = fts_transaction_rollback;
box->v.transaction_commit = fts_transaction_commit;
if (!fts_storage_module_id_set) {
fts_storage_module_id = mail_storage_module_id++;
fts_storage_module_id_set = TRUE;
}
array_idx_set(&box->module_contexts, fts_storage_module_id, &fbox);
}