fts-storage.c revision 3d4450c252790b03bb5ce054987ac91110f1ff62
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen/* Copyright (C) 2006 Timo Sirainen */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "lib.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "array.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "str.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "istream.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "message-parser.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "message-decoder.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "mail-search.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "mail-storage-private.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "fts-api-private.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "fts-plugin.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include <stdlib.h>
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#define FTS_CONTEXT(obj) \
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen *((void **)array_idx_modifiable(&(obj)->module_contexts, \
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen fts_storage_module_id))
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstruct fts_mailbox {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct mailbox_vfuncs super;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_backend *backend;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen};
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstruct fts_search_context {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ARRAY_TYPE(seq_range) result;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen unsigned int result_pos;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen unsigned int locked:1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen};
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstruct fts_transaction_context {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen bool expunges;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen};
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainenstruct fts_mail {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct mail_vfuncs super;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen};
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainenstatic unsigned int fts_storage_module_id = 0;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainenstatic bool fts_storage_module_id_set = FALSE;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic int fts_mailbox_close(struct mailbox *box)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen int ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen fts_backend_deinit(fbox->backend);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = fbox->super.close(box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen i_free(fbox);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
3fb1c1f0375ec0a2b00be90b5d61fbc8374e9b90Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic int uid_range_to_seq(struct mailbox *box,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ARRAY_TYPE(seq_range) *uid_range,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ARRAY_TYPE(seq_range) *seq_range)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen const struct seq_range *range;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct seq_range new_range;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen unsigned int i, count;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen range = array_get(uid_range, &count);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen i_array_init(seq_range, count);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen for (i = 0; i < count; i++) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (mailbox_get_uids(box, range[i].seq1, range[i].seq2,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen &new_range.seq1, &new_range.seq2) < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen array_free(seq_range);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (new_range.seq1 != 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen array_append(seq_range, &new_range, 1);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen return 0;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen}
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainenstruct fts_storage_build_context {
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen struct fts_backend_build_context *build;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen uint32_t uid;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen string_t *headers;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen bool save_part;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen};
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainenstatic int fts_build_mail_flush(struct fts_storage_build_context *ctx)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen{
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen if (str_len(ctx->headers) == 0)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen return 1;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen if (fts_backend_build_more(ctx->build, ctx->uid, str_data(ctx->headers),
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen str_len(ctx->headers)) < 0)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen return -1;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen str_truncate(ctx->headers, 0);
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen return 1;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen}
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainenstatic bool fts_build_update_save_part(struct fts_storage_build_context *ctx,
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen const struct message_block *block)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen /* we'll index only text/xxx and message/rfc822 parts for now */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if ((block->part->flags &
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen (MESSAGE_PART_FLAG_TEXT |
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen MESSAGE_PART_FLAG_MESSAGE_RFC822)) == 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return FALSE;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx->save_part = TRUE;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return TRUE;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic int fts_build_mail_header(struct fts_storage_build_context *ctx,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen const struct message_block *block)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen const struct message_header_line *hdr = block->hdr;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen /* hdr->full_value is always set because we get the block from
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen message_decoder */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_append(ctx->headers, hdr->name);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_append_n(ctx->headers, hdr->middle, hdr->middle_len);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_append_n(ctx->headers, hdr->full_value, hdr->full_value_len);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (!hdr->no_newline)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen str_append_c(ctx->headers, '\n');
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (!ctx->save_part) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (strcasecmp(hdr->name, "Content-Type") == 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (!fts_build_update_save_part(ctx, block))
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return 0;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return 1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return fts_build_mail_flush(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic int
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenfts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct istream *input;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct message_parser_ctx *parser;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct message_decoder_context *decoder;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct message_block raw_block, block;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct message_part *prev_part, *skip_part;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen int ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx->uid = mail->uid;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen input = mail_get_stream(mail, NULL, NULL);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (input == NULL)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen prev_part = skip_part = NULL;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen parser = message_parser_init(pool_datastack_create(), input);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen decoder = message_decoder_init();
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen for (;;) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = message_parser_parse_next_block(parser, &raw_block);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen i_assert(ret != 0);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (ret < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (input->stream_errno == 0)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen ret = 0;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen break;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen }
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen if (raw_block.part == skip_part)
c63544d7d2580c680b07f9569e87e9cebee383d5Timo Sirainen continue;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen if (!message_decoder_decode_next_block(decoder, &raw_block,
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen &block))
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen continue;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (block.part != prev_part &&
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen (block.hdr != NULL || block.size != 0)) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_truncate(ctx->headers, 0);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx->save_part = FALSE;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen prev_part = block.part;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen skip_part = NULL;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen }
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen if (block.hdr != NULL) {
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen ret = fts_build_mail_header(ctx, &block);
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen if (ret < 0)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen break;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (ret == 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen skip_part = raw_block.part;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen } else if (block.size == 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen /* end of headers */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fts_build_update_save_part(ctx, &block)) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = fts_build_mail_flush(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (ret < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen break;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen } else {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fts_backend_build_more(ctx->build, mail->uid,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen block.data,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen block.size) < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen break;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen (void)message_parser_deinit(&parser);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen message_decoder_deinit(&decoder);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic int fts_build_new(struct mailbox_transaction_context *t)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_storage_build_context ctx;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct mail_search_context *search_ctx;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct mail_search_seqset seqset;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct mail_search_arg search_arg;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct mail *mail;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen uint32_t last_uid, last_uid_locked;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen int ret = 0;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fts_backend_get_last_uid(fbox->backend, &last_uid) < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen memset(&seqset, 0, sizeof(seqset));
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (mailbox_get_uids(t->box, last_uid+1, (uint32_t)-1,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen &seqset.seq1, &seqset.seq2) < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (seqset.seq1 == 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen /* no new messages */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return 0;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen memset(&ctx, 0, sizeof(ctx));
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx.build = fts_backend_build_init(fbox->backend, &last_uid_locked);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (last_uid != last_uid_locked) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen /* changed, need to get again the sequences */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen i_assert(last_uid < last_uid_locked);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen last_uid = last_uid_locked;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (mailbox_get_uids(t->box, last_uid+1, (uint32_t)-1,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen &seqset.seq1, &seqset.seq2) < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen (void)fts_backend_build_deinit(ctx.build);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (seqset.seq1 == 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen /* no new messages */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen (void)fts_backend_build_deinit(ctx.build);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return 0;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen memset(&search_arg, 0, sizeof(search_arg));
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen search_arg.type = SEARCH_SEQSET;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen search_arg.value.seqset = &seqset;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx.headers = str_new(default_pool, 512);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen mail = mail_alloc(t, 0, NULL);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen search_ctx = mailbox_search_init(t, NULL, &search_arg, NULL);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen while (mailbox_search_next(search_ctx, mail) > 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fts_build_mail(&ctx, mail) < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen break;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (mailbox_search_deinit(&search_ctx) < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen mail_free(&mail);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fts_backend_build_deinit(ctx.build) < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_free(&ctx.headers);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
3fb1c1f0375ec0a2b00be90b5d61fbc8374e9b90Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic struct mail_search_context *
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenfts_mailbox_search_init(struct mailbox_transaction_context *t,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen const char *charset, struct mail_search_arg *args,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen const enum mail_sort_type *sort_program)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen struct mail_search_context *ctx;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen struct fts_search_context *fctx;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen ARRAY_TYPE(seq_range) uid_result;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx = fbox->super.search_init(t, charset, args, sort_program);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen fctx = i_new(struct fts_search_context, 1);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen array_idx_set(&ctx->module_contexts, fts_storage_module_id, &fctx);
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen /* FIXME: handle AND/OR. Maybe also header lookups? */
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen while (args != NULL &&
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen args->type != SEARCH_BODY &&
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen args->type != SEARCH_TEXT)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen args = args->next;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (args != NULL) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fts_build_new(t) < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ctx;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fts_backend_lock(fbox->backend) <= 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ctx;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen fctx->locked = TRUE;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen i_array_init(&uid_result, 64);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fts_backend_lookup(fbox->backend, args->value.str,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen &uid_result) < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen /* failed, fallback to reading everything */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen array_free(&uid_result);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fbox->backend->definite_lookups) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen args->match_always = TRUE;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen args->result = 1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen args = args->next;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen while (args != NULL) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (args->type == SEARCH_BODY ||
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen args->type == SEARCH_TEXT) {
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen if (fbox->backend->definite_lookups) {
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen args->match_always = TRUE;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen args->result = 1;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen }
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen if (fts_backend_filter(fbox->backend,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen args->value.str,
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen &uid_result) < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen /* failed, but we already have limited
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen the search, so just ignore this */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen break;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen args = args->next;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (array_is_created(&uid_result)) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen (void)uid_range_to_seq(t->box, &uid_result,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen &fctx->result);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen array_free(&uid_result);
4bffa20ff6fc61ba7b47375722e38be7f87dfe64Timo Sirainen }
4bffa20ff6fc61ba7b47375722e38be7f87dfe64Timo Sirainen }
4bffa20ff6fc61ba7b47375722e38be7f87dfe64Timo Sirainen return ctx;
4bffa20ff6fc61ba7b47375722e38be7f87dfe64Timo Sirainen}
4bffa20ff6fc61ba7b47375722e38be7f87dfe64Timo Sirainen
4bffa20ff6fc61ba7b47375722e38be7f87dfe64Timo Sirainenstatic int fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen struct seq_range *range;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen unsigned int count;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen uint32_t wanted_seq;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen int ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (!array_is_created(&fctx->result))
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return fbox->super.search_next_update_seq(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen do {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen range = array_get_modifiable(&fctx->result, &count);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen while (fctx->result_pos < count &&
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx->seq > range[fctx->result_pos].seq2)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen fctx->result_pos++;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fctx->result_pos == count)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return 0;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (ctx->seq > range[fctx->result_pos].seq1)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen range[fctx->result_pos].seq1 = ctx->seq+1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen else {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx->seq = range[fctx->result_pos].seq1 - 1;
c63544d7d2580c680b07f9569e87e9cebee383d5Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fctx->result_pos < count &&
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ctx->seq + 1 == range[fctx->result_pos].seq2)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen fctx->result_pos++;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen else
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen range[fctx->result_pos].seq1++;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen wanted_seq = ctx->seq + 1;
c63544d7d2580c680b07f9569e87e9cebee383d5Timo Sirainen ret = fbox->super.search_next_update_seq(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen } while (ret > 0 && wanted_seq != ctx->seq);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic int fts_mailbox_search_deinit(struct mail_search_context *ctx)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (fctx->locked)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen fts_backend_unlock(fbox->backend);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (array_is_created(&fctx->result))
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen array_free(&fctx->result);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen i_free(fctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return fbox->super.search_deinit(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
static int fts_mail_expunge(struct mail *_mail)
{
struct mail_private *mail = (struct mail_private *)_mail;
struct fts_mail *fmail = FTS_CONTEXT(mail);
struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
if (fmail->super.expunge(_mail) < 0)
return -1;
fts_backend_expunge(fbox->backend, _mail);
return 0;
}
static struct mail *
fts_mail_alloc(struct mailbox_transaction_context *t,
enum mail_fetch_field wanted_fields,
struct mailbox_header_lookup_ctx *wanted_headers)
{
struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
struct fts_mail *fmail;
struct mail *_mail;
struct mail_private *mail;
_mail = fbox->super.mail_alloc(t, wanted_fields, wanted_headers);
mail = (struct mail_private *)_mail;
ft->expunges = TRUE;
fmail = p_new(mail->pool, struct fts_mail, 1);
fmail->super = mail->v;
mail->v.expunge = fts_mail_expunge;
array_idx_set(&mail->module_contexts, fts_storage_module_id, &fmail);
return _mail;
}
static struct mailbox_transaction_context *
fts_transaction_begin(struct mailbox *box,
enum mailbox_transaction_flags flags)
{
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct mailbox_transaction_context *t;
struct fts_transaction_context *ft;
ft = i_new(struct fts_transaction_context, 1);
t = fbox->super.transaction_begin(box, flags);
array_idx_set(&t->module_contexts, fts_storage_module_id, &ft);
return t;
}
static void fts_transaction_rollback(struct mailbox_transaction_context *t)
{
struct mailbox *box = t->box;
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
fbox->super.transaction_rollback(t);
if (ft->expunges)
fts_backend_expunge_finish(fbox->backend, box, FALSE);
i_free(ft);
}
static int fts_transaction_commit(struct mailbox_transaction_context *t,
enum mailbox_sync_flags flags)
{
struct mailbox *box = t->box;
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
int ret;
ret = fbox->super.transaction_commit(t, flags);
if (ft->expunges)
fts_backend_expunge_finish(fbox->backend, box, ret == 0);
i_free(ft);
return ret;
}
void fts_mailbox_opened(struct mailbox *box)
{
struct fts_mailbox *fbox;
struct fts_backend *backend;
const char *env;
if (fts_next_hook_mailbox_opened != NULL)
fts_next_hook_mailbox_opened(box);
env = getenv("FTS");
if (env == NULL)
return;
backend = fts_backend_init(env, box);
if (backend == NULL)
return;
fbox = i_new(struct fts_mailbox, 1);
fbox->super = box->v;
fbox->backend = backend;
box->v.close = fts_mailbox_close;
box->v.search_init = fts_mailbox_search_init;
box->v.search_next_update_seq = fts_mailbox_search_next_update_seq;
box->v.search_deinit = fts_mailbox_search_deinit;
box->v.mail_alloc = fts_mail_alloc;
box->v.transaction_begin = fts_transaction_begin;
box->v.transaction_rollback = fts_transaction_rollback;
box->v.transaction_commit = fts_transaction_commit;
if (!fts_storage_module_id_set) {
fts_storage_module_id = mail_storage_module_id++;
fts_storage_module_id_set = TRUE;
}
array_idx_set(&box->module_contexts, fts_storage_module_id, &fbox);
}