fts-storage.c revision 437a8b0fe254057b0c1f1723d689bafa91cae2ab
2e37d45867d081db150ab78dad303b9077aea24fTimo Sirainen/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "lib.h"
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen#include "ioloop.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "array.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "str.h"
ec02103893ae3748e5b593340995bb2489c4b741Timo Sirainen#include "unichar.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "istream.h"
ba77fee05de8b4ffdb2b51b7840b21f46e174208Timo Sirainen#include "time-util.h"
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen#include "rfc822-parser.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "message-parser.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "message-decoder.h"
b8835b8a21c617ceb82ddc5a176243faf36aa8f7Timo Sirainen#include "mail-namespace.h"
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen#include "mail-search-build.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "mail-storage-private.h"
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen#include "mailbox-list-private.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "fts-api-private.h"
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen#include "fts-mailbox.h"
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen#include "fts-storage.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "fts-plugin.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include <stdlib.h>
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#define FTS_CONTEXT(obj) \
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT(obj, fts_storage_module)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#define FTS_MAIL_CONTEXT(obj) \
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT(obj, fts_mail_module)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
5e615a0e4682b75cadb5bea4bff689f3fcf8e9feTimo Sirainen#define FTS_SEARCH_NONBLOCK_COUNT 50
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen#define FTS_BUILD_NOTIFY_INTERVAL_SECS 10
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainenstruct fts_mail {
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen union mail_module_context module_ctx;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen char score[30];
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen};
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainenstruct fts_storage_build_context {
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct mail_search_context *search_ctx;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen struct mail_search_args *search_args;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct fts_backend_build_context *build;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen struct timeval search_start_time, last_notify;
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen uint32_t uid;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen string_t *headers;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen char *content_type, *content_disposition;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen};
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainenstruct fts_transaction_context {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen union mailbox_transaction_module_context module_ctx;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen struct fts_storage_build_context *build_ctx;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen ARRAY_TYPE(fts_score_map) *score_map;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen struct mail *mail;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen uint32_t last_uid;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen unsigned int free_mail:1;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen unsigned int expunges:1;
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen};
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_storage_module,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen &mail_storage_module_register);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
e03d986a74128f5ba30fcfda9f6e36578f5d8decTimo Sirainenstatic void fts_mailbox_free(struct mailbox *box)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fbox->backend_substr != NULL)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fts_backend_deinit(&fbox->backend_substr);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen if (fbox->backend_fast != NULL)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fts_backend_deinit(&fbox->backend_fast);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
e03d986a74128f5ba30fcfda9f6e36578f5d8decTimo Sirainen fbox->module_ctx.super.free(box);
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen i_free(fbox);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
c37e5edd83ff696d396131f7147ef971cf678911Timo Sirainenstatic int fts_build_mail_flush_headers(struct fts_storage_build_context *ctx)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen{
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen if (str_len(ctx->headers) == 0)
38a4c09de37bc2ebdc38427a2b958c46dfdcffb1Timo Sirainen return 0;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
ec02103893ae3748e5b593340995bb2489c4b741Timo Sirainen i_assert(uni_utf8_data_is_valid(ctx->headers->data, ctx->headers->used));
ec02103893ae3748e5b593340995bb2489c4b741Timo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen fts_backend_build_hdr(ctx->build, ctx->uid);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (fts_backend_build_more(ctx->build, str_data(ctx->headers),
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen str_len(ctx->headers)) < 0)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen return -1;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen str_truncate(ctx->headers, 0);
38a4c09de37bc2ebdc38427a2b958c46dfdcffb1Timo Sirainen return 0;
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen}
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainenstatic void fts_build_parse_content_type(struct fts_storage_build_context *ctx,
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen const struct message_header_line *hdr)
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen{
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen struct rfc822_parser_context parser;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen string_t *content_type;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen (void)rfc822_skip_lwsp(&parser);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen T_BEGIN {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen content_type = t_str_new(64);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (rfc822_parse_content_type(&parser, content_type) >= 0) {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_free(ctx->content_type);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen ctx->content_type = i_strdup(str_c(content_type));
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen }
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen } T_END;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen}
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainenstatic void
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainenfts_build_parse_content_disposition(struct fts_storage_build_context *ctx,
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen const struct message_header_line *hdr)
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen{
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen /* just pass it as-is to backend. */
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_free(ctx->content_disposition);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen ctx->content_disposition =
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_strndup(hdr->full_value, hdr->full_value_len);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen}
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainenstatic void fts_parse_mail_header(struct fts_storage_build_context *ctx,
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen const struct message_block *raw_block)
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen{
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen const struct message_header_line *hdr = raw_block->hdr;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (strcasecmp(hdr->name, "Content-Type") == 0)
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen fts_build_parse_content_type(ctx, hdr);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen else if (strcasecmp(hdr->name, "Content-Disposition") == 0)
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen fts_build_parse_content_disposition(ctx, hdr);
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen}
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen
38a4c09de37bc2ebdc38427a2b958c46dfdcffb1Timo Sirainenstatic void fts_build_mail_header(struct fts_storage_build_context *ctx,
38a4c09de37bc2ebdc38427a2b958c46dfdcffb1Timo Sirainen const struct message_block *block)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen const struct message_header_line *hdr = block->hdr;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen /* hdr->full_value is always set because we get the block from
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen message_decoder */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_append(ctx->headers, hdr->name);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_append_n(ctx->headers, hdr->middle, hdr->middle_len);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_append_n(ctx->headers, hdr->full_value, hdr->full_value_len);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (!hdr->no_newline)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen str_append_c(ctx->headers, '\n');
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainenstatic int
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainenfts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2116e38edb504e72eadaf5a51c3dd9b1a81039eaTimo Sirainen enum message_decoder_flags decoder_flags = MESSAGE_DECODER_FLAG_DTCASE;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct istream *input;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct message_parser_ctx *parser;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct message_decoder_context *decoder;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct message_block raw_block, block;
992118a50af940482b6cf884a89be56d7015580aTimo Sirainen struct message_part *prev_part, *parts;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen bool skip_body = FALSE, body_part = FALSE;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen int ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen ctx->uid = mail->uid;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen if (mail_get_stream(mail, NULL, NULL, &input) < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
38a4c09de37bc2ebdc38427a2b958c46dfdcffb1Timo Sirainen prev_part = NULL;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen parser = message_parser_init(pool_datastack_create(), input,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen 0);
2116e38edb504e72eadaf5a51c3dd9b1a81039eaTimo Sirainen
2116e38edb504e72eadaf5a51c3dd9b1a81039eaTimo Sirainen
2116e38edb504e72eadaf5a51c3dd9b1a81039eaTimo Sirainen if ((ctx->build->backend->flags & FTS_BACKEND_FLAG_BINARY_MIME_PARTS) != 0)
2116e38edb504e72eadaf5a51c3dd9b1a81039eaTimo Sirainen decoder_flags |= MESSAGE_DECODER_FLAG_RETURN_BINARY;
2116e38edb504e72eadaf5a51c3dd9b1a81039eaTimo Sirainen decoder = message_decoder_init(decoder_flags);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen for (;;) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = message_parser_parse_next_block(parser, &raw_block);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen i_assert(ret != 0);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (ret < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (input->stream_errno == 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = 0;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen break;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (raw_block.part != prev_part) {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen /* body part changed. we're now parsing the end of
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen boundary, possibly followed by message epilogue */
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (!skip_body && prev_part != NULL) {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_assert(body_part);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen fts_backend_build_body_end(ctx->build);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen }
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen prev_part = raw_block.part;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_free_and_null(ctx->content_type);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_free_and_null(ctx->content_disposition);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (raw_block.size != 0) {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen /* multipart. skip until beginning of next
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen part's headers */
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen skip_body = TRUE;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen }
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen }
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (raw_block.hdr != NULL) {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen /* always handle headers */
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen } else if (raw_block.size == 0) {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen /* end of headers */
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen const char *content_type = ctx->content_type == NULL ?
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen "text/plain" : ctx->content_type;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen skip_body = !fts_backend_build_body_begin(ctx->build,
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen ctx->uid, content_type,
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen ctx->content_disposition);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen body_part = TRUE;
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen } else {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (skip_body)
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen continue;
38a4c09de37bc2ebdc38427a2b958c46dfdcffb1Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (!message_decoder_decode_next_block(decoder, &raw_block,
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen &block))
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen continue;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (block.hdr != NULL) {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen fts_parse_mail_header(ctx, &raw_block);
38a4c09de37bc2ebdc38427a2b958c46dfdcffb1Timo Sirainen fts_build_mail_header(ctx, &block);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen } else if (block.size == 0) {
38a4c09de37bc2ebdc38427a2b958c46dfdcffb1Timo Sirainen /* end of headers */
c37e5edd83ff696d396131f7147ef971cf678911Timo Sirainen str_append_c(ctx->headers, '\n');
0f55802e8fdd95ae4290da6da077819209b71f70Timo Sirainen } else {
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_assert(body_part);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (fts_backend_build_more(ctx->build,
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen block.data, block.size) < 0) {
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen break;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen if (!skip_body && body_part)
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen fts_backend_build_body_end(ctx->build);
992118a50af940482b6cf884a89be56d7015580aTimo Sirainen if (message_parser_deinit(&parser, &parts) < 0)
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen mail_set_cache_corrupted(mail, MAIL_FETCH_MESSAGE_PARTS);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen message_decoder_deinit(&decoder);
c37e5edd83ff696d396131f7147ef971cf678911Timo Sirainen
c37e5edd83ff696d396131f7147ef971cf678911Timo Sirainen if (ret == 0) {
c37e5edd83ff696d396131f7147ef971cf678911Timo Sirainen /* Index all headers at the end. This is required for Squat,
c37e5edd83ff696d396131f7147ef971cf678911Timo Sirainen because it can handle only incremental UIDs. */
c37e5edd83ff696d396131f7147ef971cf678911Timo Sirainen ret = fts_build_mail_flush_headers(ctx);
c37e5edd83ff696d396131f7147ef971cf678911Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenstatic int fts_build_init_seq(struct fts_search_context *fctx,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct fts_backend *backend,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct mailbox_transaction_context *t,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uint32_t seq1, uint32_t seq2, uint32_t last_uid)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen struct mail_search_args *search_args;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct fts_storage_build_context *ctx;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct fts_backend_build_context *build;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uint32_t last_uid_locked;
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen
e22b857e838fe118de3f78513aad6a3c6f4306b3Timo Sirainen if (fctx->best_arg->type == SEARCH_HEADER ||
e22b857e838fe118de3f78513aad6a3c6f4306b3Timo Sirainen fctx->best_arg->type == SEARCH_HEADER_COMPRESS_LWSP) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* we're not updating the index just for header lookups */
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (seq1 < fctx->first_nonindexed_seq)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen fctx->first_nonindexed_seq = seq1;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen return 0;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fts_backend_build_init(backend, &last_uid_locked, &build) < 0)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen return -1;
48566ca412a7cf3b42512fd0ec112744778e5da0Timo Sirainen if (last_uid != last_uid_locked && last_uid_locked != (uint32_t)-1) {
c06cd6539a3dbd68eb546464076187be6bc4290fTimo Sirainen /* changed, need to get again the sequences */
c06cd6539a3dbd68eb546464076187be6bc4290fTimo Sirainen last_uid = last_uid_locked;
5afa8e2edf4f313cd56e5909f92f39c3b5b7b4d3Timo Sirainen mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1,
5afa8e2edf4f313cd56e5909f92f39c3b5b7b4d3Timo Sirainen &seq1, &seq2);
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen if (seq1 == 0) {
c06cd6539a3dbd68eb546464076187be6bc4290fTimo Sirainen /* no new messages */
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen (void)fts_backend_build_deinit(&build);
c06cd6539a3dbd68eb546464076187be6bc4290fTimo Sirainen return 0;
c06cd6539a3dbd68eb546464076187be6bc4290fTimo Sirainen }
c06cd6539a3dbd68eb546464076187be6bc4290fTimo Sirainen }
c06cd6539a3dbd68eb546464076187be6bc4290fTimo Sirainen
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen search_args = mail_search_build_init();
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen mail_search_build_add_seqset(search_args, seq1, seq2);
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen ctx = i_new(struct fts_storage_build_context, 1);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen ctx->build = build;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen ctx->headers = str_new(default_pool, 512);
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen ctx->search_ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen ctx->search_ctx->progress_hidden = TRUE;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen ctx->search_args = search_args;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen fctx->build_ctx = ctx;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return 1;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenstatic struct fts_backend *
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenfts_mailbox_get_backend(struct fts_search_context *fctx,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct mailbox *box)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (fctx->build_backend == fctx->fbox->backend_fast)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return fbox->backend_fast;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen else {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen i_assert(fctx->build_backend == fctx->fbox->backend_substr);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return fbox->backend_substr;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenstatic int fts_build_init_trans(struct fts_search_context *fctx,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct mailbox_transaction_context *t)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct fts_backend *backend;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uint32_t last_uid, seq1, seq2;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen int ret;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen backend = fts_mailbox_get_backend(fctx, t->box);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (fts_backend_get_last_uid(backend, &last_uid) < 0)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return -1;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1, &seq1, &seq2);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (seq1 == 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* no new messages */
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return 0;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen ret = fts_build_init_seq(fctx, backend, t, seq1, seq2, last_uid);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return ret < 0 ? -1 : 0;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenstatic int
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenfts_build_init_box(struct fts_search_context *fctx, struct mailbox *box,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uint32_t last_uid)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct fts_backend *backend;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uint32_t seq1, seq2;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen mailbox_get_seq_range(box, last_uid + 1, (uint32_t)-1, &seq1, &seq2);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (seq1 == 0)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return 0;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen backend = fts_mailbox_get_backend(fctx, box);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen fctx->virtual_ctx.trans = mailbox_transaction_begin(box, 0);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return fts_build_init_seq(fctx, backend, fctx->virtual_ctx.trans,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen seq1, seq2, last_uid);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainenstatic int mailbox_name_cmp(const struct fts_orig_mailboxes *box1,
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen const struct fts_orig_mailboxes *box2)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
af3f857bb3166ed99595e11a9d18e5b5cc670e1aTimo Sirainen int ret;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
af3f857bb3166ed99595e11a9d18e5b5cc670e1aTimo Sirainen T_BEGIN {
af3f857bb3166ed99595e11a9d18e5b5cc670e1aTimo Sirainen const char *vname1, *vname2;
af3f857bb3166ed99595e11a9d18e5b5cc670e1aTimo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen vname1 = mailbox_list_get_vname(box1->ns->list, box1->name);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen vname2 = mailbox_list_get_vname(box2->ns->list, box2->name);
af3f857bb3166ed99595e11a9d18e5b5cc670e1aTimo Sirainen ret = strcmp(vname1, vname2);
af3f857bb3166ed99595e11a9d18e5b5cc670e1aTimo Sirainen } T_END;
af3f857bb3166ed99595e11a9d18e5b5cc670e1aTimo Sirainen return ret;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainenstatic int
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainenfts_backend_uid_map_mailbox_cmp(const struct fts_backend_uid_map *map1,
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen const struct fts_backend_uid_map *map2)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return strcmp(map1->mailbox, map2->mailbox);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenstatic int fts_build_init_virtual_next(struct fts_search_context *fctx)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct fts_search_virtual_context *vctx = &fctx->virtual_ctx;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct mailbox_status status;
ff01c351d308504551048039304725d578978c2eTimo Sirainen const struct fts_orig_mailboxes *boxes;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen const struct fts_backend_uid_map *last_uids;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen unsigned int boxi, uidi, box_count, last_uid_count;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen int ret, vret = 0;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (vctx->pool == NULL)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return 0;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (fctx->virtual_ctx.trans != NULL)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen (void)mailbox_transaction_commit(&fctx->virtual_ctx.trans);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
ff01c351d308504551048039304725d578978c2eTimo Sirainen boxes = array_get(&vctx->orig_mailboxes, &box_count);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen last_uids = array_get(&vctx->last_uids, &last_uid_count);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen boxi = vctx->boxi;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uidi = vctx->uidi;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen while (vret == 0 && boxi < box_count && uidi < last_uid_count) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen T_BEGIN {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen const char *vname;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen vname = mailbox_list_get_vname(boxes[boxi].ns->list,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen boxes[boxi].name);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ret = strcmp(vname, last_uids[uidi].mailbox);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen } T_END;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (ret == 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* match. check also that uidvalidity matches. */
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen mailbox_get_open_status(boxes[boxi].box,
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen STATUS_UIDVALIDITY, &status);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (status.uidvalidity != last_uids[uidi].uidvalidity) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uidi++;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen continue;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
ff01c351d308504551048039304725d578978c2eTimo Sirainen vret = fts_build_init_box(fctx, boxes[boxi].box,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen last_uids[uidi].uid);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen boxi++;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uidi++;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen } else if (ret > 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* not part of this virtual mailbox */
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uidi++;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen } else {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* no messages indexed in the mailbox */
ff01c351d308504551048039304725d578978c2eTimo Sirainen vret = fts_build_init_box(fctx, boxes[boxi].box, 0);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen boxi++;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen while (vret == 0 && boxi < box_count) {
ff01c351d308504551048039304725d578978c2eTimo Sirainen vret = fts_build_init_box(fctx, boxes[boxi].box, 0);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen boxi++;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen vctx->boxi = boxi;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen vctx->uidi = uidi;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return vret;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
ff01c351d308504551048039304725d578978c2eTimo Sirainenstatic const char *
ff01c351d308504551048039304725d578978c2eTimo Sirainenfts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r)
ff01c351d308504551048039304725d578978c2eTimo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen struct mail_namespace *ns = mailbox_get_namespace(box);
ff01c351d308504551048039304725d578978c2eTimo Sirainen const char *name = box->name;
ff01c351d308504551048039304725d578978c2eTimo Sirainen
ff01c351d308504551048039304725d578978c2eTimo Sirainen while (ns->alias_for != NULL)
ff01c351d308504551048039304725d578978c2eTimo Sirainen ns = ns->alias_for;
ff01c351d308504551048039304725d578978c2eTimo Sirainen *ns_r = ns;
ff01c351d308504551048039304725d578978c2eTimo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (*name == '\0' && ns != mailbox_get_namespace(box) &&
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen (ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) {
ff01c351d308504551048039304725d578978c2eTimo Sirainen /* ugly workaround to allow selecting INBOX from a Maildir/
ff01c351d308504551048039304725d578978c2eTimo Sirainen when it's not in the inbox=yes namespace. */
ff01c351d308504551048039304725d578978c2eTimo Sirainen return "INBOX";
ff01c351d308504551048039304725d578978c2eTimo Sirainen }
ff01c351d308504551048039304725d578978c2eTimo Sirainen return name;
ff01c351d308504551048039304725d578978c2eTimo Sirainen}
ff01c351d308504551048039304725d578978c2eTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenstatic int fts_build_init_virtual(struct fts_search_context *fctx)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen struct fts_search_virtual_context *vctx = &fctx->virtual_ctx;
ff01c351d308504551048039304725d578978c2eTimo Sirainen ARRAY_TYPE(mailboxes) mailboxes;
ff01c351d308504551048039304725d578978c2eTimo Sirainen struct mailbox *const *boxes;
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen const struct fts_orig_mailboxes *orig_boxes;
ff01c351d308504551048039304725d578978c2eTimo Sirainen struct fts_orig_mailboxes orig_box;
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen unsigned int i, box_count;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen int ret;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
ff01c351d308504551048039304725d578978c2eTimo Sirainen t_array_init(&mailboxes, 64);
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen fts_mailbox_get_virtual_backend_boxes(fctx->t->box, &mailboxes, TRUE);
ff01c351d308504551048039304725d578978c2eTimo Sirainen boxes = array_get_modifiable(&mailboxes, &box_count);
ff01c351d308504551048039304725d578978c2eTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen vctx->pool = pool_alloconly_create("fts virtual build", 1024);
ff01c351d308504551048039304725d578978c2eTimo Sirainen p_array_init(&vctx->orig_mailboxes, vctx->pool, box_count);
ff01c351d308504551048039304725d578978c2eTimo Sirainen memset(&orig_box, 0, sizeof(orig_box));
ff01c351d308504551048039304725d578978c2eTimo Sirainen for (i = 0; i < box_count; i++) {
ff01c351d308504551048039304725d578978c2eTimo Sirainen orig_box.box = boxes[i];
ff01c351d308504551048039304725d578978c2eTimo Sirainen orig_box.name = fts_box_get_root(boxes[i], &orig_box.ns);
ff01c351d308504551048039304725d578978c2eTimo Sirainen array_append(&vctx->orig_mailboxes, &orig_box, 1);
ff01c351d308504551048039304725d578978c2eTimo Sirainen }
ff01c351d308504551048039304725d578978c2eTimo Sirainen
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen orig_boxes = array_get(&vctx->orig_mailboxes, &box_count);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (box_count <= 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (box_count == 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* empty virtual mailbox */
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return 0;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* virtual mailbox is built from only a single mailbox
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen (currently). check that directly. */
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen fctx->virtual_ctx.trans =
ff01c351d308504551048039304725d578978c2eTimo Sirainen mailbox_transaction_begin(orig_boxes[0].box, 0);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen ret = fts_build_init_trans(fctx, fctx->virtual_ctx.trans);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return ret;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* virtual mailbox is built from multiple mailboxes. figure out which
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen ones need updating. */
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen p_array_init(&vctx->last_uids, vctx->pool, 64);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (fts_backend_get_all_last_uids(fctx->build_backend, vctx->pool,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen &vctx->last_uids) < 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen pool_unref(&vctx->pool);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return -1;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen array_sort(&vctx->orig_mailboxes, mailbox_name_cmp);
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen array_sort(&vctx->last_uids, fts_backend_uid_map_mailbox_cmp);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen ret = fts_build_init_virtual_next(fctx);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen return ret < 0 ? -1 : 0;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenstatic int fts_build_init(struct fts_search_context *fctx)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen struct mailbox_status status;
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen int ret;
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen mailbox_get_open_status(fctx->t->box, STATUS_MESSAGES | STATUS_UIDNEXT,
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen &status);
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen if (status.messages == fctx->fbox->last_messages_count &&
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen status.uidnext == fctx->fbox->last_uidnext) {
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen /* no new messages since last check */
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen return 0;
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen }
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen
b86dc9680d4a15312c5f0607c80402a19f659eb6Timo Sirainen if (fctx->fbox->virtual &&
b86dc9680d4a15312c5f0607c80402a19f659eb6Timo Sirainen (fctx->build_backend->flags & FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS) != 0)
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen ret = fts_build_init_virtual(fctx);
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen else
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen ret = fts_build_init_trans(fctx, fctx->t);
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen if (ret == 0 && fctx->build_ctx == NULL) {
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen /* index was up-to-date */
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen fctx->fbox->last_messages_count = status.messages;
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen fctx->fbox->last_uidnext = status.uidnext;
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen }
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen return ret;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen}
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainenstatic int fts_build_deinit(struct fts_storage_build_context **_ctx)
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen{
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen struct fts_storage_build_context *ctx = *_ctx;
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen struct mailbox *box = ctx->search_ctx->transaction->box;
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen struct mailbox_status status;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen int ret = 0;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen *_ctx = NULL;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen if (mailbox_search_deinit(&ctx->search_ctx) < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fts_backend_build_deinit(&ctx->build) < 0)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen ret = -1;
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen if (ret == 0) {
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT,
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen &status);
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen fbox->last_messages_count = status.messages;
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen fbox->last_uidnext = status.uidnext;
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen }
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen if (ioloop_time - ctx->search_start_time.tv_sec >=
e4f7d8e459a030149e09656d27ccf7a55c274dbdTimo Sirainen FTS_BUILD_NOTIFY_INTERVAL_SECS &&
e4f7d8e459a030149e09656d27ccf7a55c274dbdTimo Sirainen box->storage->callbacks.notify_ok != NULL) {
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen /* we notified at least once */
ee116df08d0fdab703483e18fe8076b2ef9fd9d7Timo Sirainen box->storage->callbacks.
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen notify_ok(box, "Mailbox indexing finished",
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen box->storage->callback_context);
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen }
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen str_free(&ctx->headers);
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen mail_search_args_unref(&ctx->search_args);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_free(ctx->content_type);
12055678401e913f4be130fa41b22fbeb626cc7eTimo Sirainen i_free(ctx->content_disposition);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen i_free(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainenstatic void
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainenfts_build_notify(struct fts_storage_build_context *ctx, uint32_t seq)
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen{
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen struct mailbox *box = ctx->search_ctx->transaction->box;
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen if (ctx->last_notify.tv_sec == 0) {
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen /* set the search time in here, in case a plugin
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen already spent some time indexing the mailbox */
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen ctx->search_start_time = ioloop_timeval;
ee116df08d0fdab703483e18fe8076b2ef9fd9d7Timo Sirainen } else if (box->storage->callbacks.notify_ok != NULL) {
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen double completed_frac;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen unsigned int eta_secs;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen const struct seq_range *range;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen uint32_t seq_diff;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen range = array_idx(&ctx->search_args->args->value.seqset, 0);
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen seq_diff = range->seq2 - range->seq1;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen if (seq_diff != 0) {
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen completed_frac = (double)(seq - range->seq1) / seq_diff;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen if (completed_frac >= 0.000001) {
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen unsigned int elapsed_msecs, est_total_msecs;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen elapsed_msecs = timeval_diff_msecs(&ioloop_timeval,
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen &ctx->search_start_time);
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen est_total_msecs = elapsed_msecs / completed_frac;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen eta_secs = (est_total_msecs - elapsed_msecs) / 1000;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen } else {
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen eta_secs = 0;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen }
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen } else {
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen completed_frac = 0.0;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen eta_secs = 0;
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen }
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen T_BEGIN {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen const char *text;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen text = t_strdup_printf("Indexed %d%% of the mailbox, "
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen "ETA %d:%02d", (int)(completed_frac * 100.0),
c17099a136754ce37e346f6f98dfa1fe709ee772Timo Sirainen eta_secs/60, eta_secs%60);
ee116df08d0fdab703483e18fe8076b2ef9fd9d7Timo Sirainen box->storage->callbacks.
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen notify_ok(box, text,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen box->storage->callback_context);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen } T_END;
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen }
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen ctx->last_notify = ioloop_timeval;
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen}
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainenstatic int fts_build_more(struct fts_storage_build_context *ctx)
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen{
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen struct mail *mail = NULL;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen unsigned int count = 0;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen int ret;
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen while (mailbox_search_next(ctx->search_ctx, &mail)) {
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen T_BEGIN {
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen ret = fts_build_mail(ctx, mail);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen } T_END;
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen if (ret < 0)
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen return -1;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen if (++count == FTS_SEARCH_NONBLOCK_COUNT)
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen return 0;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen }
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen if (ioloop_time - ctx->last_notify.tv_sec >=
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen FTS_BUILD_NOTIFY_INTERVAL_SECS && mail != NULL)
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen fts_build_notify(ctx, mail->seq);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen return 1;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainenstatic void fts_search_init_lookup(struct mail_search_context *ctx,
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen struct fts_search_context *fctx)
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen{
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen fts_search_lookup(fctx);
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen if (fctx->seqs_set &&
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen strcmp(ctx->transaction->box->storage->name, "virtual") != 0) {
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen ctx->progress_max = array_count(&fctx->definite_seqs) +
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen array_count(&fctx->maybe_seqs);
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen }
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen ctx->progress_cur = 0;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen}
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainenstatic bool fts_try_build_init(struct mail_search_context *ctx,
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen struct fts_search_context *fctx)
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen{
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fctx->build_backend == NULL) {
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen fctx->build_initialized = TRUE;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen return TRUE;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen }
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fts_backend_is_building(fctx->build_backend)) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* this process is already building the indexes */
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen return FALSE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen fctx->build_initialized = TRUE;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fts_build_init(fctx) < 0) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fctx->build_backend = NULL;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen return TRUE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fctx->build_ctx == NULL) {
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen /* the index was up to date */
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen fts_search_init_lookup(ctx, fctx);
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen } else {
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen /* hide "searching" notifications */
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen ctx->progress_hidden = TRUE;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen }
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen return TRUE;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen}
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainenstatic struct mail_search_context *
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainenfts_mailbox_search_init(struct mailbox_transaction_context *t,
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen struct mail_search_args *args,
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen const enum mail_sort_type *sort_program,
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen enum mail_fetch_field wanted_fields,
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen struct mailbox_header_lookup_ctx *wanted_headers)
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen{
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct mail_search_context *ctx;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct fts_search_context *fctx;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen ctx = fbox->module_ctx.super.search_init(t, args, sort_program,
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen wanted_fields, wanted_headers);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen fctx = i_new(struct fts_search_context, 1);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fctx->fbox = fbox;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen fctx->t = t;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen fctx->args = args;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen fctx->first_nonindexed_seq = (uint32_t)-1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fbox->backend_substr == NULL && fbox->backend_fast == NULL)
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen return ctx;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen ft->score_map = &fctx->score_map;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fts_search_analyze(fctx);
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen (void)fts_try_build_init(ctx, fctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return ctx;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainenstatic int fts_mailbox_search_build_more(struct mail_search_context *ctx)
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen{
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen int ret = 1;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen while (fctx->build_ctx != NULL) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* this command is still building the indexes */
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen ret = fts_build_more(fctx->build_ctx);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (ret == 0)
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen return 0;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen /* finished / error */
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen ctx->progress_hidden = FALSE;
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen if (fts_build_deinit(&fctx->build_ctx) < 0)
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen ret = -1;
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen if (ret > 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (fts_build_init_virtual_next(fctx) == 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* all finished */
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen fts_search_init_lookup(ctx, fctx);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen }
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen }
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen return ret;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen}
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainenstatic bool
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainenfts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen struct mail **mail_r, bool *tryagain_r)
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen{
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (!fctx->build_initialized) {
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen /* we're still waiting for this process (but another command)
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen to finish building the indexes */
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (!fts_try_build_init(ctx, fctx)) {
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen *tryagain_r = TRUE;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen return FALSE;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen }
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen }
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (fts_mailbox_search_build_more(ctx) == 0) {
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen *tryagain_r = TRUE;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen return FALSE;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* if we're here, the indexes are either built or they're not used */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return fbox->module_ctx.super.
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen search_next_nonblock(ctx, mail_r, tryagain_r);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen}
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainenstatic void
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainenfts_mailbox_search_args_definite_set(struct fts_search_context *fctx)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen{
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen struct mail_search_arg *arg;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen for (arg = fctx->args->args; arg != NULL; arg = arg->next) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen switch (arg->type) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen case SEARCH_TEXT:
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen case SEARCH_BODY:
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen if (fctx->fbox->backend_substr == NULL) {
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen /* we're marking only fast args */
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen break;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen case SEARCH_BODY_FAST:
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen case SEARCH_TEXT_FAST:
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen arg->result = 1;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen break;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen default:
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen break;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen}
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainenstatic bool search_nonindexed(struct mail_search_context *ctx)
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen{
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen struct mailbox_status status;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen mailbox_get_open_status(ctx->transaction->box,
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen STATUS_MESSAGES, &status);
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen fctx->seqs_set = FALSE;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen ctx->seq = fctx->first_nonindexed_seq - 1;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen ctx->progress_cur = ctx->seq;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen ctx->progress_max = status.messages;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen return fbox->module_ctx.super.search_next_update_seq(ctx);
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen}
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen
bd1b2615928a1e8be190cb0405754f0aec8cac2fTimo Sirainenstatic bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen struct seq_range *def_range, *maybe_range, *range;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen unsigned int def_count, maybe_count;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen uint32_t wanted_seq;
bd1b2615928a1e8be190cb0405754f0aec8cac2fTimo Sirainen bool use_maybe, ret;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (!fctx->seqs_set)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return fbox->module_ctx.super.search_next_update_seq(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen wanted_seq = ctx->seq + 1;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* fts_search_lookup() was called successfully */
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen for (;;) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen def_range = array_get_modifiable(&fctx->definite_seqs,
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen &def_count);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen maybe_range = array_get_modifiable(&fctx->maybe_seqs,
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen &maybe_count);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* if we're ahead of current positions, skip them */
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen while (fctx->definite_idx < def_count &&
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen wanted_seq > def_range[fctx->definite_idx].seq2)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fctx->definite_idx++;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen while (fctx->maybe_idx < maybe_count &&
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen wanted_seq > maybe_range[fctx->maybe_idx].seq2)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fctx->maybe_idx++;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* use whichever is lower of definite/maybe */
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fctx->definite_idx == def_count) {
a1f517528af314cb6f5f39187a71dbed8a6e1152Timo Sirainen if (fctx->maybe_idx == maybe_count) {
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen /* look for the non-indexed mails */
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (fctx->first_nonindexed_seq == (uint32_t)-1)
bd1b2615928a1e8be190cb0405754f0aec8cac2fTimo Sirainen return FALSE;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen return search_nonindexed(ctx);
a1f517528af314cb6f5f39187a71dbed8a6e1152Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen use_maybe = TRUE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen } else if (fctx->maybe_idx == maybe_count) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen use_maybe = FALSE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen } else {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen use_maybe = maybe_range[fctx->maybe_idx].seq1 <
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen def_range[fctx->definite_idx].seq2;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (use_maybe)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen range = maybe_range + fctx->maybe_idx;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen else
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen range = def_range + fctx->definite_idx;
4bffa20ff6fc61ba7b47375722e38be7f87dfe64Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen i_assert(range->seq1 <= range->seq2);
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen if (wanted_seq > range->seq1) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* current sequence is already larger than where
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen range begins, so use the current sequence. */
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen range->seq1 = wanted_seq+1;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen } else {
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen wanted_seq = range->seq1;
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen range->seq1++;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen }
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen if (range->seq1 > range->seq2)
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen range->seq2 = 0;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* ctx->seq points to previous sequence we want */
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen ctx->seq = wanted_seq - 1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen ret = fbox->module_ctx.super.search_next_update_seq(ctx);
bd1b2615928a1e8be190cb0405754f0aec8cac2fTimo Sirainen if (!ret || wanted_seq == ctx->seq)
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen break;
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen wanted_seq = ctx->seq;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen mail_search_args_reset(ctx->args->args, FALSE);
c63c3c4d548416914b8c6734fe18dd69bb900775Timo Sirainen }
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (!use_maybe) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* we have definite results, update args */
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fts_mailbox_search_args_definite_set(fctx);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (ctx->seq + 1 >= fctx->first_nonindexed_seq) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* this is a virtual mailbox and we're searching headers.
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen some mailboxes had more messages indexed than others.
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen to avoid duplicates or jumping around, ignore the rest of
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen the search results and just go through the messages in
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen order. */
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen return search_nonindexed(ctx);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen ctx->progress_cur = fctx->definite_idx + fctx->maybe_idx;
94f90df2cfb7587bb5af432b2ba065d1c364e1f7Timo Sirainen return ret;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainenstatic bool
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainenfts_mailbox_search_next_update_seq_virtual(struct mail_search_context *ctx)
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen{
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen while (fbox->module_ctx.super.search_next_update_seq(ctx)) {
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen if (!fctx->seqs_set)
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen return TRUE;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen /* virtual mailbox searches don't return sequences sorted.
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen just check if the suggested sequence exists. */
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen if (seq_range_exists(&fctx->definite_seqs, ctx->seq)) {
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen fts_mailbox_search_args_definite_set(fctx);
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen return TRUE;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen }
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen if (seq_range_exists(&fctx->maybe_seqs, ctx->seq))
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen return TRUE;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen mail_search_args_reset(ctx->args->args, FALSE);
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen }
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen return FALSE;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen}
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic int fts_mailbox_search_deinit(struct mail_search_context *ctx)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen if (ft->score_map == &fctx->score_map)
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen ft->score_map = NULL;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen if (fctx->build_ctx != NULL) {
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen /* the search was cancelled */
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen (void)fts_build_deinit(&fctx->build_ctx);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen }
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (array_is_created(&fctx->definite_seqs))
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen array_free(&fctx->definite_seqs);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (array_is_created(&fctx->maybe_seqs))
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen array_free(&fctx->maybe_seqs);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen if (array_is_created(&fctx->score_map))
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen array_free(&fctx->score_map);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (fctx->virtual_ctx.trans != NULL)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen (void)mailbox_transaction_commit(&fctx->virtual_ctx.trans);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (fctx->virtual_ctx.pool != NULL)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen pool_unref(&fctx->virtual_ctx.pool);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen i_free(fctx);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return fbox->module_ctx.super.search_deinit(ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainenstatic void fts_mail_expunge(struct mail *_mail)
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen{
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
a1761856683b4bf745eb4e32cefabeb851efb301Timo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen
a1761856683b4bf745eb4e32cefabeb851efb301Timo Sirainen ft->expunges = TRUE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fbox->backend_substr != NULL)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fts_backend_expunge(fbox->backend_substr, _mail);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen if (fbox->backend_fast != NULL)
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen fts_backend_expunge(fbox->backend_fast, _mail);
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen fmail->module_ctx.super.expunge(_mail);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen}
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainenstatic int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score)
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen{
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen return *uid < score->uid ? -1 :
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen (*uid > score->uid ? 1 : 0);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen}
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainenstatic int fts_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen const char **value_r)
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen{
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen const struct fts_score_map *scores;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen if (field != MAIL_FETCH_SEARCH_SCORE || ft->score_map == NULL ||
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen !array_is_created(ft->score_map))
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen scores = NULL;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen else {
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen scores = array_bsearch(ft->score_map, &_mail->uid,
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen fts_score_cmp);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen }
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen if (scores != NULL) {
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen i_assert(scores->uid == _mail->uid);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen i_snprintf(fmail->score, sizeof(fmail->score),
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen "%f", scores->score);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen *value_r = fmail->score;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen return 0;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen }
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen return fmail->module_ctx.super.get_special(_mail, field, value_r);
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen}
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainenvoid fts_mail_allocated(struct mail *_mail)
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen{
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen struct mail_vfuncs *v = mail->vlast;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_mail *fmail;
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen if (fbox == NULL ||
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen (fbox->backend_substr == NULL && fbox->backend_fast == NULL))
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen return;
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen fmail = p_new(mail->pool, struct fts_mail, 1);
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen fmail->module_ctx.super = *v;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen mail->vlast = &fmail->module_ctx.super;
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->expunge = fts_mail_expunge;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->get_special = fts_mail_get_special;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen MODULE_CONTEXT_SET(mail, fts_mail_module, fmail);
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen}
fda168427e1950518acd6d600f1a10a29a5baef0Timo Sirainen
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainenstatic void fts_box_backends_init(struct mailbox *box)
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen{
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen struct fts_backend *backend;
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen const char *const *tmp;
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen for (tmp = t_strsplit(fbox->env, ", "); *tmp != NULL; tmp++) {
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen backend = fts_backend_init(*tmp, box);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen if (backend == NULL)
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen continue;
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen if ((backend->flags &
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS) != 0) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (fbox->backend_substr != NULL) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen i_fatal("fts: duplicate substring backend: %s",
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen *tmp);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fbox->backend_substr = backend;
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen } else {
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen if (fbox->backend_fast != NULL) {
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen i_fatal("fts: duplicate fast backend: %s",
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen *tmp);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen }
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen fbox->backend_fast = backend;
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen }
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (box->storage->set->mail_debug &&
14bb36cbb67b42e32105c3d843a8c974dc7ed436Timo Sirainen fbox->backend_substr == NULL && fbox->backend_fast == NULL)
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk i_debug("fts: No backends enabled by the fts setting");
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen}
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainenstatic struct mailbox_transaction_context *
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainenfts_transaction_begin(struct mailbox *box,
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen enum mailbox_transaction_flags flags)
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen{
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct mailbox_transaction_context *t;
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct fts_transaction_context *ft;
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen ft = i_new(struct fts_transaction_context, 1);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
a1761856683b4bf745eb4e32cefabeb851efb301Timo Sirainen /* the backend creation is delayed until the first transaction is
a1761856683b4bf745eb4e32cefabeb851efb301Timo Sirainen started. at that point the mailbox has been synced at least once. */
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen if (!fbox->backend_set) {
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen fts_box_backends_init(box);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen fbox->backend_set = TRUE;
a1761856683b4bf745eb4e32cefabeb851efb301Timo Sirainen }
a1761856683b4bf745eb4e32cefabeb851efb301Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen t = fbox->module_ctx.super.transaction_begin(box, flags);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT_SET(t, fts_storage_module, ft);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen return t;
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen}
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainenstatic void
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainenfts_storage_build_context_deinit(struct fts_storage_build_context *build_ctx)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen{
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen (void)fts_backend_build_deinit(&build_ctx->build);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen str_free(&build_ctx->headers);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen i_free(build_ctx);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen}
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainenstatic void
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainenfts_transaction_finish(struct mailbox *box, struct fts_transaction_context *ft,
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen bool committed)
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen{
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen if (ft->expunges) {
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen if (fbox->backend_fast != NULL) {
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen fts_backend_expunge_finish(fbox->backend_fast,
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen box, committed);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen }
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen }
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen i_free(ft);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen}
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainenstatic void fts_transaction_rollback(struct mailbox_transaction_context *t)
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen{
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct mailbox *box = t->box;
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (ft->build_ctx != NULL) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fts_storage_build_context_deinit(ft->build_ctx);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen ft->build_ctx = NULL;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (ft->free_mail)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen mail_free(&ft->mail);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen fbox->module_ctx.super.transaction_rollback(t);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen fts_transaction_finish(box, ft, FALSE);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen}
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainenstatic int
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainenfts_transaction_commit(struct mailbox_transaction_context *t,
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen struct mail_transaction_commit_changes *changes_r)
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen{
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct mailbox *box = t->box;
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen int ret;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (ft->build_ctx != NULL) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen fts_storage_build_context_deinit(ft->build_ctx);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen ft->build_ctx = NULL;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (ft->free_mail)
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen mail_free(&ft->mail);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen ret = fbox->module_ctx.super.transaction_commit(t, changes_r);
eb0ede66120bb63c0212bad69e67efca1eb47324Timo Sirainen fts_transaction_finish(box, ft, ret == 0);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen return ret;
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen}
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainenstatic int fts_update(struct mailbox *box)
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen{
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct mailbox_transaction_context *t;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct mail_search_args *search_args;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct mail_search_arg *arg;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct mail_search_context *ctx;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct fts_search_context *fctx;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen int ret = 0;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen t = mailbox_transaction_begin(box, 0);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen search_args = mail_search_build_init();
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen mail_search_build_add_all(search_args);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen arg = mail_search_build_add(search_args, SEARCH_BODY_FAST);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen arg->value.str = "xyzzy";
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen mail_search_args_unref(&search_args);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen fctx = FTS_CONTEXT(ctx);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (fctx->build_initialized) {
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen while ((ret = fts_mailbox_search_build_more(ctx)) == 0) ;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen }
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen (void)mailbox_search_deinit(&ctx);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen (void)mailbox_transaction_commit(&t);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen return ret < 0 ? -1 : 0;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen}
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainenstatic int fts_sync_deinit(struct mailbox_sync_context *ctx,
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct mailbox_sync_status *status_r)
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen{
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct mailbox *box = ctx->box;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(box);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen bool precache;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen precache = (ctx->flags & MAILBOX_SYNC_FLAG_PRECACHE) != 0;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (fbox->module_ctx.super.sync_deinit(ctx, status_r) < 0)
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen return -1;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen ctx = NULL;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen return !precache ? 0 : fts_update(box);
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen}
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
e726bf74fcc8d24f4c9d0d83217b3db4314d9d1fTimo Sirainenstatic void fts_mailbox_init(struct mailbox *box, const char *env)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen struct mailbox_vfuncs *v = box->vlast;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen fbox = i_new(struct fts_mailbox, 1);
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen fbox->virtual = strcmp(box->storage->name, "virtual") == 0;
a1761856683b4bf745eb4e32cefabeb851efb301Timo Sirainen fbox->env = env;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen fbox->module_ctx.super = *v;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen box->vlast = &fbox->module_ctx.super;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->free = fts_mailbox_free;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->search_init = fts_mailbox_search_init;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->search_next_nonblock = fts_mailbox_search_next_nonblock;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->search_next_update_seq = fbox->virtual ?
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen fts_mailbox_search_next_update_seq_virtual :
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen fts_mailbox_search_next_update_seq;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->search_deinit = fts_mailbox_search_deinit;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->transaction_begin = fts_transaction_begin;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->transaction_rollback = fts_transaction_rollback;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->transaction_commit = fts_transaction_commit;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen v->sync_deinit = fts_sync_deinit;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT_SET(box, fts_storage_module, fbox);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen}
e726bf74fcc8d24f4c9d0d83217b3db4314d9d1fTimo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainenvoid fts_mailbox_allocated(struct mailbox *box)
e726bf74fcc8d24f4c9d0d83217b3db4314d9d1fTimo Sirainen{
e726bf74fcc8d24f4c9d0d83217b3db4314d9d1fTimo Sirainen const char *env;
e726bf74fcc8d24f4c9d0d83217b3db4314d9d1fTimo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen env = mail_user_plugin_getenv(box->storage->user, "fts");
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (env != NULL)
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen fts_mailbox_init(box, env);
e726bf74fcc8d24f4c9d0d83217b3db4314d9d1fTimo Sirainen}