fts-storage.c revision ceae1acc3e3022c6b5fe52a4a34890dffdbcb77f
2e37d45867d081db150ab78dad303b9077aea24fTimo Sirainen/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
08d6658a4e2ec8104cd1307f6baa75fdb07a24f8Mark Washenberger#include "lib.h"
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen#include "array.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-search.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-storage-private.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mailbox-list-private.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "../virtual/virtual-storage.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "fts-api-private.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "fts-build.h"
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen#include "fts-search-serialize.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "fts-plugin.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "fts-storage.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include <stdlib.h>
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen#define FTS_CONTEXT(obj) \
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen MODULE_CONTEXT(obj, fts_storage_module)
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen#define FTS_MAIL_CONTEXT(obj) \
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen MODULE_CONTEXT(obj, fts_mail_module)
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen#define FTS_LIST_CONTEXT(obj) \
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen MODULE_CONTEXT(obj, fts_mailbox_list_module)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstruct fts_mailbox_list {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen union mailbox_list_module_context module_ctx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct fts_backend *backend;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen};
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenstruct fts_mailbox {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen union mailbox_module_context module_ctx;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen struct fts_backend_update_context *sync_update_ctx;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen};
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenstruct fts_transaction_context {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen union mailbox_transaction_module_context module_ctx;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen struct fts_scores *scores;
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen};
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainenstruct fts_mail {
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen union mail_module_context module_ctx;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen char score[30];
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen};
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_storage_module,
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen &mail_storage_module_register);
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register);
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module,
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen &mailbox_list_module_register);
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainenstatic void fts_scores_unref(struct fts_scores **_scores)
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen{
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen struct fts_scores *scores = *_scores;
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen *_scores = NULL;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen if (--scores->refcount == 0) {
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen array_free(&scores->score_map);
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen i_free(scores);
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen }
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic bool fts_try_build_init(struct mail_search_context *ctx,
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen struct fts_search_context *fctx)
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen{
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen if (fts_backend_is_updating(fctx->backend)) {
0f39a57760d93cddbce3ca43096d78e0fe2f42fdTimo Sirainen /* this process is already building the indexes */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return FALSE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
0f39a57760d93cddbce3ca43096d78e0fe2f42fdTimo Sirainen fctx->build_initialized = TRUE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen switch (fts_build_init(fctx->backend, ctx->transaction->box, FALSE,
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen &fctx->build_ctx)) {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen case -1:
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen break;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen case 0:
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen /* the index was up to date */
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen fts_search_lookup(fctx);
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen break;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen case 1:
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen /* hide "searching" notifications while building index */
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen ctx->progress_hidden = TRUE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen break;
0f39a57760d93cddbce3ca43096d78e0fe2f42fdTimo Sirainen }
b323e76cf555fa6031f3dbbdedeac0df2fff3778Timo Sirainen return TRUE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainenstatic bool fts_want_build_args(const struct mail_search_arg *args)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* we want to update index only when searching from message body.
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen it's not worth the wait for searching only from headers, which
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen could be in cache file already */
a30b52af112bc98b74b8624e9a5d20cb754b2ab7Timo Sirainen for (; args != NULL; args = args->next) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen switch (args->type) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen case SEARCH_OR:
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen case SEARCH_SUB:
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen case SEARCH_INTHREAD:
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen if (fts_want_build_args(args->value.subargs))
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen return TRUE;
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen break;
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen case SEARCH_BODY:
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen case SEARCH_TEXT:
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen return TRUE;
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen default:
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen break;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return FALSE;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen}
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainenstatic struct mail_search_context *
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenfts_mailbox_search_init(struct mailbox_transaction_context *t,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_search_args *args,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const enum mail_sort_type *sort_program,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen enum mail_fetch_field wanted_fields,
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen struct mailbox_header_lookup_ctx *wanted_headers)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(t->box->list);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_search_context *ctx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct fts_search_context *fctx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx = fbox->module_ctx.super.search_init(t, args, sort_program,
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen wanted_fields, wanted_headers);
a7bee3930831a9261fa6180d02af29c484d862e9Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen if (!fts_backend_can_lookup(flist->backend, args->args))
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return ctx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen fctx = i_new(struct fts_search_context, 1);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->box = t->box;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->backend = flist->backend;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->t = t;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->args = args;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->result_pool = pool_alloconly_create("fts results", 1024*32);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen i_array_init(&fctx->levels, 8);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->scores = i_new(struct fts_scores, 1);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->scores->refcount = 1;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen i_array_init(&fctx->scores->score_map, 64);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fctx->virtual_mailbox =
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* transaction contains the last search's scores. they can be
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen queried later with mail_get_special() */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (ft->scores != NULL)
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen fts_scores_unref(&ft->scores);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen ft->scores = fctx->scores;
7af2a19a89c1c9da8848c570190d36570afd09e6Timo Sirainen ft->scores->refcount++;
7af2a19a89c1c9da8848c570190d36570afd09e6Timo Sirainen
7af2a19a89c1c9da8848c570190d36570afd09e6Timo Sirainen if (fts_want_build_args(args->args))
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen (void)fts_try_build_init(ctx, fctx);
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen else {
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen fctx->build_initialized = TRUE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fts_search_lookup(fctx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen return ctx;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen}
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainenstatic bool fts_mailbox_build_continue(struct mail_search_context *ctx)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen{
7af2a19a89c1c9da8848c570190d36570afd09e6Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen int ret;
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (fctx == NULL)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return TRUE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (!fctx->build_initialized) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* we're still waiting for this process (but another command)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen to finish building the indexes */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (!fts_try_build_init(ctx, fctx))
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return FALSE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (fctx->build_ctx != NULL) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* this command is still building the indexes */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen ret = fts_build_more(fctx->build_ctx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (ret == 0)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return FALSE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen ctx->progress_hidden = FALSE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (fts_build_deinit(&fctx->build_ctx) < 0)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen ret = -1;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (ret > 0)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fts_search_lookup(fctx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return TRUE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen}
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenstatic bool
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenfts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen struct mail **mail_r, bool *tryagain_r)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen{
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (!fts_mailbox_build_continue(ctx)) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen *tryagain_r = TRUE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return FALSE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return fbox->module_ctx.super.
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen search_next_nonblock(ctx, mail_r, tryagain_r);
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen}
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenstatic void
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenfts_search_apply_results_level(struct mail_search_context *ctx,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen struct mail_search_arg *args, unsigned int *idx)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen{
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const struct fts_search_level *level;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen level = array_idx(&fctx->levels, *idx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (array_is_created(&level->definite_seqs) &&
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen seq_range_exists(&level->definite_seqs, ctx->seq))
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fts_search_deserialize_add_matches(args, level->args_matches);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen else if (!array_is_created(&level->maybe_seqs) ||
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen !seq_range_exists(&level->maybe_seqs, ctx->seq))
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fts_search_deserialize_add_nonmatches(args, level->args_matches);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen for (; args != NULL; args = args->next) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen continue;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen *idx += 1;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen fts_search_apply_results_level(ctx, args->value.subargs, idx);
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen }
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen}
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainenstatic bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen{
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen unsigned int idx;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (fctx == NULL || !fctx->fts_lookup_success) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* fts lookup not done for this search */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return fbox->module_ctx.super.search_next_update_seq(ctx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* restore original [non]matches */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fts_search_deserialize(ctx->args->args, fctx->orig_matches);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (!fbox->module_ctx.super.search_next_update_seq(ctx))
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return FALSE;
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (ctx->seq >= fctx->first_unindexed_seq) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* we've not indexed this far */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return TRUE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* apply [non]matches based on the FTS lookup results */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen idx = 0;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen fts_search_apply_results_level(ctx, ctx->args->args, &idx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return TRUE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen}
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainenstatic int fts_mailbox_search_deinit(struct mail_search_context *ctx)
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen if (fctx != NULL) {
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen if (fctx->build_ctx != NULL) {
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen /* the search was cancelled */
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen (void)fts_build_deinit(&fctx->build_ctx);
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen }
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_free(&fctx->orig_matches);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_free(&fctx->levels);
f3d506e525a720f214020ca0f989a1966b30edaeTimo Sirainen pool_unref(&fctx->result_pool);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen fts_scores_unref(&fctx->scores);
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen i_free(fctx);
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen }
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen return fbox->module_ctx.super.search_deinit(ctx);
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen}
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainenstatic int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score)
f8a86fdfb0048f9c87bf223373b35416ceb5856bTimo Sirainen{
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen return *uid < score->uid ? -1 :
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen (*uid > score->uid ? 1 : 0);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainenstatic int fts_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const char **value_r)
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen{
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const struct fts_score_map *scores;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen if (field != MAIL_FETCH_SEARCH_RELEVANCY || ft->scores == NULL)
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen scores = NULL;
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen else {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen scores = array_bsearch(&ft->scores->score_map, &_mail->uid,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen fts_score_cmp);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (scores != NULL) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen i_assert(scores->uid == _mail->uid);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen i_snprintf(fmail->score, sizeof(fmail->score),
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen "%f", scores->score);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen *value_r = fmail->score;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen return fmail->module_ctx.super.get_special(_mail, field, value_r);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen}
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenvoid fts_mail_allocated(struct mail *_mail)
{
struct mail_private *mail = (struct mail_private *)_mail;
struct mail_vfuncs *v = mail->vlast;
struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
struct fts_mail *fmail;
if (fbox == NULL)
return;
fmail = p_new(mail->pool, struct fts_mail, 1);
fmail->module_ctx.super = *v;
mail->vlast = &fmail->module_ctx.super;
v->get_special = fts_mail_get_special;
MODULE_CONTEXT_SET(mail, fts_mail_module, fmail);
}
static struct mailbox_transaction_context *
fts_transaction_begin(struct mailbox *box,
enum mailbox_transaction_flags flags)
{
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct mailbox_transaction_context *t;
struct fts_transaction_context *ft;
ft = i_new(struct fts_transaction_context, 1);
t = fbox->module_ctx.super.transaction_begin(box, flags);
MODULE_CONTEXT_SET(t, fts_storage_module, ft);
return t;
}
static void fts_transaction_rollback(struct mailbox_transaction_context *t)
{
struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
if (ft->scores != NULL)
fts_scores_unref(&ft->scores);
fbox->module_ctx.super.transaction_rollback(t);
}
static int
fts_transaction_commit(struct mailbox_transaction_context *t,
struct mail_transaction_commit_changes *changes_r)
{
struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
if (ft->scores != NULL)
fts_scores_unref(&ft->scores);
return fbox->module_ctx.super.transaction_commit(t, changes_r);
}
static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
enum mailbox_sync_type sync_type)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
struct fts_mailbox *fbox = FTS_CONTEXT(box);
if (fbox->module_ctx.super.sync_notify != NULL)
fbox->module_ctx.super.sync_notify(box, uid, sync_type);
if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE) {
if (uid == 0 && fbox->sync_update_ctx != NULL) {
/* this sync is finished */
(void)fts_backend_update_deinit(&fbox->sync_update_ctx);
}
return;
}
if (fbox->sync_update_ctx == NULL) {
fbox->sync_update_ctx = fts_backend_update_init(flist->backend);
fts_backend_update_set_mailbox(fbox->sync_update_ctx, box);
}
fts_backend_update_expunge(fbox->sync_update_ctx, uid);
}
static int fts_update(struct mailbox *box)
{
struct fts_storage_build_context *build_ctx;
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
int ret = 0;
if ((ret = fts_build_init(flist->backend, box,
TRUE, &build_ctx)) <= 0) {
if (box->storage->set->mail_debug)
i_debug("%s: FTS index is up to date", box->vname);
return ret;
}
if (box->storage->set->mail_debug)
i_debug("%s: Updating FTS index", box->vname);
while ((ret = fts_build_more(build_ctx)) == 0) ;
if (fts_build_deinit(&build_ctx) < 0)
ret = -1;
return ret < 0 ? -1 : 0;
}
static int fts_sync_deinit(struct mailbox_sync_context *ctx,
struct mailbox_sync_status *status_r)
{
struct mailbox *box = ctx->box;
struct fts_mailbox *fbox = FTS_CONTEXT(box);
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
bool precache, force_resync;
precache = (ctx->flags & MAILBOX_SYNC_FLAG_PRECACHE) != 0;
force_resync = (ctx->flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0;
if (fbox->module_ctx.super.sync_deinit(ctx, status_r) < 0)
return -1;
ctx = NULL;
if (force_resync) {
if (fts_backend_optimize(flist->backend) < 0) {
mail_storage_set_critical(box->storage,
"FTS optimize for mailbox %s failed",
box->vname);
}
}
if (precache) {
if (fts_update(box) < 0) {
mail_storage_set_critical(box->storage,
"FTS index update for mailbox %s failed",
box->vname);
return -1;
}
}
return 0;
}
void fts_mailbox_allocated(struct mailbox *box)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
struct mailbox_vfuncs *v = box->vlast;
struct fts_mailbox *fbox;
if (flist == NULL)
return;
fbox = p_new(box->pool, struct fts_mailbox, 1);
fbox->module_ctx.super = *v;
box->vlast = &fbox->module_ctx.super;
v->search_init = fts_mailbox_search_init;
v->search_next_nonblock = fts_mailbox_search_next_nonblock;
v->search_next_update_seq = fts_mailbox_search_next_update_seq;
v->search_deinit = fts_mailbox_search_deinit;
v->transaction_begin = fts_transaction_begin;
v->transaction_rollback = fts_transaction_rollback;
v->transaction_commit = fts_transaction_commit;
v->sync_notify = fts_mailbox_sync_notify;
v->sync_deinit = fts_sync_deinit;
MODULE_CONTEXT_SET(box, fts_storage_module, fbox);
}
static void fts_mailbox_list_deinit(struct mailbox_list *list)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list);
fts_backend_deinit(&flist->backend);
flist->module_ctx.super.deinit(list);
}
void fts_mailbox_list_created(struct mailbox_list *list)
{
struct fts_backend *backend;
const char *name, *path, *error;
name = mail_user_plugin_getenv(list->ns->user, "fts");
if (name == NULL) {
if (list->mail_set->mail_debug)
i_debug("fts: No fts setting - plugin disabled");
return;
}
path = mailbox_list_get_path(list, NULL,
MAILBOX_LIST_PATH_TYPE_INDEX);
if (*path == '\0') {
if (list->mail_set->mail_debug) {
i_debug("fts: Indexes disabled for namespace '%s'",
list->ns->prefix);
}
return;
}
if (fts_backend_init(name, list->ns, &error, &backend) < 0) {
i_error("fts: Failed to initialize backend '%s': %s",
name, error);
} else {
struct fts_mailbox_list *flist;
struct mailbox_list_vfuncs *v = list->vlast;
flist = p_new(list->pool, struct fts_mailbox_list, 1);
flist->module_ctx.super = *v;
flist->backend = backend;
list->vlast = &flist->module_ctx.super;
v->deinit = fts_mailbox_list_deinit;
MODULE_CONTEXT_SET(list, fts_mailbox_list_module, flist);
}
}
struct fts_backend *fts_mailbox_backend(struct mailbox *box)
{
struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
return flist->backend;
}