fts-storage.c revision 8d587838c414c48a331f0b54cd7ffd97e5024abd
2e37d45867d081db150ab78dad303b9077aea24fTimo Sirainen/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_backend_update_context *sync_update_ctx;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen union mailbox_transaction_module_context module_ctx;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_storage_module,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic void fts_scores_unref(struct fts_scores **_scores)
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainenstatic bool fts_try_build_init(struct mail_search_context *ctx,
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* this process is already building the indexes */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen switch (fts_build_init(fctx->backend, ctx->transaction->box, FALSE,
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen /* the index was up to date */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* hide "searching" notifications while building index */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic bool fts_want_build_args(const struct mail_search_arg *args)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* we want to update index only when searching from message body.
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen it's not worth the wait for searching only from headers, which
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen could be in cache file already */
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainenfts_mailbox_search_init(struct mailbox_transaction_context *t,
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen struct mailbox_header_lookup_ctx *wanted_headers)
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(t->box->list);
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen ctx = fbox->module_ctx.super.search_init(t, args, sort_program,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (!fts_backend_can_lookup(flist->backend, args->args))
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fctx->result_pool = pool_alloconly_create("fts results", 1024*32);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* transaction contains the last search's scores. they can be
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen queried later with mail_get_special() */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic bool fts_mailbox_build_continue(struct mail_search_context *ctx)
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* we're still waiting for this process (but another command)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen to finish building the indexes */
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen /* this command is still building the indexes */
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainenfts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen search_next_nonblock(ctx, mail_r, tryagain_r);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenfts_search_apply_results_level(struct mail_search_context *ctx,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct mail_search_arg *args, unsigned int *idx)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (array_is_created(&level->definite_seqs) &&
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen seq_range_exists(&level->definite_seqs, ctx->seq))
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_search_deserialize_add_matches(args, level->args_matches);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen else if (!array_is_created(&level->maybe_seqs) ||
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen !seq_range_exists(&level->maybe_seqs, ctx->seq))
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_search_deserialize_add_nonmatches(args, level->args_matches);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_search_apply_results_level(ctx, args->value.subargs, idx);
bd1b2615928a1e8be190cb0405754f0aec8cac2fTimo Sirainenstatic bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen unsigned int idx;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (fctx == NULL || !fctx->fts_lookup_success) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* fts lookup not done for this search */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return fbox->module_ctx.super.search_next_update_seq(ctx);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* restore original [non]matches */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_search_deserialize(ctx->args->args, fctx->orig_matches);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (!fbox->module_ctx.super.search_next_update_seq(ctx))
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* we've not indexed this far */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* apply [non]matches based on the FTS lookup results */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_search_apply_results_level(ctx, ctx->args->args, &idx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenstatic int fts_mailbox_search_deinit(struct mail_search_context *ctx)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen struct fts_search_context *fctx = FTS_CONTEXT(ctx);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* the search was cancelled */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return fbox->module_ctx.super.search_deinit(ctx);
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainenstatic int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score)
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainenstatic int fts_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen const char **value_r)
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (field != MAIL_FETCH_SEARCH_SCORE || ft->scores == NULL)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen scores = array_bsearch(&ft->scores->score_map, &_mail->uid,
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen i_snprintf(fmail->score, sizeof(fmail->score),
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen return fmail->module_ctx.super.get_special(_mail, field, value_r);
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen fmail = p_new(mail->pool, struct fts_mail, 1);
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen MODULE_CONTEXT_SET(mail, fts_mail_module, fmail);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen ft = i_new(struct fts_transaction_context, 1);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen t = fbox->module_ctx.super.transaction_begin(box, flags);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT_SET(t, fts_storage_module, ft);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainenstatic void fts_transaction_rollback(struct mailbox_transaction_context *t)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen fbox->module_ctx.super.transaction_rollback(t);
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainenfts_transaction_commit(struct mailbox_transaction_context *t,
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen struct mail_transaction_commit_changes *changes_r)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen struct fts_transaction_context *ft = FTS_CONTEXT(t);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return fbox->module_ctx.super.transaction_commit(t, changes_r);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (fbox->module_ctx.super.sync_notify != NULL)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fbox->module_ctx.super.sync_notify(box, uid, sync_type);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (uid == 0 && fbox->sync_update_ctx != NULL) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* this sync is finished */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen (void)fts_backend_update_deinit(&fbox->sync_update_ctx);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fbox->sync_update_ctx = fts_backend_update_init(flist->backend);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_backend_update_set_mailbox(fbox->sync_update_ctx, box);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fts_backend_update_expunge(fbox->sync_update_ctx, uid);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if ((ret = fts_build_init(flist->backend, box,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i_debug("%s: FTS index is up to date", box->vname);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i_debug("%s: Updating FTS index", box->vname);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen while ((ret = fts_build_more(build_ctx)) == 0) ;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainenstatic int fts_sync_deinit(struct mailbox_sync_context *ctx,
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen precache = (ctx->flags & MAILBOX_SYNC_FLAG_PRECACHE) != 0;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (fbox->module_ctx.super.sync_deinit(ctx, status_r) < 0)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen "FTS index update for mailbox %s failed",
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenvoid fts_mailbox_allocated(struct mailbox *box)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen fbox = p_new(box->pool, struct fts_mailbox, 1);
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->search_next_nonblock = fts_mailbox_search_next_nonblock;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen v->search_next_update_seq = fts_mailbox_search_next_update_seq;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->transaction_rollback = fts_transaction_rollback;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen v->transaction_commit = fts_transaction_commit;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT_SET(box, fts_storage_module, fbox);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic void fts_mailbox_list_deinit(struct mailbox_list *list)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenvoid fts_mailbox_list_created(struct mailbox_list *list)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen name = mail_user_plugin_getenv(list->ns->user, "fts");
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i_debug("fts: No fts setting - plugin disabled");
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i_debug("fts: Indexes disabled for namespace '%s'",
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (fts_backend_init(name, list->ns, &error, &backend) < 0) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i_error("fts: Failed to initialize backend '%s': %s",
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen flist = p_new(list->pool, struct fts_mailbox_list, 1);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen MODULE_CONTEXT_SET(list, fts_mailbox_list_module, flist);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstruct fts_backend *fts_mailbox_backend(struct mailbox *box)