fts-backend-lucene.c revision 809923ba26218971792182ae8894936f9ac99364
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster/* Copyright (c) 2006-2008 Dovecot authors, see the included COPYING file */
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#include "lib.h"
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#include "array.h"
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#include "mkdir-parents.h"
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#include "mail-storage-private.h"
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#include "lucene-wrapper.h"
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#include "fts-lucene-plugin.h"
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#define LUCENE_INDEX_DIR_NAME "lucene-indexes"
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#define LUCENE_LOCK_SUBDIR_NAME "locks"
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster#define LUCENE_CONTEXT(obj) \
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster MODULE_CONTEXT(obj, fts_lucene_storage_module)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstruct lucene_mail_storage {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster union mail_storage_module_context module_ctx;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_index *index;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct mailbox *selected_box;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster int refcount;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster};
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstruct lucene_fts_backend {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct fts_backend backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_mail_storage *lstorage;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct mailbox *box;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster uint32_t last_uid;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster};
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic MODULE_CONTEXT_DEFINE_INIT(fts_lucene_storage_module,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster &mail_storage_module_register);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic void fts_backend_select(struct lucene_fts_backend *backend)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenk if (backend->lstorage->selected_box != backend->box) {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lucene_index_select_mailbox(backend->lstorage->index,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster mailbox_get_name(backend->box));
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster backend->lstorage->selected_box = backend->box;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster }
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic struct fts_backend *fts_backend_lucene_init(struct mailbox *box)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_mail_storage *lstorage;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_fts_backend *backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster const char *path, *lock_path;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lstorage = LUCENE_CONTEXT(box->storage);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster if (lstorage == NULL) {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster path = mail_storage_get_mailbox_index_dir(box->storage,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster "INBOX");
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster if (path == NULL) {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster /* in-memory indexes */
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return NULL;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster }
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster path = t_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lock_path = t_strdup_printf("%s/"LUCENE_LOCK_SUBDIR_NAME, path);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster if (mkdir_parents(lock_path, 0700) < 0) {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster i_error("mkdir_parents(%s) failed: %m", lock_path);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return NULL;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster }
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lstorage = i_new(struct lucene_mail_storage, 1);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lstorage->index = lucene_index_init(path, lock_path);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster MODULE_CONTEXT_SET(box->storage, fts_lucene_storage_module,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lstorage);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster }
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lstorage->refcount++;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster backend = i_new(struct lucene_fts_backend, 1);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster backend->backend = fts_backend_lucene;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster backend->lstorage = lstorage;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster backend->box = box;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return &backend->backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic void fts_backend_lucene_deinit(struct fts_backend *_backend)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_fts_backend *backend =
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster (struct lucene_fts_backend *)_backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster if (--backend->lstorage->refcount == 0) {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster MODULE_CONTEXT_UNSET(backend->box->storage,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_lucene_storage_module);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lucene_index_deinit(backend->lstorage->index);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster i_free(backend->lstorage);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster }
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster i_free(backend);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic int
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterfts_backend_lucene_get_last_uid(struct fts_backend *_backend,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster uint32_t *last_uid_r)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_fts_backend *backend =
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster (struct lucene_fts_backend *)_backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_select(backend);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return lucene_index_get_last_uid(backend->lstorage->index, last_uid_r);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenk
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenkstatic int
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenkfts_backend_lucene_build_init(struct fts_backend *_backend,
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenk uint32_t *last_uid_r,
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenk struct fts_backend_build_context **ctx_r)
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenk{
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenk struct lucene_fts_backend *backend =
56ed5bbb263838f338eb8afc978091c01a4f2a2bjeff.schenk (struct lucene_fts_backend *)_backend;
56ed5bbb263838f338eb8afc978091c01a4f2a2bjeff.schenk struct fts_backend_build_context *ctx;
56ed5bbb263838f338eb8afc978091c01a4f2a2bjeff.schenk
4709b991352c6de69ba02928ed6cbf373ca62ed5jeff.schenk fts_backend_select(backend);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster if (lucene_index_build_init(backend->lstorage->index,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster &backend->last_uid) < 0)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return -1;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster ctx = i_new(struct fts_backend_build_context, 1);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster ctx->backend = _backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster *last_uid_r = backend->last_uid;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster *ctx_r = ctx;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return 0;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic int
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterfts_backend_lucene_build_more(struct fts_backend_build_context *ctx,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster uint32_t uid, const unsigned char *data,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster size_t size, bool headers)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_fts_backend *backend =
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster (struct lucene_fts_backend *)ctx->backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster if (ctx->failed)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return -1;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster i_assert(uid >= backend->last_uid);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster backend->last_uid = uid;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster i_assert(backend->lstorage->selected_box == backend->box);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return lucene_index_build_more(backend->lstorage->index,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster uid, data, size, headers);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic int
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterfts_backend_lucene_build_deinit(struct fts_backend_build_context *ctx)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_fts_backend *backend =
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster (struct lucene_fts_backend *)ctx->backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster int ret = ctx->failed ? -1 : 0;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster i_assert(backend->lstorage->selected_box == backend->box);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster lucene_index_build_deinit(backend->lstorage->index);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster i_free(ctx);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return ret;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic void
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterfts_backend_lucene_expunge(struct fts_backend *_backend, struct mail *mail)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_fts_backend *backend =
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster (struct lucene_fts_backend *)_backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_select(backend);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster (void)lucene_index_expunge(backend->lstorage->index, mail->uid);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic void
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterfts_backend_lucene_expunge_finish(struct fts_backend *_backend ATTR_UNUSED,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct mailbox *box ATTR_UNUSED,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster bool committed ATTR_UNUSED)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic int
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterfts_backend_lucene_lock(struct fts_backend *_backend ATTR_UNUSED)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return 1;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic void
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterfts_backend_lucene_unlock(struct fts_backend *_backend ATTR_UNUSED)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstatic int
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterfts_backend_lucene_lookup(struct fts_backend *_backend,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster const char *key, enum fts_lookup_flags flags,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster ARRAY_TYPE(seq_range) *definite_uids,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster ARRAY_TYPE(seq_range) *maybe_uids)
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster{
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster struct lucene_fts_backend *backend =
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster (struct lucene_fts_backend *)_backend;
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster array_clear(maybe_uids);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_select(backend);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster return lucene_index_lookup(backend->lstorage->index,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster flags, key, definite_uids);
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster}
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Fosterstruct fts_backend fts_backend_lucene = {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster MEMBER(name) "lucene",
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster MEMBER(flags) 0,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster {
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_init,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_deinit,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_get_last_uid,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_build_init,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_build_more,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_build_deinit,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_expunge,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_expunge_finish,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_lock,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_unlock,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster fts_backend_lucene_lookup,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster NULL,
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster NULL
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster }
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster};
8af80418ba1ec431c8027fa9668e5678658d3611Allan Foster