plugins/fts-lucene/fts-backend-lucene.c

	fts-backend-lucene.c revision de62ce819d59a529530da4b57be1b8d6dad13d6b
286N/A/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
286N/A
286N/A#include "lib.h"
286N/A#include "array.h"
286N/A#include "hash.h"
286N/A#include "hex-binary.h"
286N/A#include "strescape.h"
286N/A#include "mail-namespace.h"
286N/A#include "mail-storage-private.h"
286N/A#include "fts-expunge-log.h"
286N/A#include "lucene-wrapper.h"
286N/A#include "fts-lucene-plugin.h"
286N/A
286N/A#include <wchar.h>
286N/A
286N/A#define LUCENE_INDEX_DIR_NAME "lucene-indexes"
286N/A#define LUCENE_EXPUNGE_LOG_NAME "dovecot-expunges.log"
286N/A#define LUCENE_OPTIMIZE_BATCH_MSGS_COUNT 100
286N/A
286N/Astruct lucene_fts_backend {
286N/A    struct fts_backend backend;
286N/A    char *dir_path;
286N/A
286N/A    struct lucene_index *index;
286N/A    struct mailbox *selected_box;
286N/A    unsigned int selected_box_generation;
286N/A    guid_128_t selected_box_guid;
286N/A
286N/A    struct fts_expunge_log *expunge_log;
286N/A
286N/A    unsigned int dir_created:1;
286N/A    unsigned int updating:1;
286N/A};
286N/A
286N/Astruct lucene_fts_backend_update_context {
286N/A    struct fts_backend_update_context ctx;
286N/A
286N/A    struct mailbox *box;
286N/A    uint32_t last_uid;
286N/A    uint32_t last_indexed_uid;
286N/A    char *first_box_vname;
286N/A
286N/A    uint32_t uid;
286N/A    char *hdr_name;
286N/A
286N/A    unsigned int added_msgs;
286N/A    struct fts_expunge_log_append_ctx *expunge_ctx;
286N/A
286N/A    bool lucene_opened;
286N/A    bool last_indexed_uid_set;
286N/A};
286N/A
286N/Astatic int fts_backend_lucene_mkdir(struct lucene_fts_backend *backend)
286N/A{
286N/A    if (backend->dir_created)
286N/A        return 0;
286N/A
286N/A    backend->dir_created = TRUE;
286N/A    return mailbox_list_mkdir_root(backend->backend.ns->list,
286N/A                       backend->dir_path,
286N/A                       MAILBOX_LIST_PATH_TYPE_INDEX);
286N/A}
286N/A
286N/Astatic int
286N/Afts_lucene_get_mailbox_guid(struct mailbox *box, guid_128_t guid_r)
286N/A{
286N/A    struct mailbox_metadata metadata;
286N/A
286N/A    if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
286N/A                 &metadata) < 0) {
286N/A        i_error("lucene: Couldn't get mailbox %s GUID: %s",
286N/A            box->vname, mailbox_get_last_error(box, NULL));
286N/A        return -1;
286N/A    }
286N/A    memcpy(guid_r, metadata.guid, GUID_128_SIZE);
286N/A    return 0;
286N/A}
286N/A
286N/Astatic int
286N/Afts_backend_select(struct lucene_fts_backend *backend, struct mailbox *box)
286N/A{
286N/A    guid_128_t guid;
286N/A    unsigned char guid_hex[MAILBOX_GUID_HEX_LENGTH];
286N/A    wchar_t wguid_hex[MAILBOX_GUID_HEX_LENGTH];
286N/A    buffer_t buf;
286N/A    unsigned int i;
286N/A
286N/A    if (backend->selected_box == box &&
286N/A        backend->selected_box_generation == box->generation_sequence)
286N/A        return 0;
286N/A
286N/A    if (box != NULL) {
286N/A        if (fts_lucene_get_mailbox_guid(box, guid) < 0)
286N/A            return -1;
286N/A        buffer_create_data(&buf, guid_hex, MAILBOX_GUID_HEX_LENGTH);
286N/A        binary_to_hex_append(&buf, guid, GUID_128_SIZE);
286N/A        for (i = 0; i < N_ELEMENTS(wguid_hex); i++)
286N/A            wguid_hex[i] = guid_hex[i];
286N/A
286N/A        lucene_index_select_mailbox(backend->index, wguid_hex);
286N/A    } else {
286N/A        lucene_index_unselect_mailbox(backend->index);
286N/A        memset(&guid, 0, sizeof(guid));
286N/A    }
286N/A    backend->selected_box = box;
286N/A    memcpy(backend->selected_box_guid, guid,
286N/A           sizeof(backend->selected_box_guid));
286N/A    backend->selected_box_generation =
286N/A        box == NULL ? 0 : box->generation_sequence;
286N/A    return 0;
286N/A}
286N/A
286N/Astatic struct fts_backend *fts_backend_lucene_alloc(void)
286N/A{
286N/A    struct lucene_fts_backend *backend;
286N/A
286N/A    backend = i_new(struct lucene_fts_backend, 1);
286N/A    backend->backend = fts_backend_lucene;
286N/A    return &backend->backend;
286N/A}
286N/A
286N/Astatic int
286N/Afts_backend_lucene_init(struct fts_backend *_backend,
286N/A            const char **error_r ATTR_UNUSED)
286N/A{
286N/A    struct lucene_fts_backend *backend =
286N/A        (struct lucene_fts_backend *)_backend;
286N/A    struct fts_lucene_user *fuser =
286N/A        FTS_LUCENE_USER_CONTEXT(_backend->ns->user);
286N/A    const char *path;
286N/A
286N/A    path = mailbox_list_get_path(_backend->ns->list, NULL,
286N/A                     MAILBOX_LIST_PATH_TYPE_INDEX);
286N/A    i_assert(path != NULL); /* fts already checked this */
286N/A
286N/A    backend->dir_path = i_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL);
286N/A    if (fuser != NULL) {
286N/A        backend->index = lucene_index_init(backend->dir_path,
286N/A                           _backend->ns->list,
286N/A                           &fuser->set);
286N/A    } else {
286N/A        backend->index = lucene_index_init(backend->dir_path,
286N/A                           _backend->ns->list,
286N/A                           NULL);
286N/A    }
286N/A
286N/A    path = t_strconcat(backend->dir_path, "/"LUCENE_EXPUNGE_LOG_NAME, NULL);
286N/A    backend->expunge_log = fts_expunge_log_init(path);
286N/A    return 0;
286N/A}
286N/A
286N/Astatic void fts_backend_lucene_deinit(struct fts_backend *_backend)
286N/A{
286N/A    struct lucene_fts_backend *backend =
286N/A        (struct lucene_fts_backend *)_backend;
286N/A
286N/A    lucene_index_deinit(backend->index);
286N/A    fts_expunge_log_deinit(&backend->expunge_log);
286N/A    i_free(backend->dir_path);
286N/A    i_free(backend);
286N/A}
286N/A
286N/Astatic int
286N/Afts_backend_lucene_get_last_uid(struct fts_backend *_backend,
286N/A                struct mailbox *box, uint32_t *last_uid_r)
286N/A{
286N/A    struct lucene_fts_backend *backend =
286N/A        (struct lucene_fts_backend *)_backend;
286N/A
286N/A    if (fts_index_get_last_uid(box, last_uid_r))
        return 0;

    /* either nothing has been indexed, or the index was corrupted.
       do it the slow way. */
    if (fts_backend_select(backend, box) < 0)
        return -1;
    if (lucene_index_get_last_uid(backend->index, last_uid_r) < 0)
        return -1;

    (void)fts_index_set_last_uid(box, *last_uid_r);
    return 0;
}

static struct fts_backend_update_context *
fts_backend_lucene_update_init(struct fts_backend *_backend)
{
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_backend;
    struct lucene_fts_backend_update_context *ctx;

    i_assert(!backend->updating);

    ctx = i_new(struct lucene_fts_backend_update_context, 1);
    ctx->ctx.backend = _backend;
    backend->updating = TRUE;
    return &ctx->ctx;
}

static bool
fts_backend_lucene_need_optimize(struct lucene_fts_backend_update_context *ctx)
{
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)ctx->ctx.backend;
    unsigned int expunges;
    uint32_t numdocs;

    if (ctx->added_msgs >= LUCENE_OPTIMIZE_BATCH_MSGS_COUNT)
        return TRUE;
    if (lucene_index_get_doc_count(backend->index, &numdocs) < 0)
        return FALSE;

    if (fts_expunge_log_uid_count(backend->expunge_log, &expunges) < 0)
        return FALSE;
    return expunges > 0 &&
        numdocs / expunges <= 50; /* >2% of index has been expunged */
}

static int
fts_backend_lucene_update_deinit(struct fts_backend_update_context *_ctx)
{
    struct lucene_fts_backend_update_context *ctx =
        (struct lucene_fts_backend_update_context *)_ctx;
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_ctx->backend;
    int ret = _ctx->failed ? -1 : 0;

    i_assert(backend->updating);

    backend->updating = FALSE;
    if (ctx->lucene_opened)
        lucene_index_build_deinit(backend->index);

    if (ctx->expunge_ctx != NULL) {
        if (fts_expunge_log_append_commit(&ctx->expunge_ctx) < 0)
            ret = -1;
    }

    if (fts_backend_lucene_need_optimize(ctx)) {
        if (ctx->ctx.backend->syncing)
            (void)fts_backend_optimize(_ctx->backend);
        else {
            struct mail_user *user = backend->backend.ns->user;
            const char *cmd, *path;
            int fd;

            /* the optimize affects all mailboxes within namespace,
               so just use any mailbox name in it */
            cmd = t_strdup_printf("OPTIMIZE\t0\t%s\t%s\n",
                str_tabescape(user->username),
                str_tabescape(ctx->first_box_vname));
            fd = fts_indexer_cmd(user, cmd, &path);
            if (fd != -1)
                (void)close(fd);
        }
    }

    i_free(ctx->first_box_vname);
    i_free(ctx);
    return ret;
}

static void
fts_backend_lucene_update_set_mailbox(struct fts_backend_update_context *_ctx,
                      struct mailbox *box)
{
    struct lucene_fts_backend_update_context *ctx =
        (struct lucene_fts_backend_update_context *)_ctx;

    if (ctx->last_uid != 0) {
        (void)fts_index_set_last_uid(ctx->box, ctx->last_uid);
        ctx->last_uid = 0;
    }
    if (ctx->first_box_vname == NULL)
        ctx->first_box_vname = i_strdup(box->vname);
    ctx->box = box;
    ctx->last_indexed_uid_set = FALSE;
}

static void
fts_backend_lucene_update_expunge(struct fts_backend_update_context *_ctx,
                  uint32_t uid)
{
    struct lucene_fts_backend_update_context *ctx =
        (struct lucene_fts_backend_update_context *)_ctx;
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_ctx->backend;

    if (!ctx->last_indexed_uid_set) {
        if (!fts_index_get_last_uid(ctx->box, &ctx->last_indexed_uid))
            ctx->last_indexed_uid = 0;
        ctx->last_indexed_uid_set = TRUE;
    }
    if (ctx->last_indexed_uid == 0 ||
        uid > ctx->last_indexed_uid + 100) {
        /* don't waste time adding expunge to log for a message that
           isn't even indexed. this check is racy, because indexer may
           just be in the middle of indexing this message. we'll
           attempt to avoid that by skipping the expunging only if
           indexing hasn't been done for a while (100 msgs). */
        return;
    }

    if (ctx->expunge_ctx == NULL) {
        ctx->expunge_ctx =
            fts_expunge_log_append_begin(backend->expunge_log);
    }

    if (fts_backend_select(backend, ctx->box) < 0)
        _ctx->failed = TRUE;

    fts_expunge_log_append_next(ctx->expunge_ctx,
                    backend->selected_box_guid, uid);
}

static bool
fts_backend_lucene_update_set_build_key(struct fts_backend_update_context *_ctx,
                    const struct fts_backend_build_key *key)
{
    struct lucene_fts_backend_update_context *ctx =
        (struct lucene_fts_backend_update_context *)_ctx;
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_ctx->backend;

    if (!ctx->lucene_opened) {
        if (fts_backend_lucene_mkdir(backend) < 0)
            ctx->ctx.failed = TRUE;
        if (lucene_index_build_init(backend->index) < 0)
            ctx->ctx.failed = TRUE;
        ctx->lucene_opened = TRUE;
    }

    if (fts_backend_select(backend, ctx->box) < 0)
        _ctx->failed = TRUE;

    switch (key->type) {
    case FTS_BACKEND_BUILD_KEY_HDR:
    case FTS_BACKEND_BUILD_KEY_MIME_HDR:
        i_assert(key->hdr_name != NULL);

        i_free(ctx->hdr_name);
        ctx->hdr_name = i_strdup(key->hdr_name);
        break;
    case FTS_BACKEND_BUILD_KEY_BODY_PART:
        i_free_and_null(ctx->hdr_name);
        break;
    case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
        i_unreached();
    }

    if (key->uid != ctx->last_uid) {
        i_assert(key->uid >= ctx->last_uid);
        ctx->last_uid = key->uid;
        ctx->added_msgs++;
    }

    ctx->uid = key->uid;
    return TRUE;
}

static void
fts_backend_lucene_update_unset_build_key(struct fts_backend_update_context *_ctx)
{
    struct lucene_fts_backend_update_context *ctx =
        (struct lucene_fts_backend_update_context *)_ctx;

    ctx->uid = 0;
    i_free_and_null(ctx->hdr_name);
}

static int
fts_backend_lucene_update_build_more(struct fts_backend_update_context *_ctx,
                     const unsigned char *data, size_t size)
{
    struct lucene_fts_backend_update_context *ctx =
        (struct lucene_fts_backend_update_context *)_ctx;
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_ctx->backend;
    int ret;

    i_assert(ctx->uid != 0);

    if (_ctx->failed)
        return -1;

    T_BEGIN {
        ret = lucene_index_build_more(backend->index, ctx->uid,
                          data, size, ctx->hdr_name);
    } T_END;
    return ret;
}

static int
fts_backend_lucene_refresh(struct fts_backend *_backend)
{
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_backend;

    lucene_index_close(backend->index);
    return 0;
}

static int fts_backend_lucene_rescan(struct fts_backend *_backend)
{
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_backend;

    if (lucene_index_rescan(backend->index) < 0)
        return -1;
    return lucene_index_optimize(backend->index);
}

static int fts_backend_lucene_optimize(struct fts_backend *_backend)
{
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_backend;
    int ret;

    ret = lucene_index_expunge_from_log(backend->index,
                        backend->expunge_log);
    if (ret == 0) {
        /* log was corrupted, need to rescan */
        ret = lucene_index_rescan(backend->index);
    }
    if (ret >= 0)
        ret = lucene_index_optimize(backend->index);
    return ret;
}

static int
fts_backend_lucene_lookup(struct fts_backend *_backend, struct mailbox *box,
              struct mail_search_arg *args, bool and_args,
              struct fts_result *result)
{
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_backend;
    int ret;

    if (fts_backend_select(backend, box) < 0)
        return -1;
    T_BEGIN {
        ret = lucene_index_lookup(backend->index, args, and_args,
                      result);
    } T_END;
    return ret;
}

/* a char* hash function from ASU -- from glib */
static unsigned int wstr_hash(const void *p)
{
        const wchar_t *s = p;
    unsigned int g, h = 0;

    while (*s != '\0') {
        h = (h << 4) + *s;
        if ((g = h & 0xf0000000UL)) {
            h = h ^ (g >> 24);
            h = h ^ g;
        }
        s++;
    }

    return h;
}

static int
mailboxes_get_guids(struct mailbox *const boxes[],
            struct hash_table *guids, struct fts_multi_result *result)
{
    ARRAY_DEFINE(box_results, struct fts_result);
    struct fts_result *box_result;
    const char *guid;
    wchar_t *guid_dup;
    unsigned int i, j;

    p_array_init(&box_results, result->pool, 32);
    for (i = 0; boxes[i] != NULL; i++) {
        if (fts_mailbox_get_guid(boxes[i], &guid) < 0)
            return -1;

        i_assert(strlen(guid) == MAILBOX_GUID_HEX_LENGTH);
        guid_dup = t_new(wchar_t, MAILBOX_GUID_HEX_LENGTH + 1);
        for (j = 0; j < MAILBOX_GUID_HEX_LENGTH; j++)
            guid_dup[j] = guid[j];

        box_result = array_append_space(&box_results);
        box_result->box = boxes[i];
        hash_table_insert(guids, guid_dup, box_result);
    }

    (void)array_append_space(&box_results);
    result->box_results = array_idx_modifiable(&box_results, 0);
    return 0;
}

static int
fts_backend_lucene_lookup_multi(struct fts_backend *_backend,
                struct mailbox *const boxes[],
                struct mail_search_arg *args, bool and_args,
                struct fts_multi_result *result)
{
    struct lucene_fts_backend *backend =
        (struct lucene_fts_backend *)_backend;
    int ret;

    T_BEGIN {
        struct hash_table *guids;

        guids = hash_table_create(default_pool, default_pool, 0,
                      wstr_hash,
                      (hash_cmp_callback_t *)wcscmp);
        ret = mailboxes_get_guids(boxes, guids, result);
        if (ret == 0) {
            ret = lucene_index_lookup_multi(backend->index,
                            guids, args, and_args,
                            result);
        }
        hash_table_destroy(&guids);
    } T_END;
    return ret;
}

struct fts_backend fts_backend_lucene = {
    .name = "lucene",
    .flags = 0,

    {
        fts_backend_lucene_alloc,
        fts_backend_lucene_init,
        fts_backend_lucene_deinit,
        fts_backend_lucene_get_last_uid,
        fts_backend_lucene_update_init,
        fts_backend_lucene_update_deinit,
        fts_backend_lucene_update_set_mailbox,
        fts_backend_lucene_update_expunge,
        fts_backend_lucene_update_set_build_key,
        fts_backend_lucene_update_unset_build_key,
        fts_backend_lucene_update_build_more,
        fts_backend_lucene_refresh,
        fts_backend_lucene_rescan,
        fts_backend_lucene_optimize,
        fts_backend_default_can_lookup,
        fts_backend_lucene_lookup,
        fts_backend_lucene_lookup_multi
    }
};