lib-storage/index/index-storage.c

	index-storage.c revision 09af804b422b9d9b29b6f0532b6308be0c20a8d8
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen/* Copyright (C) 2002-2003 Timo Sirainen */
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "lib.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "buffer.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "ioloop.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "mail-index.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "index-storage.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include <stdlib.h>
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen#include <time.h>
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen#include <unistd.h>
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen#include <sys/stat.h>
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen/* How many seconds to keep index opened for reuse after it's been closed */
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#define INDEX_CACHE_TIMEOUT 10
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen/* How many closed indexes to keep */
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#define INDEX_CACHE_MAX 3
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen#define LOCK_NOTIFY_INTERVAL 30
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenstruct index_list {
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    struct index_list *next;
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    struct mail_index *index;
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen    char *mailbox_path;
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen    int refcount;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    dev_t index_dir_dev;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    ino_t index_dir_ino;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen    time_t destroy_time;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen};
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenstatic struct index_list *indexes = NULL;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenstatic struct timeout *to_index = NULL;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenstatic int index_storage_refcount = 0;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenvoid index_storage_init(struct index_storage *storage __attr_unused__)
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen{
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    index_storage_refcount++;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen}
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenvoid index_storage_deinit(struct index_storage *storage)
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen{
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    i_free(storage->storage.namespace);
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    i_free(storage->storage.error);
19e8858fad5128326481e6cd85c9070b1345bad8Timo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    if (--index_storage_refcount > 0)
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen        return;
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen        index_storage_destroy_unrefed();
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen}
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainenstatic void index_storage_add(struct mail_index *index,
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen                  const char *mailbox_path, struct stat *st)
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen{
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    struct index_list *list;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    list = i_new(struct index_list, 1);
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    list->refcount = 1;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    list->index = index;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    list->mailbox_path = i_strdup(mailbox_path);
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    list->index_dir_dev = st->st_dev;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    list->index_dir_ino = st->st_ino;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    list->next = indexes;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    indexes = list;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen}
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainenstatic void index_list_free(struct index_list *list)
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen{
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    mail_index_free(list->index);
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    i_free(list->mailbox_path);
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen    i_free(list);
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen}
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainenstruct mail_index *
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainenindex_storage_alloc(const char *index_dir, const char *mailbox_path,
f8ead0942a9b7c8fcf91414ed1b534d5807ca555Timo Sirainen            const char *prefix)
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen{
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen    struct index_list **list, *rec;
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen    struct mail_index *index;
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen    struct stat st;
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen    int destroy_count;
882897aa8345370a82487bd5c706cf26767b9f83Timo Sirainen
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen    if (index_dir == NULL || stat(index_dir, &st) < 0)
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen        memset(&st, 0, sizeof(st));
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen    /* compare index_dir inodes so we don't break even with symlinks.
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen       for in-memory indexes compare just mailbox paths */
882897aa8345370a82487bd5c706cf26767b9f83Timo Sirainen    destroy_count = 0; index = NULL;
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen    for (list = &indexes; *list != NULL;) {
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen        rec = *list;
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen        if ((index_dir != NULL && st.st_ino == rec->index_dir_ino &&
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen             CMP_DEV_T(st.st_dev, rec->index_dir_dev)) ||
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen            (index_dir == NULL && st.st_ino == 0 &&
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen             strcmp(mailbox_path, rec->mailbox_path) == 0)) {
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen            rec->refcount++;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen            index = rec->index;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen        }
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen        if (rec->refcount == 0) {
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen            if (rec->destroy_time <= ioloop_time ||
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen                destroy_count >= INDEX_CACHE_MAX) {
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen                *list = rec->next;
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen                index_list_free(rec);
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen                continue;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen            } else {
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen                destroy_count++;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen            }
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen        }
51dbc2d559815da774f8ee5faf0e28df3c8d40c0Timo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen                list = &(*list)->next;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    }
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    if (index == NULL) {
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen        index = mail_index_alloc(index_dir, prefix);
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen        index_storage_add(index, mailbox_path, &st);
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    }
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen    return index;
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen}
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenstatic void destroy_unrefed(int all)
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen{
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen    struct index_list **list, *rec;
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen    for (list = &indexes; *list != NULL;) {
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen        rec = *list;

        if (rec->refcount == 0 &&
            (all || rec->destroy_time <= ioloop_time)) {
            *list = rec->next;
            index_list_free(rec);
        } else {
            list = &(*list)->next;
        }
    }

    if (indexes == NULL && to_index != NULL) {
        timeout_remove(to_index);
        to_index = NULL;
    }
}

static void index_removal_timeout(void *context __attr_unused__)
{
    destroy_unrefed(FALSE);
}

void index_storage_unref(struct mail_index *index)
{
    struct index_list *list;

    for (list = indexes; list != NULL; list = list->next) {
        if (list->index == index)
            break;
    }

    i_assert(list != NULL);
    i_assert(list->refcount > 0);

    list->refcount--;
    list->destroy_time = ioloop_time + INDEX_CACHE_TIMEOUT;
    if (to_index == NULL)
        to_index = timeout_add(1000, index_removal_timeout, NULL);
}

void index_storage_destroy_unrefed(void)
{
    destroy_unrefed(TRUE);
}

static void set_cache_fields(const char *fields,
                 enum mail_cache_decision_type dest[32],
                 enum mail_cache_decision_type dec)
{
    static enum mail_cache_field field_enums[] = {
        MAIL_CACHE_SENT_DATE,
        MAIL_CACHE_RECEIVED_DATE,
        MAIL_CACHE_VIRTUAL_FULL_SIZE,
        MAIL_CACHE_BODY,
        MAIL_CACHE_BODYSTRUCTURE,
        MAIL_CACHE_MESSAGEPART
    };
    static const char *field_names[] = {
        "sent_date",
        "received_date",
        "virtual_size",
        "body",
        "bodystructure",
        "messagepart",
        NULL
    };

    const char *const *arr;
    int i;

    if (fields == NULL || *fields == '\0')
        return;

    for (arr = t_strsplit_spaces(fields, " ,"); *arr != NULL; arr++) {
        for (i = 0; field_names[i] != NULL; i++) {
            if (strcasecmp(field_names[i], *arr) == 0) {
                dest[field_enums[i]] = dec;
                break;
            }
        }
        if (field_names[i] == NULL) {
            i_error("Invalid cache field name '%s', ignoring ",
                *arr);
        }
    }
}

static const enum mail_cache_decision_type *get_default_cache_decisions(void)
{
    static enum mail_cache_decision_type dec[32];
    static int dec_set = FALSE;

    if (dec_set)
        return dec;

    memset(dec, 0, sizeof(dec));
    set_cache_fields(getenv("MAIL_CACHE_FIELDS"), dec,
             MAIL_CACHE_DECISION_TEMP);
    set_cache_fields(getenv("MAIL_NEVER_CACHE_FIELDS"), dec,
             MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED);
    return dec;
}

void index_storage_lock_notify(struct index_mailbox *ibox,
                   enum mailbox_lock_notify_type notify_type,
                   unsigned int secs_left)
{
    struct index_storage *storage = ibox->storage;
    const char *str;
    time_t now;

    if ((secs_left % 15) != 0) {
        /* update alarm() so that we get back here around the same
           time we want the next notify. also try to use somewhat
           rounded times. this affects only fcntl() locking, dotlock
           and flock() calls should be calling us constantly */
        alarm(secs_left%15);
    }

    /* if notify type changes, print the message immediately */
    now = time(NULL);
    if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE ||
        ibox->last_notify_type == notify_type) {
        if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE &&
            notify_type == MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE) {
            /* first override notification, show it */
        } else {
            if (now < ibox->next_lock_notify || secs_left < 15)
                return;
        }
    }

    ibox->next_lock_notify = now + LOCK_NOTIFY_INTERVAL;
        ibox->last_notify_type = notify_type;

    switch (notify_type) {
    case MAILBOX_LOCK_NOTIFY_NONE:
        break;
    case MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT:
        str = t_strdup_printf("Mailbox is locked, will abort in "
                      "%u seconds", secs_left);
        storage->callbacks->notify_no(&ibox->box, str,
                          storage->callback_context);
        break;
    case MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE:
        str = t_strdup_printf("Stale mailbox lock file detected, "
                      "will override in %u seconds", secs_left);
        storage->callbacks->notify_ok(&ibox->box, str,
                          storage->callback_context);
        break;
    }
}

void index_storage_lock_notify_reset(struct index_mailbox *ibox)
{
    ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL;
    ibox->last_notify_type = MAILBOX_LOCK_NOTIFY_NONE;
}

struct index_mailbox *
index_storage_mailbox_init(struct index_storage *storage, struct mailbox *box,
               struct mail_index *index, const char *name,
               enum mailbox_open_flags flags)
{
    struct index_mailbox *ibox;
    enum mail_index_open_flags index_flags;

    i_assert(name != NULL);

    index_flags = MAIL_INDEX_OPEN_FLAG_CREATE;
    if ((flags & MAILBOX_OPEN_FAST) != 0)
        index_flags |= MAIL_INDEX_OPEN_FLAG_FAST;
    if (getenv("MMAP_DISABLE") != NULL)
        index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE;
    if (getenv("MMAP_NO_WRITE") != NULL)
        index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE;
    if (getenv("FCNTL_LOCKS_DISABLE") != NULL)
        index_flags |= MAIL_INDEX_OPEN_FLAG_FCNTL_LOCKS_DISABLE;

    do {
        ibox = i_new(struct index_mailbox, 1);
        ibox->box = *box;
        ibox->storage = storage;

        ibox->box.storage = &storage->storage;
        ibox->box.name = i_strdup(name);
        ibox->readonly = (flags & MAILBOX_OPEN_READONLY) != 0;
        ibox->keep_recent = (flags & MAILBOX_OPEN_KEEP_RECENT) != 0;

        ibox->index = index;

        ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL;
        ibox->commit_log_file_seq = 0;
        ibox->mail_read_mmaped = getenv("MAIL_READ_MMAPED") != NULL;

        if (mail_index_open(index, index_flags) < 0)
            break;

        ibox->cache = mail_index_get_cache(index);
        mail_cache_set_defaults(ibox->cache,
                    get_default_cache_decisions());
        ibox->view = mail_index_view_open(index);
        return ibox;
    } while (0);

    mail_storage_set_index_error(ibox);
    index_storage_mailbox_free(&ibox->box);
    return NULL;
}

void index_storage_mailbox_free(struct mailbox *box)
{
    struct index_mailbox *ibox = (struct index_mailbox *) box;

    if (ibox->view != NULL)
        mail_index_view_close(ibox->view);

    index_mailbox_check_remove_all(ibox);
    if (ibox->index != NULL)
        index_storage_unref(ibox->index);
    i_free(ibox->path);
    i_free(ibox->control_dir);

    i_free(box->name);
    i_free(box);
}

int index_storage_is_readonly(struct mailbox *box)
{
    struct index_mailbox *ibox = (struct index_mailbox *) box;

    return ibox->readonly;
}

int index_storage_allow_new_keywords(struct mailbox *box)
{
    struct index_mailbox *ibox = (struct index_mailbox *) box;

    /* FIXME: return FALSE if we're full */
    return !ibox->readonly;
}

int index_storage_is_inconsistent(struct mailbox *box)
{
    struct index_mailbox *ibox = (struct index_mailbox *) box;

    return mail_index_view_is_inconsistent(ibox->view);
}

void index_storage_set_callbacks(struct mail_storage *_storage,
                 struct mail_storage_callbacks *callbacks,
                 void *context)
{
    struct index_storage *storage = (struct index_storage *) _storage;

    *storage->callbacks = *callbacks;
    storage->callback_context = context;
}

const char *index_storage_get_last_error(struct mail_storage *storage,
                     int *syntax_error_r)
{
    *syntax_error_r = storage->syntax_error;
    return storage->error;
}

int mail_storage_set_index_error(struct index_mailbox *ibox)
{
    switch (mail_index_get_last_error(ibox->index)) {
    case MAIL_INDEX_ERROR_NONE:
    case MAIL_INDEX_ERROR_INTERNAL:
        mail_storage_set_internal_error(ibox->box.storage);
        break;
    case MAIL_INDEX_ERROR_DISKSPACE:
        mail_storage_set_error(ibox->box.storage, "Out of disk space");
        break;
    }

    if (ibox->view != NULL)
        mail_index_view_unlock(ibox->view);
    mail_index_reset_error(ibox->index);
    return FALSE;
}

int index_mailbox_fix_keywords(struct index_mailbox *ibox,
                   enum mail_flags *flags,
                   const char *keywords[],
                   unsigned int keywords_count)
{
    /*FIXME:int ret;

    ret = mail_keywords_fix_list(ibox->index, flags, keywords,
                     keywords_count);
    switch (ret) {
    case 1:
        return TRUE;
    case 0:
        mail_storage_set_error(ibox->box.storage,
            "Maximum number of different keywords exceeded");
        return FALSE;
    default:
        return mail_storage_set_index_error(ibox);
    }*/
}