src/lib-index/mail-index.c

	mail-index.c revision 7cb128dc4cae2a03a742f63ba7afee23c78e3af0
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen/* Copyright (c) 2003-2015 Dovecot authors, see the included COPYING file */
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "lib.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "ioloop.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "array.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "buffer.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "eacces-error.h"
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen#include "hash.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "str-sanitize.h"
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen#include "mmap-util.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "nfs-workarounds.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "read-full.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "write-full.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "mail-index-alloc-cache.h"
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen#include "mail-index-private.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "mail-index-view-private.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "mail-index-sync-private.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "mail-index-modseq.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "mail-transaction-log.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include "mail-cache.h"
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include <stdio.h>
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen#include <stddef.h>
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen#include <time.h>
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen#include <sys/stat.h>
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainenstruct mail_index_module_register mail_index_module_register = { 0 };
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainenstatic void mail_index_close_nonopened(struct mail_index *index);
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen{
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    struct mail_index *index;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen    index = i_new(struct mail_index, 1);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->dir = i_strdup(dir);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->prefix = i_strdup(prefix);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->fd = -1;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen    index->extension_pool =
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen        pool_alloconly_create(MEMPOOL_GROWING"index extension", 1024);
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen    p_array_init(&index->extensions, index->extension_pool, 5);
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen    i_array_init(&index->sync_lost_handlers, 4);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_array_init(&index->module_contexts,
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen             I_MIN(5, mail_index_module_register.id));
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->mode = 0600;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->gid = (gid_t)-1;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->lock_method = FILE_LOCK_METHOD_FCNTL;
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen    index->max_lock_timeout_secs = UINT_MAX;
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->keywords_ext_id =
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen        mail_index_ext_register(index, MAIL_INDEX_EXT_KEYWORDS,
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen                    128, 2, 1);
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen    index->keywords_pool = pool_alloconly_create("keywords", 512);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_array_init(&index->keywords, 16);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    hash_table_create(&index->keywords_hash, index->keywords_pool, 0,
9844b5359f5cab77e4c31a7ac9e4a60a0073929eTimo Sirainen              strcase_hash, strcasecmp);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->log = mail_transaction_log_alloc(index);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    mail_index_modseq_init(index);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    return index;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen}
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainenvoid mail_index_free(struct mail_index **_index)
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen{
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    struct mail_index *index = *_index;
9844b5359f5cab77e4c31a7ac9e4a60a0073929eTimo Sirainen
419cf63077e755935ce105747d6ebc67b7d38a7fTimo Sirainen    *_index = NULL;
9844b5359f5cab77e4c31a7ac9e4a60a0073929eTimo Sirainen
9844b5359f5cab77e4c31a7ac9e4a60a0073929eTimo Sirainen    i_assert(index->open_count == 0);
9844b5359f5cab77e4c31a7ac9e4a60a0073929eTimo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    mail_transaction_log_free(&index->log);
9844b5359f5cab77e4c31a7ac9e4a60a0073929eTimo Sirainen    hash_table_destroy(&index->keywords_hash);
9844b5359f5cab77e4c31a7ac9e4a60a0073929eTimo Sirainen    pool_unref(&index->extension_pool);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    pool_unref(&index->keywords_pool);
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    array_free(&index->sync_lost_handlers);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    array_free(&index->keywords);
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    array_free(&index->module_contexts);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    i_free(index->ext_hdr_init_data);
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    i_free(index->gid_origin);
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    i_free(index->error);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_free(index->dir);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_free(index->prefix);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_free(index);
f8ead0942a9b7c8fcf91414ed1b534d5807ca555Timo Sirainen}
f8ead0942a9b7c8fcf91414ed1b534d5807ca555Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainenvoid mail_index_set_fsync_mode(struct mail_index *index,
f8ead0942a9b7c8fcf91414ed1b534d5807ca555Timo Sirainen                   enum fsync_mode mode,
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen                   enum mail_index_fsync_mask mask)
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen{
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->fsync_mode = mode;
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen    index->fsync_mask = mask;
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen}
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainenvoid mail_index_set_permissions(struct mail_index *index,
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen                mode_t mode, gid_t gid, const char *gid_origin)
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen{
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->mode = mode & 0666;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->gid = gid;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_free(index->gid_origin);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->gid_origin = i_strdup(gid_origin);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen}
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainenvoid mail_index_set_lock_method(struct mail_index *index,
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen                enum file_lock_method lock_method,
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen                unsigned int max_timeout_secs)
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen{
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->lock_method = lock_method;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->max_lock_timeout_secs = max_timeout_secs;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen}
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainenvoid mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id,
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen                  const void *data, size_t size)
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen{
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    const struct mail_index_registered_ext *rext;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_assert(index->ext_hdr_init_data == NULL ||
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen         index->ext_hdr_init_id == ext_id);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    rext = array_idx(&index->extensions, ext_id);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_assert(rext->hdr_size == size);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->ext_hdr_init_id = ext_id;
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    i_free(index->ext_hdr_init_data);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    index->ext_hdr_init_data = i_malloc(size);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    memcpy(index->ext_hdr_init_data, data, size);
2be7df5df08ac4639ad83559ec5fcf552c84fb4aTimo Sirainen}
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen                 uint32_t default_hdr_size,
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen                 uint16_t default_record_size,
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen                 uint16_t default_record_align)
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen{
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen    struct mail_index_registered_ext rext;
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen    uint32_t ext_id;
92d1458b00f4f236c4cec96a696253d3bbf8b05aTimo Sirainen
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen    if (*name == '\0' || strcmp(name, str_sanitize(name, -1)) != 0)
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen        i_panic("mail_index_ext_register(%s): Invalid name", name);
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
2be7df5df08ac4639ad83559ec5fcf552c84fb4aTimo Sirainen    if (default_record_size != 0 && default_record_align == 0) {
52de9d38fcad80df481667bac821cb8222785fe6Timo Sirainen        i_panic("mail_index_ext_register(%s): "
2be7df5df08ac4639ad83559ec5fcf552c84fb4aTimo Sirainen            "Invalid record alignment", name);
2be7df5df08ac4639ad83559ec5fcf552c84fb4aTimo Sirainen    }
28c75d59f1d1a7caeb85635964f3881c0038eb23Timo Sirainen
    if (mail_index_ext_lookup(index, name, &ext_id))
        return ext_id;

    memset(&rext, 0, sizeof(rext));
    rext.name = p_strdup(index->extension_pool, name);
    rext.index_idx = array_count(&index->extensions);
    rext.hdr_size = default_hdr_size;
    rext.record_size = default_record_size;
    rext.record_align = default_record_align;

    array_append(&index->extensions, &rext, 1);
    return rext.index_idx;
}

bool mail_index_ext_lookup(struct mail_index *index, const char *name,
               uint32_t *ext_id_r)
{
        const struct mail_index_registered_ext *extensions;
    unsigned int i, count;

    extensions = array_get(&index->extensions, &count);
    for (i = 0; i < count; i++) {
        if (strcmp(extensions[i].name, name) == 0) {
            *ext_id_r = i;
            return TRUE;
        }
    }

    *ext_id_r = (uint32_t)-1;
    return FALSE;
}

void mail_index_register_expunge_handler(struct mail_index *index,
                     uint32_t ext_id, bool call_always,
                     mail_index_expunge_handler_t *cb,
                     void *context)
{
    struct mail_index_registered_ext *rext;

    rext = array_idx_modifiable(&index->extensions, ext_id);
    i_assert(rext->expunge_handler == NULL || rext->expunge_handler == cb);

    rext->expunge_handler = cb;
    rext->expunge_context = context;
    rext->expunge_handler_call_always = call_always;
}

void mail_index_unregister_expunge_handler(struct mail_index *index,
                       uint32_t ext_id)
{
    struct mail_index_registered_ext *rext;

    rext = array_idx_modifiable(&index->extensions, ext_id);
    i_assert(rext->expunge_handler != NULL);

    rext->expunge_handler = NULL;
}

void mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,
                      mail_index_sync_handler_t *cb,
                      enum mail_index_sync_handler_type type)
{
    struct mail_index_registered_ext *rext;

    rext = array_idx_modifiable(&index->extensions, ext_id);
    i_assert(rext->sync_handler.callback == NULL);

    rext->sync_handler.callback = cb;
    rext->sync_handler.type = type;
}

void mail_index_unregister_sync_handler(struct mail_index *index,
                    uint32_t ext_id)
{
    struct mail_index_registered_ext *rext;

    rext = array_idx_modifiable(&index->extensions, ext_id);
    i_assert(rext->sync_handler.callback != NULL);

    rext->sync_handler.callback = NULL;
    rext->sync_handler.type = 0;
}

void mail_index_register_sync_lost_handler(struct mail_index *index,
                       mail_index_sync_lost_handler_t *cb)
{
    array_append(&index->sync_lost_handlers, &cb, 1);
}

void mail_index_unregister_sync_lost_handler(struct mail_index *index,
                         mail_index_sync_lost_handler_t *cb)
{
    mail_index_sync_lost_handler_t *const *handlers;
    unsigned int i, count;

    handlers = array_get(&index->sync_lost_handlers, &count);
    for (i = 0; i < count; i++) {
        if (handlers[i] == cb) {
            array_delete(&index->sync_lost_handlers, i, 1);
            break;
        }
    }
}

bool mail_index_keyword_lookup(struct mail_index *index,
                   const char *keyword, unsigned int *idx_r)
{
    char *key;
    void *value;

    /* keywords_hash keeps a name => index mapping of keywords.
       Keywords are never removed from it, so the index values are valid
       for the lifetime of the mail_index. */
    if (hash_table_lookup_full(index->keywords_hash, keyword,
                   &key, &value)) {
        *idx_r = POINTER_CAST_TO(value, unsigned int);
        return TRUE;
    }

    *idx_r = UINT_MAX;
    return FALSE;
}

void mail_index_keyword_lookup_or_create(struct mail_index *index,
                     const char *keyword,
                     unsigned int *idx_r)
{
    char *keyword_dup;

    i_assert(*keyword != '\0');

    if (mail_index_keyword_lookup(index, keyword, idx_r))
        return;

    keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
    *idx_r = array_count(&index->keywords);

    hash_table_insert(index->keywords_hash, keyword_dup,
              POINTER_CAST(*idx_r));
    array_append(&index->keywords, &keyword, 1);

    /* keep the array NULL-terminated, but the NULL itself invisible */
    array_append_zero(&index->keywords);
    array_delete(&index->keywords, array_count(&index->keywords)-1, 1);
}

const ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index)
{
    return &index->keywords;
}

struct mail_keywords *
mail_index_keywords_create(struct mail_index *index,
               const char *const keywords[])
{
    struct mail_keywords *k;
    unsigned int src, dest, i, count;

    count = str_array_length(keywords);
    if (count == 0) {
        k = i_new(struct mail_keywords, 1);
        k->index = index;
        k->refcount = 1;
        return k;
    }

    /* @UNSAFE */
    k = i_malloc(sizeof(struct mail_keywords) +
             (sizeof(k->idx) * (count-1)));
    k->index = index;
    k->refcount = 1;

    /* look up the keywords from index. they're never removed from there
       so we can permanently store indexes to them. */
    for (src = dest = 0; src < count; src++) {
        mail_index_keyword_lookup_or_create(index, keywords[src],
                            &k->idx[dest]);
        /* ignore if this is a duplicate */
        for (i = 0; i < src; i++) {
            if (k->idx[i] == k->idx[dest])
                break;
        }
        if (i == src)
            dest++;
    }
    k->count = dest;
    return k;
}

struct mail_keywords *
mail_index_keywords_create_from_indexes(struct mail_index *index,
                    const ARRAY_TYPE(keyword_indexes)
                        *keyword_indexes)
{
    struct mail_keywords *k;
    const unsigned int *indexes;
    unsigned int src, dest, i, count;

    indexes = array_get(keyword_indexes, &count);
    if (count == 0) {
        k = i_new(struct mail_keywords, 1);
        k->index = index;
        k->refcount = 1;
        return k;
    }

    /* @UNSAFE */
    k = i_malloc(sizeof(struct mail_keywords) +
             (sizeof(k->idx) * (count-1)));
    k->index = index;
    k->refcount = 1;

    /* copy but skip duplicates */
    for (src = dest = 0; src < count; src++) {
        for (i = 0; i < src; i++) {
            if (k->idx[i] == indexes[src])
                break;
        }
        if (i == src)
            k->idx[dest++] = indexes[src];
    }
    k->count = dest;
    return k;
}

void mail_index_keywords_ref(struct mail_keywords *keywords)
{
    keywords->refcount++;
}

void mail_index_keywords_unref(struct mail_keywords **_keywords)
{
    struct mail_keywords *keywords = *_keywords;

    i_assert(keywords->refcount > 0);

    *_keywords = NULL;
    if (--keywords->refcount == 0)
        i_free(keywords);
}

int mail_index_try_open_only(struct mail_index *index)
{
    i_assert(index->fd == -1);
    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

        /* Note that our caller must close index->fd by itself. */
    if (index->readonly)
        errno = EACCES;
    else {
        index->fd = nfs_safe_open(index->filepath, O_RDWR);
        index->readonly = FALSE;
    }

    if (index->fd == -1 && errno == EACCES) {
        index->fd = open(index->filepath, O_RDONLY);
        index->readonly = TRUE;
    }

    if (index->fd == -1) {
        if (errno != ENOENT) {
            mail_index_set_syscall_error(index, "open()");
            return -1;
        }

        /* have to create it */
        return 0;
    }
    return 1;
}

static int
mail_index_try_open(struct mail_index *index)
{
    int ret;

        i_assert(index->fd == -1);

    if (MAIL_INDEX_IS_IN_MEMORY(index))
        return 0;

    ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD);
    if (ret == 0) {
        /* it's corrupted - recreate it */
        if (index->fd != -1) {
            if (close(index->fd) < 0)
                mail_index_set_syscall_error(index, "close()");
            index->fd = -1;
        }
    }
    return ret;
}

int mail_index_create_tmp_file(struct mail_index *index,
                   const char *path_prefix, const char **path_r)
{
        mode_t old_mask;
    const char *path;
    int fd;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

    path = *path_r = t_strconcat(path_prefix, ".tmp", NULL);
    old_mask = umask(0);
    fd = open(path, O_RDWR|O_CREAT|O_EXCL, index->mode);
    umask(old_mask);
    if (fd == -1 && errno == EEXIST) {
        /* stale temp file. unlink and recreate rather than overwriting,
           just to make sure locking problems won't cause corruption */
        if (unlink(path) < 0) {
            i_error("unlink(%s) failed: %m", path);
            return -1;
        }
        old_mask = umask(0);
        fd = open(path, O_RDWR|O_CREAT|O_EXCL, index->mode);
        umask(old_mask);
    }
    if (fd == -1) {
        mail_index_file_set_syscall_error(index, path, "creat()");
        return -1;
    }

    mail_index_fchown(index, fd, path);
    return fd;
}

static int mail_index_open_files(struct mail_index *index,
                 enum mail_index_open_flags flags)
{
    int ret;

    ret = mail_transaction_log_open(index->log);
    if (ret == 0) {
        if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
            return 0;

        /* if dovecot.index exists, read it first so that we can get
           the correct indexid and log sequence */
        (void)mail_index_try_open(index);

        if (index->indexid == 0) {
            /* Create a new indexid for us. If we're opening index
               into memory, index->map doesn't exist yet. */
            index->indexid = ioloop_time;
            index->initial_create = TRUE;
            if (index->map != NULL)
                index->map->hdr.indexid = index->indexid;
        }

        ret = mail_transaction_log_create(index->log, FALSE);
        if (index->map != NULL) {
            /* log creation could have changed it if someone else
               just created it. */
            index->map->hdr.indexid = index->indexid;
        }
        index->initial_create = FALSE;
    }
    if (ret >= 0) {
        ret = index->map != NULL ? 1 : mail_index_try_open(index);
        if (ret == 0) {
            /* corrupted */
            mail_transaction_log_close(index->log);
            ret = mail_transaction_log_create(index->log, TRUE);
            if (ret == 0) {
                if (index->map != NULL)
                    mail_index_unmap(&index->map);
                index->map = mail_index_map_alloc(index);
            }
        }
    }
    if (ret < 0) {
        /* open/create failed, fallback to in-memory indexes */
        if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
            return -1;

        if (mail_index_move_to_memory(index) < 0)
            return -1;
    }

    if (index->cache == NULL)
        index->cache = mail_cache_open_or_create(index);
    return 1;
}

static int
mail_index_open_opened(struct mail_index *index,
               enum mail_index_open_flags flags)
{
    int ret;

    i_assert(index->map != NULL);

    if ((index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
        /* index was marked corrupted. we'll probably need to
           recreate the files. */
        mail_index_unmap(&index->map);
        mail_index_close_file(index);
        mail_transaction_log_close(index->log);
        if ((ret = mail_index_open_files(index, flags)) <= 0)
            return ret;
    }

    index->open_count++;
    return 1;
}

int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags)
{
    int ret;

    if (index->open_count > 0) {
        if ((ret = mail_index_open_opened(index, flags)) <= 0) {
            /* doesn't exist and create flag not used */
        }
        return ret;
    }

    index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
        i_strdup("(in-memory index)") :
        i_strconcat(index->dir, "/", index->prefix, NULL);

    index->readonly = FALSE;
    index->nodiskspace = FALSE;
    index->index_lock_timeout = FALSE;
    index->log_sync_locked = FALSE;
    index->flags = flags;
    index->readonly = (flags & MAIL_INDEX_OPEN_FLAG_READONLY) != 0;

    if ((flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0 &&
        index->fsync_mode != FSYNC_MODE_ALWAYS)
        i_fatal("nfs flush requires mail_fsync=always");
    if ((flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0 &&
        (flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0)
        i_fatal("nfs flush requires mmap_disable=yes");

    /* NOTE: increase open_count only after mail_index_open_files().
       it's used elsewhere to check if we're doing an initial opening
       of the index files */
    if ((ret = mail_index_open_files(index, flags)) <= 0) {
        /* doesn't exist and create flag not used */
        mail_index_close_nonopened(index);
        return ret;
    }
    index->open_count++;

    i_assert(index->map != NULL);
    mail_index_alloc_cache_index_opened(index);
    return 1;
}

int mail_index_open_or_create(struct mail_index *index,
                  enum mail_index_open_flags flags)
{
    int ret;

    flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
    ret = mail_index_open(index, flags);
    i_assert(ret != 0);
    return ret < 0 ? -1 : 0;
}

void mail_index_close_file(struct mail_index *index)
{
    if (index->fd != -1) {
        if (close(index->fd) < 0)
            mail_index_set_syscall_error(index, "close()");
        index->fd = -1;
    }
}

static void mail_index_close_nonopened(struct mail_index *index)
{
    i_assert(!index->syncing);
    i_assert(index->views == NULL);

    if (index->map != NULL)
        mail_index_unmap(&index->map);

    mail_index_close_file(index);
    mail_transaction_log_close(index->log);
    if (index->cache != NULL)
        mail_cache_free(&index->cache);

    i_free_and_null(index->filepath);

    index->indexid = 0;
}

void mail_index_close(struct mail_index *index)
{
    i_assert(index->open_count > 0);

    mail_index_alloc_cache_index_closing(index);
    if (--index->open_count == 0)
        mail_index_close_nonopened(index);
}

int mail_index_unlink(struct mail_index *index)
{
    const char *path;
    int last_errno = 0;

    if (MAIL_INDEX_IS_IN_MEMORY(index) || index->readonly)
        return 0;

    /* main index */
    if (unlink(index->filepath) < 0 && errno != ENOENT)
        last_errno = errno;

    /* logs */
    path = t_strconcat(index->filepath, MAIL_TRANSACTION_LOG_SUFFIX, NULL);
    if (unlink(path) < 0 && errno != ENOENT)
        last_errno = errno;

    path = t_strconcat(index->filepath,
               MAIL_TRANSACTION_LOG_SUFFIX".2", NULL);
    if (unlink(path) < 0 && errno != ENOENT)
        last_errno = errno;

    /* cache */
    path = t_strconcat(index->filepath, MAIL_CACHE_FILE_SUFFIX, NULL);
    if (unlink(path) < 0 && errno != ENOENT)
        last_errno = errno;

    if (last_errno == 0)
        return 0;
    else {
        errno = last_errno;
        return -1;
    }
}

int mail_index_reopen_if_changed(struct mail_index *index)
{
    struct stat st1, st2;

    if (MAIL_INDEX_IS_IN_MEMORY(index))
        return 0;

    if (index->fd == -1)
        return mail_index_try_open_only(index);

    if ((index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0)
        nfs_flush_file_handle_cache(index->filepath);
    if (nfs_safe_stat(index->filepath, &st2) < 0) {
        if (errno == ENOENT)
            return 0;
        mail_index_set_syscall_error(index, "stat()");
        return -1;
    }

    if (fstat(index->fd, &st1) < 0) {
        if (!ESTALE_FSTAT(errno)) {
            mail_index_set_syscall_error(index, "fstat()");
            return -1;
        }
        /* deleted/recreated, reopen */
    } else if (st1.st_ino == st2.st_ino &&
           CMP_DEV_T(st1.st_dev, st2.st_dev)) {
        /* the same file */
        return 1;
    }

    /* new file, new locks. the old fd can keep its locks, they don't
       matter anymore as no-one's going to modify the file. */
    mail_index_close_file(index);

    return mail_index_try_open_only(index);
}

int mail_index_refresh(struct mail_index *index)
{
    int ret;

    ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD);
    return ret <= 0 ? -1 : 0;
}

struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
    return index->cache;
}

void mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
    va_list va;

    i_free(index->error);

    if (fmt == NULL)
        index->error = NULL;
    else {
        va_start(va, fmt);
        index->error = i_strdup_vprintf(fmt, va);
        va_end(va);

        i_error("%s", index->error);
    }
}

bool mail_index_is_in_memory(struct mail_index *index)
{
    return MAIL_INDEX_IS_IN_MEMORY(index);
}

int mail_index_move_to_memory(struct mail_index *index)
{
    struct mail_index_map *map;

    if (MAIL_INDEX_IS_IN_MEMORY(index))
        return index->map == NULL ? -1 : 0;

    if ((index->flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0)
        return -1;

    /* set the index as being into memory */
    i_free_and_null(index->dir);

    i_free(index->filepath);
    index->filepath = i_strdup("(in-memory index)");

    if (index->map == NULL) {
        /* index was never even opened. just mark it as being in
           memory and let the caller re-open the index. */
        i_assert(index->fd == -1);
        return -1;
    }

    /* move index map to memory */
    if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
        map = mail_index_map_clone(index->map);
        mail_index_unmap(&index->map);
        index->map = map;
    }

    if (index->log != NULL) {
        /* move transaction log to memory */
        mail_transaction_log_move_to_memory(index->log);
    }

    if (index->fd != -1) {
        if (close(index->fd) < 0)
            mail_index_set_syscall_error(index, "close()");
        index->fd = -1;
    }
    return 0;
}

void mail_index_mark_corrupted(struct mail_index *index)
{
    index->indexid = 0;

    index->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
    if (!index->readonly) {
        if (unlink(index->filepath) < 0 &&
            errno != ENOENT && errno != ESTALE)
            mail_index_set_syscall_error(index, "unlink()");
        (void)mail_transaction_log_unlink(index->log);
    }
}

bool mail_index_is_deleted(struct mail_index *index)
{
    return index->index_delete_requested || index->index_deleted;
}

int mail_index_get_modification_time(struct mail_index *index, time_t *mtime_r)
{
    struct stat st;

    if (mail_transaction_log_get_mtime(index->log, mtime_r) < 0)
        return -1;

    if (*mtime_r == 0) {
        if (stat(index->filepath, &st) < 0) {
            mail_index_set_syscall_error(index, "stat()");
            return -1;
        }
        *mtime_r = st.st_mtime;
    }
    return 0;
}

void mail_index_fchown(struct mail_index *index, int fd, const char *path)
{
    mode_t mode;

    if (index->gid == (gid_t)-1) {
        /* no gid changing */
        return;
    } else if (fchown(fd, (uid_t)-1, index->gid) == 0) {
        /* success */
        return;
    } if ((index->mode & 0060) >> 3 == (index->mode & 0006)) {
        /* group and world permissions are the same, so group doesn't
           really matter. ignore silently. */
        return;
    }
    if (errno != EPERM)
        mail_index_file_set_syscall_error(index, path, "fchown()");
    else {
        mail_index_set_error(index, "%s",
            eperm_error_get_chgrp("fchown", path, index->gid,
                          index->gid_origin));
    }

    /* continue, but change permissions so that only the common
       subset of group and world is used. this makes sure no one
       gets any extra permissions. */
    mode = ((index->mode & 0060) >> 3) & (index->mode & 0006);
    mode |= (mode << 3) | (index->mode & 0600);
    if (fchmod(fd, mode) < 0)
        mail_index_file_set_syscall_error(index, path, "fchmod()");
}

void mail_index_set_syscall_error(struct mail_index *index,
                  const char *function)
{
    mail_index_file_set_syscall_error(index, index->filepath, function);
}

void mail_index_file_set_syscall_error(struct mail_index *index,
                       const char *filepath,
                       const char *function)
{
    const char *errstr;

    i_assert(filepath != NULL);
    i_assert(function != NULL);

    if (ENOSPACE(errno)) {
        index->nodiskspace = TRUE;
        if ((index->flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) == 0)
            return;
    }

    if (errno == EACCES) {
        function = t_strcut(function, '(');
        if (strcmp(function, "creat") == 0 ||
            strncmp(function, "file_dotlock_", 13) == 0)
            errstr = eacces_error_get_creating(function, filepath);
        else
            errstr = eacces_error_get(function, filepath);
        mail_index_set_error(index, "%s", errstr);
    } else {
        const char *suffix = errno != EFBIG ? "" :
            " (process was started with ulimit -f limit)";
        mail_index_set_error(index, "%s failed with file %s: "
                     "%m%s", function, filepath, suffix);
    }
}

const char *mail_index_get_error_message(struct mail_index *index)
{
    return index->error;
}

void mail_index_reset_error(struct mail_index *index)
{
    if (index->error != NULL) {
        i_free(index->error);
        index->error = NULL;
    }

    index->nodiskspace = FALSE;
        index->index_lock_timeout = FALSE;
}