src/lib-index/mail-index.c

	mail-index.c revision 64541374b58e4c702b1926e87df421d180ffa006
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen#include "lib.h"
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen#include "array.h"
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen#include "buffer.h"
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen#include "hash.h"
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen#include "mmap-util.h"
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainen#include "read-full.h"
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainen#include "write-full.h"
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen#include "mail-index-private.h"
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen#include "mail-index-sync-private.h"
9446c7a5d400cba60d097c528bd08312552438e3Timo Sirainen#include "mail-transaction-log.h"
9446c7a5d400cba60d097c528bd08312552438e3Timo Sirainen#include "mail-cache.h"
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen#include <stdio.h>
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainen#include <stddef.h>
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainen#include <time.h>
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainen#include <sys/stat.h>
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainen
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainenstatic void mail_index_create_in_memory(struct mail_index *index,
7c7d2244502af05f77c74736dc1aec0123fdba80Timo Sirainen                    const struct mail_index_header *hdr);
b494ffed8ded8d170d9ace3dc607b1d278048241Timo Sirainen
struct mail_index *mail_index_alloc(const char *dir, const char *prefix)
{
    struct mail_index *index;

    index = i_new(struct mail_index, 1);
    index->dir = i_strdup(dir);
    index->prefix = i_strdup(prefix);
    index->fd = -1;

    index->extension_pool = pool_alloconly_create("extension", 512);
    ARRAY_CREATE(&index->extensions, index->extension_pool,
             struct mail_index_ext, 5);

    ARRAY_CREATE(&index->expunge_handlers, default_pool,
             mail_index_expunge_handler_t *, 4);
    ARRAY_CREATE(&index->sync_handlers, default_pool,
             struct mail_index_sync_handler, 4);
    ARRAY_CREATE(&index->sync_lost_handlers, default_pool,
             mail_index_sync_lost_handler_t *, 4);

    index->mode = 0600;
    index->gid = (gid_t)-1;

    index->keywords_ext_id =
        mail_index_ext_register(index, "keywords", 128, 2, 1);
    index->keywords_pool = pool_alloconly_create("keywords", 512);
    ARRAY_CREATE(&index->keywords, default_pool, const char *, 16);
    index->keywords_hash =
        hash_create(default_pool, index->keywords_pool, 0,
                strcase_hash, (hash_cmp_callback_t *)strcasecmp);
    return index;
}

void mail_index_free(struct mail_index *index)
{
    mail_index_close(index);

    hash_destroy(index->keywords_hash);
    pool_unref(index->extension_pool);
    pool_unref(index->keywords_pool);

    array_free(&index->sync_handlers);
    array_free(&index->sync_lost_handlers);
    array_free(&index->expunge_handlers);
    array_free(&index->keywords);

    i_free(index->error);
    i_free(index->dir);
    i_free(index->prefix);
    i_free(index);
}

void mail_index_set_permissions(struct mail_index *index,
                mode_t mode, gid_t gid)
{
    index->mode = mode & 0666;
    index->gid = gid;
}

uint32_t mail_index_ext_register(struct mail_index *index, const char *name,
                 uint32_t default_hdr_size,
                 uint16_t default_record_size,
                 uint16_t default_record_align)
{
        const struct mail_index_ext *extensions;
    struct mail_index_ext ext;
    unsigned int i, ext_count;

    extensions = array_get(&index->extensions, &ext_count);

    i_assert(array_count(&index->sync_handlers) == ext_count);

    /* see if it's already there */
    for (i = 0; i < ext_count; i++) {
        if (strcmp(extensions[i].name, name) == 0)
            return i;
    }

    memset(&ext, 0, sizeof(ext));
    ext.name = p_strdup(index->extension_pool, name);
    ext.index_idx = ext_count;
    ext.hdr_size = default_hdr_size;
    ext.record_size = default_record_size;
    ext.record_align = default_record_align;

    array_append(&index->extensions, &ext, 1);
    (void)array_append_space(&index->sync_handlers);
    return ext_count;
}

void mail_index_register_expunge_handler(struct mail_index *index,
                     uint32_t ext_id,
                     mail_index_expunge_handler_t *cb)
{
        mail_index_expunge_handler_t **p;

    p = array_idx_modifyable(&index->expunge_handlers, ext_id);
    i_assert(*p == NULL);

    *p = cb;
}

void mail_index_unregister_expunge_handler(struct mail_index *index,
                       uint32_t ext_id)
{
        mail_index_expunge_handler_t **p;

    p = array_idx_modifyable(&index->expunge_handlers, ext_id);
    i_assert(*p != NULL);

    *p = NULL;
}

void mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,
                      mail_index_sync_handler_t *cb,
                      enum mail_index_sync_handler_type type)
{
    struct mail_index_sync_handler *h;

    h = array_idx_modifyable(&index->sync_handlers, ext_id);
    i_assert(h->callback == NULL);

    h->callback = cb;
    h->type = type;
}

void mail_index_unregister_sync_handler(struct mail_index *index,
                    uint32_t ext_id)
{
    struct mail_index_sync_handler *h;

    h = array_idx_modifyable(&index->sync_handlers, ext_id);
    i_assert(h->callback != NULL);

    h->callback = NULL;
    h->type = 0;
}

void mail_index_register_sync_lost_handler(struct mail_index *index,
                       mail_index_sync_lost_handler_t *cb)
{
    array_append(&index->sync_lost_handlers, &cb, 1);
}

void mail_index_unregister_sync_lost_handler(struct mail_index *index,
                         mail_index_sync_lost_handler_t *cb)
{
    mail_index_sync_lost_handler_t *const *handlers;
    unsigned int i, count;

    handlers = array_get(&index->sync_lost_handlers, &count);
    for (i = 0; i < count; i++) {
        if (handlers[i] == cb) {
            array_delete(&index->sync_lost_handlers, i, 1);
            break;
        }
    }
}

static void mail_index_map_init_extbufs(struct mail_index_map *map,
                    unsigned int initial_count)
{
#define EXTENSION_NAME_APPROX_LEN 20
    size_t size;

    if (map->extension_pool == NULL) {
        size = (sizeof(array_t) + BUFFER_APPROX_SIZE) * 2 +
            initial_count * (EXTENSION_NAME_APPROX_LEN +
                     sizeof(struct mail_index_ext) +
                     sizeof(uint32_t));
        map->extension_pool =
            pool_alloconly_create("extensions",
                          nearest_power(size));
    } else {
        p_clear(map->extension_pool);
    }

    ARRAY_CREATE(&map->extensions, map->extension_pool,
             struct mail_index_ext, initial_count);
    ARRAY_CREATE(&map->ext_id_map, map->extension_pool,
             uint32_t, initial_count);
}

uint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
{
    const struct mail_index_ext *extensions;
    unsigned int i, size;

    if (!array_is_created(&map->extensions))
        return (uint32_t)-1;

    extensions = array_get(&map->extensions, &size);
    for (i = 0; i < size; i++) {
        if (strcmp(extensions[i].name, name) == 0)
            return i;
    }
    return (uint32_t)-1;
}

uint32_t
mail_index_map_register_ext(struct mail_index *index,
                struct mail_index_map *map, const char *name,
                uint32_t hdr_offset, uint32_t hdr_size,
                uint32_t record_offset, uint32_t record_size,
                uint32_t record_align, uint32_t reset_id)
{
    struct mail_index_ext *ext;
    uint32_t idx, empty_idx = (uint32_t)-1;

    if (!array_is_created(&map->extensions)) {
                mail_index_map_init_extbufs(map, 5);
        idx = 0;
    } else {
        idx = array_count(&map->extensions);
    }
    i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);

    ext = array_append_space(&map->extensions);
    memset(ext, 0, sizeof(*ext));

    ext->name = p_strdup(map->extension_pool, name);
    ext->hdr_offset = hdr_offset;
    ext->hdr_size = hdr_size;
    ext->record_offset = record_offset;
    ext->record_size = record_size;
    ext->record_align = record_align;
    ext->reset_id = reset_id;

    ext->index_idx = mail_index_ext_register(index, name, hdr_size,
                         record_size, record_align);

    while (array_count(&map->ext_id_map) < ext->index_idx)
        array_append(&map->ext_id_map, &empty_idx, 1);
    array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
    return idx;
}

static int size_check(size_t *size_left, size_t size)
{
    if (size > *size_left)
        return FALSE;
    *size_left -= size;
    return TRUE;
}

static size_t get_align(size_t name_len)
{
    size_t size = sizeof(struct mail_index_ext_header) + name_len;
    return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
}

static int mail_index_read_extensions(struct mail_index *index,
                      struct mail_index_map *map)
{
    const struct mail_index_ext_header *ext_hdr;
    unsigned int i, old_count;
    const char *name;
    uint32_t ext_id, offset, name_offset;
    size_t size_left;

    /* extension headers always start from 64bit offsets, so if base header
       doesn't happen to be 64bit aligned we'll skip some bytes */
    offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
    if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
        /* nothing to do, skip allocatations and all */
        return 1;
    }

    old_count = array_count(&index->extensions);
    mail_index_map_init_extbufs(map, old_count + 5);

    ext_id = (uint32_t)-1;
    for (i = 0; i < old_count; i++)
        array_append(&map->ext_id_map, &ext_id, 1);

    while (offset < map->hdr.header_size) {
        ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);

        size_left = map->hdr.header_size - offset;
        if (!size_check(&size_left, sizeof(*ext_hdr)) ||
            !size_check(&size_left, ext_hdr->name_size) ||
            !size_check(&size_left, get_align(ext_hdr->name_size)) ||
            !size_check(&size_left, ext_hdr->hdr_size)) {
            mail_index_set_error(index, "Corrupted index file %s: "
                "Header extension goes outside header",
                index->filepath);
            return -1;
        }

        offset += sizeof(*ext_hdr);
        name_offset = offset;
        offset += ext_hdr->name_size + get_align(ext_hdr->name_size);

        t_push();
        name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
                 ext_hdr->name_size);

        if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {
            mail_index_set_error(index, "Corrupted index file %s: "
                "Duplicate header extension %s",
                index->filepath, name);
            t_pop();
            return -1;
        }

        if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
            (map->hdr.record_size % ext_hdr->record_align) != 0) {
            mail_index_set_error(index, "Corrupted index file %s: "
                "Record field %s alignmentation %u not used",
                index->filepath, name, ext_hdr->record_align);
            t_pop();
            return -1;
        }

        mail_index_map_register_ext(index, map, name,
                        offset, ext_hdr->hdr_size,
                        ext_hdr->record_offset,
                        ext_hdr->record_size,
                        ext_hdr->record_align,
                        ext_hdr->reset_id);
        t_pop();

        offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
    }
    return 1;
}

int mail_index_keyword_lookup(struct mail_index *index,
                  const char *keyword, int autocreate,
                  unsigned int *idx_r)
{
    char *keyword_dup;
    void *value;

    if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) {
        *idx_r = POINTER_CAST_TO(value, unsigned int);
        return TRUE;
    }

    if (!autocreate) {
        *idx_r = (unsigned int)-1;
        return FALSE;
    }

    keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
    *idx_r = array_count(&index->keywords);

    hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r));
    array_append(&index->keywords, &keyword, 1);
    return TRUE;
}

int mail_index_map_read_keywords(struct mail_index *index,
                 struct mail_index_map *map)
{
    const struct mail_index_ext *ext;
    const struct mail_index_keyword_header *kw_hdr;
    const struct mail_index_keyword_header_rec *kw_rec;
    const char *name;
    unsigned int i, name_len, old_count;
    uint32_t ext_id;

    ext_id = mail_index_map_lookup_ext(map, "keywords");
    if (ext_id == (uint32_t)-1) {
        if (array_is_created(&map->keyword_idx_map))
            array_clear(&map->keyword_idx_map);
        return 0;
    }

    ext = array_idx(&map->extensions, ext_id);

    kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
    kw_rec = (const void *)(kw_hdr + 1);
    name = (const char *)(kw_rec + kw_hdr->keywords_count);

    old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
        array_count(&map->keyword_idx_map);

    /* Keywords can only be added in mapping. */
    if (kw_hdr->keywords_count == old_count) {
        /* nothing changed */
        return 0;
    }

    /* make sure the header is valid */
    if (kw_hdr->keywords_count < old_count) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "Keywords removed unexpectedly",
                     index->filepath);
        return -1;
    }

    if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "keywords_count larger than header size",
                     index->filepath);
        return -1;
    }

    name_len = (const char *)kw_hdr + ext->hdr_size - name;
    for (i = 0; i < kw_hdr->keywords_count; i++) {
        if (kw_rec[i].name_offset > name_len) {
            mail_index_set_error(index, "Corrupted index file %s: "
                "name_offset points outside allocated header",
                index->filepath);
            return -1;
        }
    }
    if (name[name_len-1] != '\0') {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "header doesn't end with NUL",
                     index->filepath);
        return -1;
    }

    /* create file -> index mapping */
    if (!array_is_created(&map->keyword_idx_map)) {
        ARRAY_CREATE(&map->keyword_idx_map, default_pool,
                 unsigned int, kw_hdr->keywords_count);
    }

#ifdef DEBUG
    for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
        const char *keyword = name + kw_rec[i].name_offset;
        const unsigned int *old_idx;
        unsigned int idx;

        old_idx = array_idx(&map->keyword_idx_map, i);
        if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) ||
            idx != *old_idx) {
            mail_index_set_error(index, "Corrupted index file %s: "
                         "Keywords changed unexpectedly",
                         index->filepath);
            return -1;
        }
    }
#endif
    i = array_count(&map->keyword_idx_map);
    for (; i < kw_hdr->keywords_count; i++) {
        const char *keyword = name + kw_rec[i].name_offset;
        unsigned int idx;

        (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx);
        array_append(&map->keyword_idx_map, &idx, 1);
    }
    return 0;
}

const array_t *mail_index_get_keywords(struct mail_index *index)
{
    (void)mail_index_map_read_keywords(index, index->map);
    return &index->keywords;
}

static int mail_index_check_header(struct mail_index *index,
                   struct mail_index_map *map)
{
    const struct mail_index_header *hdr = &map->hdr;
    unsigned char compat_data[sizeof(hdr->compat_data)];

    memset(compat_data, 0, sizeof(compat_data));
#ifndef WORDS_BIGENDIAN
    compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
    compat_data[1] = sizeof(uoff_t);
    compat_data[2] = sizeof(time_t);

    if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
        /* major version change - handle silently(?) */
        return -1;
    }
    if (memcmp(hdr->compat_data, compat_data, sizeof(compat_data)) != 0) {
        /* architecture change - handle silently(?) */
        return -1;
    }

    if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
        /* we've already complained about it */
        return -1;
    }

    /* following some extra checks that only take a bit of CPU */
    if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "uid_validity = 0, next_uid = %u",
                     index->filepath, hdr->next_uid);
        return -1;
    }

    if (hdr->record_size < sizeof(struct mail_index_record)) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "record_size too small: %u < %"PRIuSIZE_T,
                     index->filepath, hdr->record_size,
                     sizeof(struct mail_index_record));
        return -1;
    }

    if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
        return 0;

    if (hdr->next_uid == 0)
        return 0;

    if (hdr->recent_messages_count > hdr->messages_count ||
        hdr->seen_messages_count > hdr->messages_count ||
        hdr->deleted_messages_count > hdr->messages_count)
        return 0;
    if (hdr->first_recent_uid_lowwater > hdr->next_uid ||
        hdr->first_unseen_uid_lowwater > hdr->next_uid ||
        hdr->first_deleted_uid_lowwater > hdr->next_uid)
        return 0;

    return mail_index_read_extensions(index, map);
}

static void mail_index_map_clear(struct mail_index *index,
                 struct mail_index_map *map)
{
    if (map->buffer != NULL) {
        i_assert(map->mmap_base == NULL);
        buffer_free(map->buffer);
        map->buffer = NULL;
    } else if (map->mmap_base != NULL) {
        i_assert(map->buffer == NULL);
        if (munmap(map->mmap_base, map->mmap_size) < 0)
            mail_index_set_syscall_error(index, "munmap()");
        map->mmap_base = NULL;
    }

    if (map->refcount > 0) {
        memset(&map->hdr, 0, sizeof(map->hdr));
        map->mmap_size = 0;
        map->mmap_used_size = 0;
        map->records = NULL;
        map->records_count = 0;
    }
}

void mail_index_unmap(struct mail_index *index, struct mail_index_map *map)
{
    if (--map->refcount > 0)
        return;

    i_assert(map->refcount == 0);
    mail_index_map_clear(index, map);
    if (map->extension_pool != NULL)
        pool_unref(map->extension_pool);
    if (array_is_created(&map->keyword_idx_map))
        array_free(&map->keyword_idx_map);
    buffer_free(map->hdr_copy_buf);
    i_free(map);
}

static void mail_index_unmap_forced(struct mail_index *index,
                    struct mail_index_map *map)
{
    mail_index_map_clear(index, map);
    mail_index_unmap(index, map);
}

static void mail_index_map_copy_hdr(struct mail_index_map *map,
                    const struct mail_index_header *hdr)
{
    if (hdr->base_header_size < sizeof(map->hdr)) {
        /* header smaller than ours, make a copy so our newer headers
           won't have garbage in them */
        memset(&map->hdr, 0, sizeof(map->hdr));
        memcpy(&map->hdr, hdr, hdr->base_header_size);
    } else {
        map->hdr = *hdr;
    }
}

static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
{
    const struct mail_index_header *hdr;
    unsigned int records_count;

    i_assert(!map->write_to_disk);

    if (map->buffer != NULL) {
        /* we had temporarily used a buffer, eg. for updating index */
        buffer_free(map->buffer);
        map->buffer = NULL;
    }

    map->mmap_base = index->readonly ?
        mmap_ro_file(index->fd, &map->mmap_size) :
        mmap_rw_file(index->fd, &map->mmap_size);
    if (map->mmap_base == MAP_FAILED) {
        map->mmap_base = NULL;
        mail_index_set_syscall_error(index, "mmap()");
        return -1;
    }

    hdr = map->mmap_base;
    if (map->mmap_size >
        offsetof(struct mail_index_header, major_version) &&
        hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
        /* major version change - handle silently */
        return 0;
    }

    if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "File too small (%"PRIuSIZE_T")",
                     index->filepath, map->mmap_size);
        return 0;
    }

    map->mmap_used_size = hdr->header_size +
        hdr->messages_count * hdr->record_size;

    if (map->mmap_used_size > map->mmap_size) {
        records_count = (map->mmap_size - hdr->header_size) /
            hdr->record_size;
        mail_index_set_error(index, "Corrupted index file %s: "
                     "messages_count too large (%u > %u)",
                     index->filepath, hdr->messages_count,
                     records_count);
        return 0;
    }

    mail_index_map_copy_hdr(map, hdr);

    map->hdr_base = map->mmap_base;
    map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
    map->records_count = map->hdr.messages_count;
    return 1;
}

static int mail_index_read_header(struct mail_index *index,
                  struct mail_index_header *hdr, size_t *pos_r)
{
    size_t pos;
    int ret;

    memset(hdr, 0, sizeof(*hdr));

    ret = 1;
    for (pos = 0; ret > 0 && pos < sizeof(*hdr); ) {
        ret = pread(index->fd, PTR_OFFSET(hdr, pos),
                sizeof(*hdr) - pos, pos);
        if (ret > 0)
            pos += ret;
    }
    *pos_r = pos;
    return ret;
}

static int mail_index_read_map(struct mail_index *index,
                   struct mail_index_map *map, int *retry_r)
{
    struct mail_index_header hdr;
    void *data = NULL;
    ssize_t ret;
    size_t pos, records_size;

    i_assert(map->mmap_base == NULL);

    *retry_r = FALSE;
    ret = mail_index_read_header(index, &hdr, &pos);

    if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
        hdr.major_version != MAIL_INDEX_MAJOR_VERSION) {
        /* major version change - handle silently */
        return 0;
    }

    if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
        (ret > 0 || pos >= hdr.base_header_size)) {
        if (hdr.base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
            hdr.header_size < hdr.base_header_size) {
            mail_index_set_error(index, "Corrupted index file %s: "
                "Corrupted header sizes (base %u, full %u)",
                index->filepath, hdr.base_header_size,
                hdr.header_size);
            return 0;
        }

        buffer_reset(map->hdr_copy_buf);
        buffer_append(map->hdr_copy_buf, &hdr, hdr.base_header_size);

        /* @UNSAFE */
        data = buffer_append_space_unsafe(map->hdr_copy_buf,
                          hdr.header_size -
                          hdr.base_header_size);
        ret = pread_full(index->fd, data,
                 hdr.header_size - hdr.base_header_size,
                 hdr.base_header_size);
    }

    if (ret > 0) {
        records_size = hdr.messages_count * hdr.record_size;

        if (map->buffer == NULL) {
            map->buffer = buffer_create_dynamic(default_pool,
                                records_size);
        }

        /* @UNSAFE */
        buffer_set_used_size(map->buffer, 0);
        data = buffer_append_space_unsafe(map->buffer, records_size);

        ret = pread_full(index->fd, data, records_size,
                 hdr.header_size);
    }

    if (ret < 0) {
        if (errno == ESTALE) {
            *retry_r = TRUE;
            return 0;
        }
        mail_index_set_syscall_error(index, "pread_full()");
        return -1;
    }
    if (ret == 0) {
        mail_index_set_error(index,
            "Corrupted index file %s: File too small",
            index->filepath);
        return 0;
    }

    map->records = data;
    map->records_count = hdr.messages_count;

    mail_index_map_copy_hdr(map, &hdr);
    map->hdr_base = map->hdr_copy_buf->data;

    index->sync_log_file_seq = hdr.log_file_seq;
    index->sync_log_file_offset = hdr.log_file_int_offset;
    return 1;
}

static int mail_index_sync_from_transactions(struct mail_index *index,
                         struct mail_index_map **map,
                         int sync_to_index)
{
    const struct mail_index_header *map_hdr = &(*map)->hdr;
    struct mail_index_view *view;
    struct mail_transaction_log_view *log_view;
    struct mail_index_sync_map_ctx sync_map_ctx;
    struct mail_index_header hdr;
    const struct mail_transaction_header *thdr;
    const void *tdata;
    uint32_t prev_seq, max_seq;
    uoff_t prev_offset, max_offset;
    size_t pos;
    int ret, skipped;

    if (sync_to_index) {
        /* read the real log position where we are supposed to be
           synced */
        ret = mail_index_read_header(index, &hdr, &pos);
        if (ret < 0 && errno != ESTALE) {
            mail_index_set_syscall_error(index, "pread()");
            return -1;
        }
        if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
            return 0;

        if (map_hdr->log_file_seq == hdr.log_file_seq &&
            map_hdr->log_file_int_offset == hdr.log_file_int_offset) {
            /* nothing to do */
            return 1;
        }

        if (map_hdr->log_file_seq > hdr.log_file_seq ||
            (map_hdr->log_file_seq == hdr.log_file_seq &&
             map_hdr->log_file_int_offset > hdr.log_file_int_offset)) {
            /* we went too far, have to re-read the file */
            return 0;
        }
        if (map_hdr->log_file_ext_offset !=
            map_hdr->log_file_int_offset ||
            hdr.log_file_ext_offset != hdr.log_file_int_offset) {
            /* too much trouble to get this right. */
            return 0;
        }
        max_seq = hdr.log_file_seq;
        max_offset = hdr.log_file_int_offset;
    } else {
        /* sync everything there is */
        max_seq = (uint32_t)-1;
        max_offset = (uoff_t)-1;
    }

    log_view = mail_transaction_log_view_open(index->log);
    if (mail_transaction_log_view_set(log_view,
                      map_hdr->log_file_seq,
                      map_hdr->log_file_int_offset,
                      max_seq, max_offset,
                      MAIL_TRANSACTION_TYPE_MASK) < 0) {
        mail_transaction_log_view_close(log_view);
        return 0;
    }

    index->map = *map;

    view = mail_index_view_open(index);
    mail_index_sync_map_init(&sync_map_ctx, view,
                 MAIL_INDEX_SYNC_HANDLER_VIEW);

    while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,
                             &skipped)) > 0) {
        if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {
            ret = -1;
            break;
        }
    }

    mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
                           &prev_offset);
    i_assert(prev_seq <= max_seq &&
         (prev_seq != max_seq || prev_offset <= max_offset));

    index->map->hdr.log_file_seq = prev_seq;
    index->map->hdr.log_file_int_offset =
        index->map->hdr.log_file_ext_offset = prev_offset;

    mail_index_sync_map_deinit(&sync_map_ctx);
    mail_index_view_close(view);
    mail_transaction_log_view_close(log_view);

    *map = index->map;
    index->map = NULL;

    if (sync_to_index) {
        /* make sure we did everything right. note that although the
           message counts should be equal, the flag counters may not */
        i_assert(hdr.messages_count == (*map)->hdr.messages_count);
        i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq);
        i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset);
        i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset);
    }

    return ret < 0 ? -1 : 1;
}

static int mail_index_read_map_with_retry(struct mail_index *index,
                      struct mail_index_map **map,
                      int sync_to_index)
{
    mail_index_sync_lost_handler_t *const *handlers;
    unsigned int i, count;
    int ret, retry;

    if (index->log_locked) {
        /* we're most likely syncing the index and we really don't
           want to read more than what was synced last time. */
        sync_to_index = TRUE;
    }

    if ((*map)->hdr.indexid != 0 && index->log != NULL) {
        /* we're not creating the index, or opening transaction log.
           sync this as a view from transaction log. */
        index->sync_update = TRUE;
        ret = mail_index_sync_from_transactions(index, map,
                            sync_to_index);
        index->sync_update = FALSE;
        if (ret != 0)
            return ret;

        /* transaction log lost/broken, fallback to re-reading it */
    }

    /* notify all "sync lost" handlers */
    handlers = array_get(&index->sync_lost_handlers, &count);
    for (i = 0; i < count; i++)
        (*handlers[i])(index);

    for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) {
        ret = mail_index_read_map(index, *map, &retry);
        if (ret != 0 || !retry)
            return ret;

        /* ESTALE - reopen index file */
        if (close(index->fd) < 0)
            mail_index_set_syscall_error(index, "close()");
        index->fd = -1;

        ret = mail_index_try_open_only(index);
        if (ret <= 0) {
            if (ret == 0) {
                /* the file was lost */
                errno = ENOENT;
                mail_index_set_syscall_error(index, "open()");
            }
            return -1;
        }
    }

    /* Too many ESTALE retries */
    mail_index_set_syscall_error(index, "read_map()");
    return -1;
}

static int mail_index_map_try_existing(struct mail_index_map *map)
{
    const struct mail_index_header *hdr;
    size_t used_size;

    if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
        return 0;

    hdr = map->mmap_base;

    /* always check corrupted-flag to avoid errors later */
    if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)
        return -1;

    used_size = hdr->header_size + hdr->messages_count * hdr->record_size;
    if (map->mmap_size >= used_size && map->hdr_base == hdr) {
        map->records_count = hdr->messages_count;
        mail_index_map_copy_hdr(map, hdr);
        return 1;
    }
    return 0;
}

int mail_index_map(struct mail_index *index, int force)
{
    struct mail_index_map *map;
    int ret;

    i_assert(!index->mapping);
    i_assert(index->map == NULL || index->map->refcount > 0);
    i_assert(index->lock_type != F_UNLCK);

    if (MAIL_INDEX_IS_IN_MEMORY(index)) {
        if (index->map == NULL)
            mail_index_create_in_memory(index, NULL);
        return 1;
    }

    index->mapping = TRUE;

    if (!force && index->map != NULL) {
        i_assert(index->hdr != NULL);
        ret = mail_index_map_try_existing(index->map);
        if (ret != 0) {
            index->mapping = FALSE;
            return ret;
        }

        if (index->lock_type == F_WRLCK) {
            /* we're syncing, don't break the mapping */
            index->mapping = FALSE;
            return 1;
        }
    }

    if (index->map != NULL && index->map->refcount > 1) {
        /* this map is already used by some views and they may have
           pointers into it. leave them and create a new mapping. */
        if (!index->mmap_disable) {
            map = NULL;
        } else {
            /* create a copy of the mapping instead so we don't
               have to re-read it */
            map = mail_index_map_clone(index->map,
                           index->map->hdr.record_size);
        }
        index->map->refcount--;
        index->map = NULL;
    } else {
        map = index->map;
    }

    if (map == NULL) {
        map = i_new(struct mail_index_map, 1);
        map->refcount = 1;
        map->hdr_copy_buf =
            buffer_create_dynamic(default_pool, sizeof(map->hdr));
    } else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
        i_assert(!map->write_to_disk);
    } else if (map->mmap_base != NULL) {
        i_assert(map->buffer == NULL);
        if (munmap(map->mmap_base, map->mmap_size) < 0)
            mail_index_set_syscall_error(index, "munmap()");
        map->mmap_base = NULL;
    }

    index->hdr = NULL;
    index->map = NULL;

    if (!index->mmap_disable)
        ret = mail_index_mmap(index, map);
    else
        ret = mail_index_read_map_with_retry(index, &map, force);
    i_assert(index->map == NULL);

    if (ret > 0) {
        ret = mail_index_check_header(index, map);
        if (ret < 0)
            ret = 0;
        else if (ret == 0) {
            index->fsck = TRUE;
            ret = 1;
        }
    }

    if (ret <= 0) {
        mail_index_unmap_forced(index, map);
        index->mapping = FALSE;
        return ret;
    }

    index->hdr = &map->hdr;
    index->map = map;
    i_assert(map->hdr.messages_count == map->records_count);
    index->mapping = FALSE;
    return 1;
}

int mail_index_get_latest_header(struct mail_index *index,
                 struct mail_index_header *hdr_r)
{
    size_t pos;
    int ret;

    if (!index->mmap_disable) {
        ret = mail_index_map(index, FALSE);
        if (ret > 0)
            *hdr_r = *index->hdr;
        else
            memset(hdr_r, 0, sizeof(*hdr_r));
    } else {
        ret = mail_index_read_header(index, hdr_r, &pos);
    }
    return ret;
}

struct mail_index_map *
mail_index_map_clone(struct mail_index_map *map, uint32_t new_record_size)
{
    struct mail_index_map *mem_map;
    struct mail_index_header *hdr;
    struct mail_index_ext *extensions;
    void *src, *dest;
    size_t size, copy_size;
    unsigned int i, count;

        size = map->records_count * new_record_size;

    mem_map = i_new(struct mail_index_map, 1);
    mem_map->refcount = 1;
    mem_map->buffer = buffer_create_dynamic(default_pool, size);
    if (map->hdr.record_size == new_record_size)
        buffer_append(mem_map->buffer, map->records, size);
    else {
        copy_size = I_MIN(map->hdr.record_size, new_record_size);
        src = map->records;
        for (i = 0; i < map->records_count; i++) {
            dest = buffer_append_space_unsafe(mem_map->buffer,
                              new_record_size);
            memcpy(dest, src, copy_size);
            src = PTR_OFFSET(src, map->hdr.record_size);
        }
    }

    mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL);
    mem_map->records_count = map->records_count;

    mem_map->hdr_copy_buf =
        buffer_create_dynamic(default_pool, map->hdr.header_size);
    if (map->hdr.base_header_size < sizeof(*hdr))
        buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr));
    buffer_write(mem_map->hdr_copy_buf, 0,
             &map->hdr, map->hdr.base_header_size);
    buffer_append(mem_map->hdr_copy_buf,
              CONST_PTR_OFFSET(map->hdr_base,
                       map->hdr.base_header_size),
              map->hdr.header_size - map->hdr.base_header_size);

    hdr = buffer_get_modifyable_data(mem_map->hdr_copy_buf, NULL);
    if (hdr->base_header_size < sizeof(*hdr))
        hdr->base_header_size = sizeof(*hdr);
    hdr->record_size = new_record_size;
    mem_map->hdr = *hdr;
    mem_map->hdr_base = hdr;

    /* copy extensions */
    if (array_is_created(&map->ext_id_map)) {
        count = array_count(&map->ext_id_map);
        mail_index_map_init_extbufs(mem_map, count + 2);

        array_append_array(&mem_map->extensions, &map->extensions);
        array_append_array(&mem_map->ext_id_map, &map->ext_id_map);

        /* fix the name pointers to use our own pool */
        extensions = array_get_modifyable(&mem_map->extensions, &count);
        for (i = 0; i < count; i++) {
            extensions[i].name = p_strdup(mem_map->extension_pool,
                              extensions[i].name);
        }
    }

    return mem_map;
}

int mail_index_map_get_ext_idx(struct mail_index_map *map,
                   uint32_t ext_id, uint32_t *idx_r)
{
    const uint32_t *id;

    if (!array_is_created(&map->ext_id_map) ||
        ext_id >= array_count(&map->ext_id_map))
        return 0;

    id = array_idx(&map->ext_id_map, ext_id);
    *idx_r = *id;
    return *idx_r != (uint32_t)-1;
}

static int mail_index_try_open_only(struct mail_index *index)
{
    int i;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

    for (i = 0; i < 3; i++) {
        index->fd = open(index->filepath, O_RDWR);
        if (index->fd == -1 && errno == EACCES) {
            index->fd = open(index->filepath, O_RDONLY);
            index->readonly = TRUE;
        }
        if (index->fd != -1 || errno != ESTALE)
            break;

        /* May happen with some OSes with NFS. Try again, although
           there's still a race condition with another computer
           creating the index file again. However, we can't try forever
           as ESTALE happens also if index directory has been deleted
           from server.. */
    }
    if (index->fd == -1) {
        if (errno != ENOENT)
            return mail_index_set_syscall_error(index, "open()");

        /* have to create it */
        return 0;
    }
    return 1;
}

static int
mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)
{
    unsigned int lock_id;
    int ret;

    if (lock_id_r != NULL)
        *lock_id_r = 0;

    if (MAIL_INDEX_IS_IN_MEMORY(index))
        return 0;

    ret = mail_index_try_open_only(index);
    if (ret <= 0)
        return ret;

    if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) {
        (void)close(index->fd);
        index->fd = -1;
        return -1;
    }
    ret = mail_index_map(index, FALSE);
    if (ret == 0) {
        /* it's corrupted - recreate it */
        mail_index_unlock(index, lock_id);
        if (lock_id_r != NULL)
            *lock_id_r = 0;

        (void)close(index->fd);
        index->fd = -1;
    } else {
        if (lock_id_r != NULL)
            *lock_id_r = lock_id;
        else
            mail_index_unlock(index, lock_id);
    }
    return ret;
}

int mail_index_write_base_header(struct mail_index *index,
                 const struct mail_index_header *hdr)
{
    size_t hdr_size;

    hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);

    if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
        memcpy(index->map->mmap_base, hdr, hdr_size);
        if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
            return mail_index_set_syscall_error(index, "msync()");
        index->map->hdr = *hdr;
    } else {
        if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
            if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
                mail_index_set_syscall_error(index,
                                 "pwrite_full()");
                return -1;
            }
        }

        index->map->hdr = *hdr;
        buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size);
    }

    return 0;
}

int mail_index_create_tmp_file(struct mail_index *index, const char **path_r)
{
        mode_t old_mask;
    const char *path;
    int fd;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

    path = *path_r = t_strconcat(index->filepath, ".tmp", NULL);
    old_mask = umask(0);
    fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode);
    umask(old_mask);
    if (fd == -1)
        return mail_index_file_set_syscall_error(index, path, "open()");

    if (index->gid != (gid_t)-1 && fchown(fd, (uid_t)-1, index->gid) < 0) {
        mail_index_file_set_syscall_error(index, path, "fchown()");
        return -1;
    }

    return fd;
}

static int mail_index_create(struct mail_index *index,
                 struct mail_index_header *hdr)
{
    const char *path;
    uint32_t seq;
    uoff_t offset;
    int ret;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
    i_assert(index->lock_type == F_UNLCK);

    /* log file lock protects index creation */
    if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
        return -1;

    ret = mail_index_try_open(index, NULL);
    if (ret != 0) {
        mail_transaction_log_sync_unlock(index->log);
        return ret < 0 ? -1 : 0;
    }

    /* create it fully in index.tmp first */
    index->fd = mail_index_create_tmp_file(index, &path);
    if (index->fd == -1)
        ret = -1;
    else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
        mail_index_file_set_syscall_error(index, path, "write_full()");
        ret = -1;
    } else {
        index->lock_type = F_WRLCK;
        ret = mail_index_map(index, FALSE);
        index->lock_type = F_UNLCK;
    }

    if (ret == 0) {
        /* it's corrupted even while we just created it,
           should never happen unless someone pokes the file directly */
        mail_index_set_error(index,
            "Newly created index file is corrupted: %s", path);
        ret = -1;
    }

    if (ret < 0) {
        if (unlink(path) < 0 && errno != ENOENT) {
            mail_index_file_set_syscall_error(index, path,
                              "unlink()");
        }
    } else {
        /* make it visible to others */
        if (rename(path, index->filepath) < 0) {
            mail_index_set_error(index, "rename(%s, %s) failed: %m",
                         path, index->filepath);
            ret = -1;
        }
    }

    mail_transaction_log_sync_unlock(index->log);
    return ret;
}

static void mail_index_header_init(struct mail_index_header *hdr)
{
    time_t now = time(NULL);

    i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);

    memset(hdr, 0, sizeof(*hdr));

    hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
    hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
    hdr->base_header_size = sizeof(*hdr);
    hdr->header_size = sizeof(*hdr);
    hdr->record_size = sizeof(struct mail_index_record);

#ifndef WORDS_BIGENDIAN
    hdr->compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
    hdr->compat_data[1] = sizeof(uoff_t);
    hdr->compat_data[2] = sizeof(time_t);

    hdr->indexid = now;

    hdr->next_uid = 1;
}

static void mail_index_create_in_memory(struct mail_index *index,
                    const struct mail_index_header *hdr)
{
        struct mail_index_header tmp_hdr;
    struct mail_index_map tmp_map;

    if (hdr == NULL) {
        mail_index_header_init(&tmp_hdr);
        hdr = &tmp_hdr;
    }

    memset(&tmp_map, 0, sizeof(tmp_map));
    tmp_map.hdr = *hdr;
    tmp_map.hdr_base = hdr;

    /* a bit kludgy way to do this, but it initializes everything
       nicely and correctly */
    index->map = mail_index_map_clone(&tmp_map, hdr->record_size);
    index->hdr = &index->map->hdr;
}

/* returns -1 = error, 0 = won't create, 1 = ok */
static int mail_index_open_files(struct mail_index *index,
                 enum mail_index_open_flags flags)
{
    struct mail_index_header hdr;
    unsigned int lock_id = 0;
    int ret, created = FALSE;

    ret = mail_index_try_open(index, &lock_id);
    if (ret > 0)
        hdr = *index->hdr;
    else if (ret == 0) {
        /* doesn't exist, or corrupted */
        if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
            return 0;
        mail_index_header_init(&hdr);
        index->hdr = &hdr;
    } else if (ret < 0)
        return -1;

    index->indexid = hdr.indexid;

    index->log = mail_transaction_log_open_or_create(index);
    if (index->log == NULL)
        return -1;

    if (index->fd == -1) {
        if (lock_id != 0) {
            mail_index_unlock(index, lock_id);
            lock_id = 0;
        }
        if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
            if (mail_index_create(index, &hdr) < 0)
                return -1;
        } else {
            mail_index_create_in_memory(index, &hdr);
        }
        created = TRUE;
    }

    if (lock_id == 0) {
        if (mail_index_lock_shared(index, FALSE, &lock_id) < 0)
            return -1;

    }

    index->cache = created ? mail_cache_create(index) :
        mail_cache_open_or_create(index);

    mail_index_unlock(index, lock_id);
    return 1;
}

int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags,
            enum mail_index_lock_method lock_method)
{
    int i = 0, ret;

    if (index->opened) {
        if (index->hdr != NULL &&
            (index->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
            /* corrupted, reopen files */
                        mail_index_close(index);
        } else {
            return 0;
        }
    }

    index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
        i_strdup("(in-memory index)") :
        i_strconcat(index->dir, "/", index->prefix, NULL);

    do {
        index->shared_lock_count = 0;
        index->excl_lock_count = 0;
        index->lock_type = F_UNLCK;
        index->lock_id = 2;

        index->readonly = FALSE;
        index->nodiskspace = FALSE;
        index->index_lock_timeout = FALSE;
        index->log_locked = FALSE;
        index->mmap_disable =
            (flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
        index->mmap_no_write =
            (flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0;
        index->lock_method = lock_method;

        /* don't even bother to handle dotlocking without mmap being
           disabled. that combination simply doesn't make any sense */
        if (lock_method == MAIL_INDEX_LOCK_DOTLOCK &&
            !index->mmap_disable) {
            i_fatal("lock_method=dotlock and mmap_disable=no "
                "combination isn't supported. "
                "You don't _really_ want it anyway.");
        }

        ret = mail_index_open_files(index, flags);
        if (ret <= 0)
            break;

        index->opened = TRUE;
        if (index->fsck) {
            index->fsck = FALSE;
            ret = mail_index_fsck(index);
            if (ret == 0) {
                /* completely broken, reopen */
                if (i++ < 3)
                    continue;
                /* too many tries */
                ret = -1;
            }
        }
        break;
    } while (1);

    if (ret <= 0)
        mail_index_close(index);

    return ret;
}

void mail_index_close(struct mail_index *index)
{
    if (index->log != NULL) {
        mail_transaction_log_close(index->log);
        index->log = NULL;
    }

    if (index->map != NULL) {
        mail_index_unmap(index, index->map);
        index->map = NULL;
    }

    if (index->cache != NULL) {
        mail_cache_free(index->cache);
        index->cache = NULL;
    }

    if (index->fd != -1) {
        if (close(index->fd) < 0)
            mail_index_set_syscall_error(index, "close()");
        index->fd = -1;
    }

    i_free(index->copy_lock_path);
    index->copy_lock_path = NULL;
    i_free(index->filepath);
    index->filepath = NULL;

    index->indexid = 0;
    index->opened = FALSE;
}

int mail_index_reopen(struct mail_index *index, int fd)
{
    struct mail_index_map *old_map;
    unsigned int old_shared_locks, old_lock_id, lock_id = 0;
    int ret, old_fd, old_lock_type;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

    old_map = index->map;
    old_fd = index->fd;
    old_map->refcount++;

    /* new file, new locks. the old fd can keep it's locks, they don't
       matter anymore as no-one's going to modify the file. */
    old_lock_type = index->lock_type;
    old_lock_id = index->lock_id;
    old_shared_locks = index->shared_lock_count;

    if (index->lock_type == F_RDLCK)
        index->lock_type = F_UNLCK;
    index->lock_id += 2;
    index->shared_lock_count = 0;

    if (fd != -1) {
        index->fd = fd;
        ret = 0;
    } else {
        i_assert(index->excl_lock_count == 0);
        ret = mail_index_try_open_only(index);
        if (ret > 0)
            ret = mail_index_lock_shared(index, FALSE, &lock_id);
        else if (ret == 0) {
            /* index file is lost */
            ret = -1;
        }
    }

    if (ret == 0) {
        /* read the new mapping. note that with mmap_disable we want
           to keep the old mapping in index->map so we can update it
           by reading transaction log. */
        if (mail_index_map(index, TRUE) <= 0)
            ret = -1;
    }

    if (lock_id != 0)
        mail_index_unlock(index, lock_id);

    if (ret == 0) {
        mail_index_unmap(index, old_map);
        if (close(old_fd) < 0)
            mail_index_set_syscall_error(index, "close()");
    } else {
        if (index->map != NULL)
            mail_index_unmap(index, index->map);
        if (index->fd != -1) {
            if (close(index->fd) < 0)
                mail_index_set_syscall_error(index, "close()");
        }

        index->map = old_map;
        index->hdr = &index->map->hdr;
        index->fd = old_fd;
        index->lock_type = old_lock_type;
        index->lock_id = old_lock_id;
        index->shared_lock_count = old_shared_locks;
    }
    return ret;
}

int mail_index_reopen_if_needed(struct mail_index *index)
{
    struct stat st1, st2;

    if (MAIL_INDEX_IS_IN_MEMORY(index))
        return 0;

    if (fstat(index->fd, &st1) < 0)
        return mail_index_set_syscall_error(index, "fstat()");
    if (stat(index->filepath, &st2) < 0) {
        mail_index_set_syscall_error(index, "stat()");
        if (errno != ENOENT)
            return -1;

        /* lost it? recreate */
        (void)mail_index_mark_corrupted(index);
        return -1;
    }

    if (st1.st_ino != st2.st_ino ||
        !CMP_DEV_T(st1.st_dev, st2.st_dev)) {
        if (mail_index_reopen(index, -1) < 0)
            return -1;
        return 1;
    } else {
        return 0;
    }
}

int mail_index_refresh(struct mail_index *index)
{
    unsigned int lock_id;
    int ret;

    if (MAIL_INDEX_IS_IN_MEMORY(index))
        return 0;

    if (index->excl_lock_count > 0) {
        /* we have index exclusively locked, nothing could
           have changed. */
        return 0;
    }

    if (!index->mmap_disable) {
        /* reopening is all we need */
        return mail_index_reopen_if_needed(index);
    }

    /* mail_index_map() simply reads latest changes from transaction log,
       which makes us fully refreshed. */
    if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
        return -1;

    ret = mail_index_map(index, FALSE);
    mail_index_unlock(index, lock_id);
    return ret <= 0 ? -1 : 0;
}

struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
    return index->cache;
}

int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
    va_list va;

    i_free(index->error);

    if (fmt == NULL)
        index->error = NULL;
    else {
        va_start(va, fmt);
        index->error = i_strdup_vprintf(fmt, va);
        va_end(va);

        i_error("%s", index->error);
    }

    return -1;
}

void mail_index_set_inconsistent(struct mail_index *index)
{
    index->indexid = 0;
}

void mail_index_mark_corrupted(struct mail_index *index)
{
    struct mail_index_header hdr;

    mail_index_set_inconsistent(index);

    if (index->readonly)
        return;

    hdr = *index->hdr;
    hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
    if (mail_index_write_base_header(index, &hdr) == 0) {
        if (!MAIL_INDEX_IS_IN_MEMORY(index) && fsync(index->fd) < 0)
            mail_index_set_syscall_error(index, "fsync()");
    }
}

int mail_index_set_syscall_error(struct mail_index *index,
                 const char *function)
{
    i_assert(function != NULL);

    if (ENOSPACE(errno)) {
        index->nodiskspace = TRUE;
        return -1;
    }

    return mail_index_set_error(index, "%s failed with index file %s: %m",
                    function, index->filepath);
}

int mail_index_file_set_syscall_error(struct mail_index *index,
                      const char *filepath,
                      const char *function)
{
    i_assert(filepath != NULL);
    i_assert(function != NULL);

    if (ENOSPACE(errno)) {
        index->nodiskspace = TRUE;
        return -1;
    }

    return mail_index_set_error(index, "%s failed with file %s: %m",
                    function, filepath);
}

enum mail_index_error mail_index_get_last_error(struct mail_index *index)
{
    if (index->nodiskspace)
        return MAIL_INDEX_ERROR_DISKSPACE;
    if (index->error != NULL)
        return MAIL_INDEX_ERROR_INTERNAL;

    return MAIL_INDEX_ERROR_NONE;
}

const char *mail_index_get_error_message(struct mail_index *index)
{
    return index->error;
}

void mail_index_reset_error(struct mail_index *index)
{
    if (index->error != NULL) {
        i_free(index->error);
        index->error = NULL;
    }

    index->nodiskspace = FALSE;
        index->index_lock_timeout = FALSE;
}

uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
    unsigned char buf[4];

    i_assert(offset < 0x40000000);
    i_assert((offset & 3) == 0);

    offset >>= 2;
    buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
    buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
    buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
    buf[3] = 0x80 |  (offset & 0x0000007f);
    return *((uint32_t *) buf);
}

uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
    const unsigned char *buf = (const unsigned char *) &offset;

    if ((offset & 0x80808080) != 0x80808080)
        return 0;

    return (((uint32_t)buf[3] & 0x7f) << 2) |
        (((uint32_t)buf[2] & 0x7f) << 9) |
        (((uint32_t)buf[1] & 0x7f) << 16) |
        (((uint32_t)buf[0] & 0x7f) << 23);
}