src/lib-index/mail-index-map.c

	mail-index-map.c revision d051664df497582e1eb75a9f238d04b65e858db8
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina/* Copyright (C) 2003-2007 Timo Sirainen */
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#include "lib.h"
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#include "array.h"
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#include "nfs-workarounds.h"
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#include "mmap-util.h"
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#include "read-full.h"
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#include "mail-index-private.h"
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#include "mail-index-sync-private.h"
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#include "mail-transaction-log-private.h"
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březinastatic void mail_index_map_init_extbufs(struct mail_index_map *map,
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina                    unsigned int initial_count)
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina{
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#define EXTENSION_NAME_APPROX_LEN 20
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#define EXT_GLOBAL_ALLOC_SIZE \
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina#define EXT_PER_ALLOC_SIZE \
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    (EXTENSION_NAME_APPROX_LEN + \
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina     sizeof(struct mail_index_ext) + sizeof(uint32_t))
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    size_t size;
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    if (map->extension_pool == NULL) {
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina        size = EXT_GLOBAL_ALLOC_SIZE +
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina            initial_count * EXT_PER_ALLOC_SIZE;
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina        map->extension_pool =
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina            pool_alloconly_create("map extensions",
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina                          nearest_power(size));
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    } else {
beeef7f627a5ed9264de25ee4c76eb9620c1c984Pavel Březina        p_clear(map->extension_pool);
beeef7f627a5ed9264de25ee4c76eb9620c1c984Pavel Březina
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina        /* try to use the existing pool's size for initial_count so
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina           we don't grow it unneededly */
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina        size = p_get_max_easy_alloc_size(map->extension_pool);
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina        if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina            initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina                EXT_PER_ALLOC_SIZE;
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina        }
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    }
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    p_array_init(&map->extensions, map->extension_pool, initial_count);
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina}
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březinabool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name,
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina                   uint32_t *idx_r)
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina{
beeef7f627a5ed9264de25ee4c76eb9620c1c984Pavel Březina    const struct mail_index_ext *extensions;
beeef7f627a5ed9264de25ee4c76eb9620c1c984Pavel Březina    unsigned int i, size;
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina    if (array_is_created(&map->extensions)) {
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina        extensions = array_get(&map->extensions, &size);
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina        for (i = 0; i < size; i++) {
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina            if (strcmp(extensions[i].name, name) == 0) {
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina                if (idx_r != NULL)
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina                    *idx_r = i;
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina                return TRUE;
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina            }
beeef7f627a5ed9264de25ee4c76eb9620c1c984Pavel Březina        }
beeef7f627a5ed9264de25ee4c76eb9620c1c984Pavel Březina    }
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina    return FALSE;
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina}
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březinastatic size_t get_ext_size(size_t name_len)
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina{
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina    size_t size = sizeof(struct mail_index_ext_header) + name_len;
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina    return MAIL_INDEX_HEADER_SIZE_ALIGN(size);
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina}
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březinauint32_t
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březinamail_index_map_register_ext(struct mail_index_map *map, const char *name,
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina                uint32_t ext_offset, uint32_t hdr_size,
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina                uint32_t record_offset, uint32_t record_size,
4e5d19f659d8c545c4ed3c307c95cfe4f2ca33cbPavel Březina                uint32_t record_align, uint32_t reset_id)
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina{
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina    struct mail_index_ext *ext;
9e9ad4cb181c6c0ec70caacfb31319753f889e98Pavel Březina    uint32_t idx, empty_idx = (uint32_t)-1;
9e9ad4cb181c6c0ec70caacfb31319753f889e98Pavel Březina
9e9ad4cb181c6c0ec70caacfb31319753f889e98Pavel Březina    if (!array_is_created(&map->extensions)) {
bd4c2ed5aec7f57ea04500f0e43f151eedfdde45Pavel Březina                mail_index_map_init_extbufs(map, 5);
bd4c2ed5aec7f57ea04500f0e43f151eedfdde45Pavel Březina        idx = 0;
c6cf752337f5977ce3753b7113dc1a2342c86319Pavel Březina    } else {
c6cf752337f5977ce3753b7113dc1a2342c86319Pavel Březina        idx = array_count(&map->extensions);
9e9ad4cb181c6c0ec70caacfb31319753f889e98Pavel Březina    }
9e9ad4cb181c6c0ec70caacfb31319753f889e98Pavel Březina    i_assert(!mail_index_map_lookup_ext(map, name, NULL));
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina    ext = array_append_space(&map->extensions);
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina    ext->name = p_strdup(map->extension_pool, name);
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina    ext->ext_offset = ext_offset;
827a016a07d5f911cc4195be89896a376fd71f59Sumit Bose    ext->hdr_offset = ext_offset + get_ext_size(strlen(name));
861dbe0794739a1c93a5bed00913c7442a2bdac9Sumit Bose    ext->hdr_size = hdr_size;
ef55b0e470a8fbcf6e6d0a55883145e02a907842Sumit Bose    ext->record_offset = record_offset;
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina    ext->record_size = record_size;
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina    ext->record_align = record_align;
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina    ext->reset_id = reset_id;
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina    ext->index_idx = mail_index_ext_register(map->index, name, hdr_size,
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina                         record_size, record_align);
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina    /* Update index ext_id -> map ext_id mapping. Fill non-used
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina       ext_ids with (uint32_t)-1 */
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina    while (array_count(&map->ext_id_map) < ext->index_idx)
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina        array_append(&map->ext_id_map, &empty_idx, 1);
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina    array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina    return idx;
e9a2e7afbd09c23dd8748246e09831ed7b17d7c5Thomas Equeter}
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina
37d2194cc9ea4d0254c88a3419e2376572562babPavel Březinastatic int mail_index_parse_extensions(struct mail_index_map *map)
37d2194cc9ea4d0254c88a3419e2376572562babPavel Březina{
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina    struct mail_index *index = map->index;
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina    const struct mail_index_ext_header *ext_hdr;
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina    unsigned int i, old_count;
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina    const char *name;
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina    uint32_t ext_id, ext_offset, offset, name_offset;
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina    size_t size_left;
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina    /* extension headers always start from 64bit offsets, so if base header
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina       doesn't happen to be 64bit aligned we'll skip some bytes */
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina    offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina    if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina        /* nothing to do, skip allocatations and all */
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina        return 1;
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina    }
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina    old_count = array_count(&index->extensions);
e9a2e7afbd09c23dd8748246e09831ed7b17d7c5Thomas Equeter    mail_index_map_init_extbufs(map, old_count + 5);
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina    ext_id = (uint32_t)-1;
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina    for (i = 0; i < old_count; i++)
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina        array_append(&map->ext_id_map, &ext_id, 1);
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina    for (i = 0; offset < map->hdr.header_size; i++) {
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        ext_offset = offset;
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        /* Extension header contains:
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina           - struct mail_index_ext_header
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina           - name (not 0-terminated)
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina           - 64bit alignment padding
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina           - extension header contents
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina           - 64bit alignment padding
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        */
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        size_left = map->hdr.header_size - offset;
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        if (size_left < sizeof(*ext_hdr) ||
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina            size_left < get_ext_size(ext_hdr->name_size) +
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina            ext_hdr->hdr_size) {
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina            mail_index_set_error(index, "Corrupted index file %s: "
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina                "Header extension goes outside header",
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina                index->filepath);
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina            return -1;
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        }
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        name_offset = offset + sizeof(*ext_hdr);
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        offset += get_ext_size(ext_hdr->name_size);
397bc52dd09a8c032abc7ea47a6d81dba5957464Pavel Březina
62ebed8582285bd24efba92b9a06366511507946Pavel Březina        t_push();
b420aae3becdbf501deb2637e2a06636bd6ce1fePavel Březina        name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
62ebed8582285bd24efba92b9a06366511507946Pavel Březina                 ext_hdr->name_size);
9e9ad4cb181c6c0ec70caacfb31319753f889e98Pavel Březina
62ebed8582285bd24efba92b9a06366511507946Pavel Březina        if (mail_index_map_lookup_ext(map, name, NULL)) {
c747b0c875785ce693f70b50bdda0237c4b04e35Pavel Březina            mail_index_set_error(index, "Corrupted index file %s: "
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina                "Duplicate header extension %s",
a1e4113a5388e34c08459c5b69679c82ac2bddc9Pavel Březina                index->filepath, name);
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina            t_pop();
132e477d69e07e02fe6e4d668c0bb6226206474aPavel Březina            return -1;
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        }
8fe171bf5a7a570591418e6548105f1d5a0097b3Pavel Březina
d3c82d0170d6d7407549afdadd08aa7e11aeb9a2Pavel Březina        if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina            (ext_hdr->record_align == 0 && ext_hdr->record_size != 0) ||
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina            *name == '\0') {
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina            mail_index_set_error(index, "Corrupted index file %s: "
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina                         "Broken header extension %s",
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina                         index->filepath, *name == '\0' ?
397bc52dd09a8c032abc7ea47a6d81dba5957464Pavel Březina                         t_strdup_printf("#%d", i) : name);
772199031f0ec687fa1fefd939206858c440e5a1Pavel Březina            t_pop();
            return -1;
        }
        if (map->hdr.record_size <
            ext_hdr->record_offset + ext_hdr->record_size) {
            mail_index_set_error(index, "Corrupted index file %s: "
                "Record field %s points outside record size "
                "(%u < %u+%u)", index->filepath, name,
                map->hdr.record_size,
                ext_hdr->record_offset, ext_hdr->record_size);
            t_pop();
            return -1;
        }

        if (ext_hdr->record_size > 0 &&
            ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
             (map->hdr.record_size % ext_hdr->record_align) != 0)) {
            mail_index_set_error(index, "Corrupted index file %s: "
                "Record field %s alignmentation %u not used",
                index->filepath, name, ext_hdr->record_align);
            t_pop();
            return -1;
        }

        mail_index_map_register_ext(map, name, ext_offset,
                        ext_hdr->hdr_size,
                        ext_hdr->record_offset,
                        ext_hdr->record_size,
                        ext_hdr->record_align,
                        ext_hdr->reset_id);
        t_pop();

        offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
    }
    return 1;
}

static bool mail_index_check_header_compat(struct mail_index *index,
                       const struct mail_index_header *hdr,
                       uoff_t file_size)
{
        enum mail_index_header_compat_flags compat_flags = 0;

#ifndef WORDS_BIGENDIAN
    compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif

    if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
        /* major version change - handle silently(?) */
        return FALSE;
    }
    if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
        /* we've already complained about it */
        return FALSE;
    }

    if (hdr->compat_flags != compat_flags) {
        /* architecture change */
        mail_index_set_error(index, "Rebuilding index file %s: "
                     "CPU architecture changed",
                     index->filepath);
        return FALSE;
    }

    if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
        hdr->header_size < hdr->base_header_size) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "Corrupted header sizes (base %u, full %u)",
                     index->filepath, hdr->base_header_size,
                     hdr->header_size);
        return FALSE;
    }
    if (hdr->header_size > file_size) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "Corrupted header size (%u > %"PRIuUOFF_T")",
                     index->filepath, hdr->header_size,
                     file_size);
        return FALSE;
    }

    if (hdr->indexid != index->indexid) {
        if (index->indexid != 0) {
            mail_index_set_error(index, "Index file %s: "
                         "indexid changed: %u -> %u",
                         index->filepath, index->indexid,
                         hdr->indexid);
        }
        index->indexid = hdr->indexid;
        mail_transaction_log_indexid_changed(index->log);
    }

    return TRUE;
}

int mail_index_map_check_header(struct mail_index_map *map)
{
    struct mail_index *index = map->index;
    const struct mail_index_header *hdr = &map->hdr;

    if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1))
        return -1;

    /* following some extra checks that only take a bit of CPU */
    if (hdr->record_size < sizeof(struct mail_index_record)) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "record_size too small: %u < %"PRIuSIZE_T,
                     index->filepath, hdr->record_size,
                     sizeof(struct mail_index_record));
        return -1;
    }

    if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
        return 0;

    if (hdr->uid_validity == 0 && hdr->next_uid != 1)
        return 0;
    if (hdr->next_uid == 0)
        return 0;

    if (hdr->seen_messages_count > hdr->messages_count ||
        hdr->deleted_messages_count > hdr->messages_count)
        return 0;
    if (hdr->first_recent_uid == 0 ||
        hdr->first_recent_uid > hdr->next_uid ||
        hdr->first_unseen_uid_lowwater > hdr->next_uid ||
        hdr->first_deleted_uid_lowwater > hdr->next_uid)
        return 0;

    if (hdr->messages_count > 0) {
        /* last message's UID must be smaller than next_uid.
           also make sure it's not zero. */
        const struct mail_index_record *rec;

        rec = MAIL_INDEX_MAP_IDX(map, hdr->messages_count-1);
        if (rec->uid == 0 || rec->uid >= hdr->next_uid)
            return 0;
    }

    return 1;
}

static void mail_index_map_copy_hdr(struct mail_index_map *map,
                    const struct mail_index_header *hdr)
{
    if (hdr->base_header_size < sizeof(map->hdr)) {
        /* header smaller than ours, make a copy so our newer headers
           won't have garbage in them */
        memset(&map->hdr, 0, sizeof(map->hdr));
        memcpy(&map->hdr, hdr, hdr->base_header_size);
    } else {
        map->hdr = *hdr;
    }

    /* FIXME: backwards compatibility, remove later. In case this index is
       accessed with Dovecot v1.0, avoid recent message counter errors. */
    map->hdr.unused_old_recent_messages_count = 0;
}

static int mail_index_mmap(struct mail_index_map *map, uoff_t file_size)
{
    struct mail_index *index = map->index;
    struct mail_index_record_map *rec_map = map->rec_map;
    const struct mail_index_header *hdr;
    unsigned int records_count;

    i_assert(rec_map->mmap_base == NULL);

    buffer_free(rec_map->buffer);
    rec_map->buffer = NULL;

    if (file_size > SSIZE_T_MAX) {
        /* too large file to map into memory */
        mail_index_set_error(index, "Index file too large: %s",
                     index->filepath);
        return -1;
    }

    rec_map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
                  MAP_PRIVATE, index->fd, 0);
    if (rec_map->mmap_base == MAP_FAILED) {
        rec_map->mmap_base = NULL;
        mail_index_set_syscall_error(index, "mmap()");
        return -1;
    }
    rec_map->mmap_size = file_size;

    hdr = rec_map->mmap_base;
    if (rec_map->mmap_size >
        offsetof(struct mail_index_header, major_version) &&
        hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
        /* major version change - handle silently */
        return 0;
    }

    if (rec_map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
        mail_index_set_error(index, "Corrupted index file %s: "
                     "File too small (%"PRIuSIZE_T")",
                     index->filepath, rec_map->mmap_size);
        return 0;
    }

    if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size)) {
        /* Can't use this file */
        return 0;
    }

    rec_map->mmap_used_size = hdr->header_size +
        hdr->messages_count * hdr->record_size;

    if (rec_map->mmap_used_size > rec_map->mmap_size) {
        records_count = (rec_map->mmap_size - hdr->header_size) /
            hdr->record_size;
        mail_index_set_error(index, "Corrupted index file %s: "
                     "messages_count too large (%u > %u)",
                     index->filepath, hdr->messages_count,
                     records_count);
        return 0;
    }

    mail_index_map_copy_hdr(map, hdr);

    map->hdr_base = rec_map->mmap_base;
    rec_map->records = PTR_OFFSET(rec_map->mmap_base, map->hdr.header_size);
    rec_map->records_count = map->hdr.messages_count;
    return 1;
}

static int mail_index_read_header(struct mail_index *index,
                  void *buf, size_t buf_size, size_t *pos_r)
{
    size_t pos;
    int ret;

    memset(buf, 0, sizeof(struct mail_index_header));

        /* try to read the whole header, but it's not necessarily an error to
       read less since the older versions of the index format could be
       smaller. Request reading up to buf_size, but accept if we only got
       the header. */
        pos = 0;
    do {
        ret = pread(index->fd, PTR_OFFSET(buf, pos),
                buf_size - pos, pos);
        if (ret > 0)
            pos += ret;
    } while (ret > 0 && pos < sizeof(struct mail_index_header));

    *pos_r = pos;
    return ret;
}

static int
mail_index_try_read_map(struct mail_index_map *map,
            uoff_t file_size, bool *retry_r, bool try_retry)
{
    struct mail_index *index = map->index;
    const struct mail_index_header *hdr;
    unsigned char read_buf[4096];
    const void *buf;
    void *data = NULL;
    ssize_t ret;
    size_t pos, records_size, initial_buf_pos = 0;
    unsigned int records_count, extra;

    i_assert(map->rec_map->mmap_base == NULL);

    *retry_r = FALSE;
    ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
    buf = read_buf; hdr = buf;

    if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
        hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
        /* major version change - handle silently */
        return 0;
    }

    if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
        (ret > 0 || pos >= hdr->base_header_size)) {
        if (!mail_index_check_header_compat(index, hdr, file_size)) {
            /* Can't use this file */
            return 0;
        }

        initial_buf_pos = pos;
        if (pos > hdr->header_size)
            pos = hdr->header_size;

        /* place the base header into memory. */
        buffer_reset(map->hdr_copy_buf);
        buffer_append(map->hdr_copy_buf, buf, pos);

        if (pos != hdr->header_size) {
            /* @UNSAFE: read the rest of the header into memory */
            data = buffer_append_space_unsafe(map->hdr_copy_buf,
                              hdr->header_size -
                              pos);
            ret = pread_full(index->fd, data,
                     hdr->header_size - pos, pos);
        }
    }

    if (ret > 0) {
        /* header read, read the records now. */
        records_size = (size_t)hdr->messages_count * hdr->record_size;

        if (file_size - hdr->header_size < records_size ||
            (hdr->record_size != 0 &&
             records_size / hdr->record_size != hdr->messages_count)) {
            records_count = (file_size - hdr->header_size) /
                hdr->record_size;
            mail_index_set_error(index, "Corrupted index file %s: "
                "messages_count too large (%u > %u)",
                index->filepath, hdr->messages_count,
                records_count);
            return 0;
        }

        if (map->rec_map->buffer == NULL) {
            map->rec_map->buffer =
                buffer_create_dynamic(default_pool,
                              records_size);
        }

        /* @UNSAFE */
        buffer_set_used_size(map->rec_map->buffer, 0);
        if (initial_buf_pos <= hdr->header_size)
            extra = 0;
        else {
            extra = initial_buf_pos - hdr->header_size;
            buffer_append(map->rec_map->buffer,
                      CONST_PTR_OFFSET(buf, hdr->header_size),
                      extra);
        }
        if (records_size > extra) {
            data = buffer_append_space_unsafe(map->rec_map->buffer,
                              records_size - extra);
            ret = pread_full(index->fd, data, records_size - extra,
                     hdr->header_size + extra);
        }
    }

    if (ret < 0) {
        if (errno == ESTALE && try_retry) {
            /* a new index file was renamed over this one. */
            *retry_r = TRUE;
            return 0;
        }
        mail_index_set_syscall_error(index, "pread_full()");
        return -1;
    }
    if (ret == 0) {
        mail_index_set_error(index,
            "Corrupted index file %s: File too small",
            index->filepath);
        return 0;
    }

    map->rec_map->records =
        buffer_get_modifiable_data(map->rec_map->buffer, NULL);
    map->rec_map->records_count = hdr->messages_count;

    mail_index_map_copy_hdr(map, hdr);
    map->hdr_base = map->hdr_copy_buf->data;
    return 1;
}

static int mail_index_read_map(struct mail_index_map *map, uoff_t file_size)
{
    struct mail_index *index = map->index;
    mail_index_sync_lost_handler_t *const *handlers;
    struct stat st;
    unsigned int i, count;
    int ret;
    bool try_retry, retry;

    /* notify all "sync lost" handlers */
    handlers = array_get(&index->sync_lost_handlers, &count);
    for (i = 0; i < count; i++)
        (*handlers[i])(index);

    for (i = 0;; i++) {
        try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
        if (file_size == (uoff_t)-1) {
            /* fstat() below failed */
            ret = 0;
            retry = try_retry;
        } else {
            ret = mail_index_try_read_map(map, file_size,
                              &retry, try_retry);
        }
        if (ret != 0 || !retry)
            break;

        /* ESTALE - reopen index file */
                if (close(index->fd) < 0)
            mail_index_set_syscall_error(index, "close()");
        index->fd = -1;

                ret = mail_index_try_open_only(index);
        if (ret <= 0) {
            if (ret == 0) {
                /* the file was lost */
                errno = ENOENT;
                mail_index_set_syscall_error(index, "open()");
            }
            return -1;
        }
        if (fstat(index->fd, &st) == 0)
            file_size = st.st_size;
        else {
            if (errno != ESTALE) {
                mail_index_set_syscall_error(index, "fstat()");
                return -1;
            }
            file_size = (uoff_t)-1;
        }
    }
    return ret;
}

static void mail_index_header_init(struct mail_index *index,
                   struct mail_index_header *hdr)
{
    i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);

    memset(hdr, 0, sizeof(*hdr));

    hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
    hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
    hdr->base_header_size = sizeof(*hdr);
    hdr->header_size = sizeof(*hdr);
    hdr->record_size = sizeof(struct mail_index_record);

#ifndef WORDS_BIGENDIAN
    hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif

    hdr->indexid = index->indexid;
    hdr->log_file_seq = 1;
    hdr->next_uid = 1;
    hdr->first_recent_uid = 1;
}

struct mail_index_map *mail_index_map_alloc(struct mail_index *index)
{
    struct mail_index_map tmp_map;

    memset(&tmp_map, 0, sizeof(tmp_map));
    mail_index_header_init(index, &tmp_map.hdr);
    tmp_map.index = index;
    tmp_map.hdr_base = &tmp_map.hdr;

    /* a bit kludgy way to do this, but it initializes everything
       nicely and correctly */
    return mail_index_map_clone(&tmp_map);
}

static int mail_index_map_latest_file(struct mail_index *index,
                      struct mail_index_map **map)
{
    struct mail_index_map *new_map;
    struct stat st;
    unsigned int lock_id;
    uoff_t file_size;
    bool use_mmap;
    int ret;

    ret = mail_index_reopen_if_changed(index);
    if (ret <= 0) {
        if (ret < 0)
            return -1;

        /* the index file is lost/broken. let's hope that we can
           build it from the transaction log. */
        return 0;
    }

    /* the index file is still open, lock it */
    if (mail_index_lock_shared(index, &lock_id) < 0)
        return -1;

    if (index->nfs_flush)
        nfs_flush_attr_cache_fd(index->filepath, index->fd);

    if (fstat(index->fd, &st) == 0)
        file_size = st.st_size;
    else {
        if (errno != ESTALE) {
            mail_index_set_syscall_error(index, "fstat()");
            mail_index_unlock(index, &lock_id);
            return -1;
        }
        file_size = (uoff_t)-1;
    }

    /* mmaping seems to be slower than just reading the file, so even if
       mmap isn't disabled don't use it unless the file is large enough */
    use_mmap = !index->mmap_disable && file_size != (uoff_t)-1 &&
        file_size > MAIL_INDEX_MMAP_MIN_SIZE;

    new_map = mail_index_map_alloc(index);
    if (use_mmap) {
        new_map->rec_map->lock_id = lock_id;
        ret = mail_index_mmap(new_map, file_size);
    } else {
        ret = mail_index_read_map(new_map, file_size);
        mail_index_unlock(index, &lock_id);
    }
    if (ret > 0) {
        /* make sure the header is ok before using this mapping */
        ret = mail_index_map_check_header(new_map);
        if (ret >= 0) {
            ret = mail_index_parse_extensions(new_map);
            if (ret > 0) {
                if (mail_index_map_parse_keywords(new_map) < 0)
                    ret = -1;
            }
        }
        if (ret == 0)
            index->fsck = TRUE;
        else if (ret < 0)
            ret = 0;
    }
    if (ret <= 0) {
        mail_index_unmap(&new_map);
        return ret;
    }
    i_assert(new_map->rec_map->records != NULL);

    index->last_read_log_file_seq = new_map->hdr.log_file_seq;
    index->last_read_log_file_head_offset =
        new_map->hdr.log_file_head_offset;
    index->last_read_log_file_tail_offset =
        new_map->hdr.log_file_tail_offset;
    index->last_read_stat = st;

    mail_index_unmap(map);
    *map = new_map;
    return 1;
}

int mail_index_map(struct mail_index *index,
           enum mail_index_sync_handler_type type)
{
    int ret;

    i_assert(index->lock_type != F_WRLCK);
    i_assert(!index->mapping);

    index->mapping = TRUE;

    if (index->map == NULL)
        index->map = mail_index_map_alloc(index);

    /* first try updating the existing mapping from transaction log. */
    if (index->map->hdr.indexid != 0 && index->indexid != 0) {
        /* we're not creating the index, or opening transaction log.
           sync this as a view from transaction log. */
        ret = mail_index_sync_map(&index->map, type, FALSE);
    } else {
        ret = 0;
    }

    if (ret == 0) {
        /* try to open and read the latest index. if it fails for
           any reason, we'll fallback to updating the existing mapping
           from transaction logs (which we'll also do even if the
           reopening succeeds) */
        (void)mail_index_map_latest_file(index, &index->map);

        /* if we're creating the index file, we don't have any
           logs yet */
        if (index->log->head != NULL && index->indexid != 0) {
            /* and update the map with the latest changes from
               transaction log */
            ret = mail_index_sync_map(&index->map, type, TRUE);
        }
    }

    index->mapping = FALSE;
    return ret;
}

static void mail_index_record_map_free(struct mail_index_map *map,
                       struct mail_index_record_map *rec_map)
{
    if (rec_map->lock_id != 0)
        mail_index_unlock(map->index, &rec_map->lock_id);

    if (rec_map->buffer != NULL) {
        i_assert(rec_map->mmap_base == NULL);
        buffer_free(rec_map->buffer);
        rec_map->buffer = NULL;
    } else if (rec_map->mmap_base != NULL) {
        i_assert(rec_map->buffer == NULL);
        if (munmap(rec_map->mmap_base, rec_map->mmap_size) < 0)
            mail_index_set_syscall_error(map->index, "munmap()");
        rec_map->mmap_base = NULL;
    }
    array_free(&rec_map->maps);
    i_free(rec_map);
}

static void mail_index_record_map_unlink(struct mail_index_map *map)
{
    struct mail_index_map *const *maps;
    unsigned int i, count;

    maps = array_get(&map->rec_map->maps, &count);
    for (i = 0; i < count; i++) {
        if (maps[i] == map) {
            array_delete(&map->rec_map->maps, i, 1);
            if (i == 0 && count == 1)
                mail_index_record_map_free(map, map->rec_map);
            return;
        }
    }
    i_unreached();
}

void mail_index_unmap(struct mail_index_map **_map)
{
    struct mail_index_map *map = *_map;

    *_map = NULL;
    if (--map->refcount > 0)
        return;

    i_assert(map->refcount == 0);
    mail_index_record_map_unlink(map);

    if (map->extension_pool != NULL)
        pool_unref(map->extension_pool);
    if (array_is_created(&map->keyword_idx_map))
        array_free(&map->keyword_idx_map);
    buffer_free(map->hdr_copy_buf);
    i_free(map);
}

static void mail_index_map_copy_records(struct mail_index_record_map *dest,
                    const struct mail_index_record_map *src,
                    unsigned int record_size)
{
    size_t size;

    size = src->records_count * record_size;
    dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024));
    buffer_append(dest->buffer, src->records, size);

    dest->records = buffer_get_modifiable_data(dest->buffer, NULL);
    dest->records_count = src->records_count;

    /* if the map is ever written back to disk, we need to keep track of
       what has changed. */
    dest->write_seq_first = src->write_seq_first;
    dest->write_seq_last = src->write_seq_last;
}

static void mail_index_map_copy_header(struct mail_index_map *dest,
                       const struct mail_index_map *src)
{
    /* use src->hdr copy directly, because if we got here
       from syncing it has the latest changes. */
    dest->hdr = src->hdr;
    if (dest->hdr_copy_buf != NULL) {
        if (src == dest)
            return;

        buffer_set_used_size(dest->hdr_copy_buf, 0);
    } else {
        dest->hdr_copy_buf =
            buffer_create_dynamic(default_pool,
                          dest->hdr.header_size);
    }
    buffer_append(dest->hdr_copy_buf, &dest->hdr,
              I_MIN(sizeof(dest->hdr), src->hdr.base_header_size));
    if (src != dest) {
        buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size,
                 CONST_PTR_OFFSET(src->hdr_base,
                          src->hdr.base_header_size),
                 src->hdr.header_size - src->hdr.base_header_size);
    }
    dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
    i_assert(dest->hdr_copy_buf->used == dest->hdr.header_size);
}

static struct mail_index_record_map *
mail_index_record_map_alloc(struct mail_index_map *map)
{
    struct mail_index_record_map *rec_map;

    rec_map = i_new(struct mail_index_record_map, 1);
    i_array_init(&rec_map->maps, 4);
    array_append(&rec_map->maps, &map, 1);
    return rec_map;
}

struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map)
{
    struct mail_index_map *mem_map;
    struct mail_index_ext *extensions;
    unsigned int i, count;

    mem_map = i_new(struct mail_index_map, 1);
    mem_map->index = map->index;
    mem_map->refcount = 1;
    if (map->rec_map == NULL) {
        mem_map->rec_map = mail_index_record_map_alloc(mem_map);
        mem_map->rec_map->buffer =
            buffer_create_dynamic(default_pool, 1024);
    } else {
        mem_map->rec_map = map->rec_map;
        array_append(&mem_map->rec_map->maps, &mem_map, 1);
    }

    mail_index_map_copy_header(mem_map, map);

    mem_map->write_atomic = map->write_atomic;
    mem_map->write_base_header = map->write_base_header;
    mem_map->write_ext_header = map->write_ext_header;

    /* copy extensions */
    if (array_is_created(&map->ext_id_map)) {
        count = array_count(&map->ext_id_map);
        mail_index_map_init_extbufs(mem_map, count + 2);

        array_append_array(&mem_map->extensions, &map->extensions);
        array_append_array(&mem_map->ext_id_map, &map->ext_id_map);

        /* fix the name pointers to use our own pool */
        extensions = array_get_modifiable(&mem_map->extensions, &count);
        for (i = 0; i < count; i++) {
            i_assert(extensions[i].record_offset +
                 extensions[i].record_size <=
                 mem_map->hdr.record_size);
            extensions[i].name = p_strdup(mem_map->extension_pool,
                              extensions[i].name);
        }
    }

    /* copy keyword map */
    if (array_is_created(&map->keyword_idx_map)) {
        i_array_init(&mem_map->keyword_idx_map,
                 array_count(&map->keyword_idx_map) + 4);
        array_append_array(&mem_map->keyword_idx_map,
                   &map->keyword_idx_map);
    }

    return mem_map;
}

void mail_index_record_map_move_to_private(struct mail_index_map *map)
{
    struct mail_index_record_map *new_map;

    if (array_count(&map->rec_map->maps) == 1)
        return;

    new_map = mail_index_record_map_alloc(map);
    mail_index_map_copy_records(new_map, map->rec_map,
                    map->hdr.record_size);

    mail_index_record_map_unlink(map);
    map->rec_map = new_map;
}

void mail_index_map_move_to_memory(struct mail_index_map *map)
{
    struct mail_index_record_map *new_map;

    if (map->rec_map->mmap_base == NULL)
        return;

    i_assert(map->rec_map->lock_id != 0);

    new_map = array_count(&map->rec_map->maps) == 1 ? map->rec_map :
        mail_index_record_map_alloc(map);

    mail_index_map_copy_records(new_map, map->rec_map,
                    map->hdr.record_size);
    mail_index_map_copy_header(map, map);

    if (new_map != map->rec_map) {
        mail_index_record_map_unlink(map);
        map->rec_map = new_map;
    } else {
        mail_index_unlock(map->index, &new_map->lock_id);
        if (munmap(new_map->mmap_base, new_map->mmap_size) < 0)
            mail_index_set_syscall_error(map->index, "munmap()");
        new_map->mmap_base = NULL;
    }
}

bool mail_index_map_get_ext_idx(struct mail_index_map *map,
                uint32_t ext_id, uint32_t *idx_r)
{
    const uint32_t *id;

    if (!array_is_created(&map->ext_id_map) ||
        ext_id >= array_count(&map->ext_id_map))
        return FALSE;

    id = array_idx(&map->ext_id_map, ext_id);
    *idx_r = *id;
    return *idx_r != (uint32_t)-1;
}