src/lib-index/mail-index-map.c

	mail-index-map.c revision 1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen/* Copyright (C) 2003-2007 Timo Sirainen */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "lib.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "array.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "nfs-workarounds.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mmap-util.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "read-full.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mail-index-private.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mail-index-sync-private.h"
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen#include "mail-transaction-log-private.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                    unsigned int initial_count)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define EXTENSION_NAME_APPROX_LEN 20
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define EXT_GLOBAL_ALLOC_SIZE \
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define EXT_PER_ALLOC_SIZE \
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    (EXTENSION_NAME_APPROX_LEN + \
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen     sizeof(struct mail_index_ext) + sizeof(uint32_t))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    size_t size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->extension_pool == NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        size = EXT_GLOBAL_ALLOC_SIZE +
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            initial_count * EXT_PER_ALLOC_SIZE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->extension_pool =
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            pool_alloconly_create("map extensions",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                          nearest_power(size));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        p_clear(map->extension_pool);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* try to use the existing pool's size for initial_count so
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen           we don't grow it unneededly */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        size = p_get_max_easy_alloc_size(map->extension_pool);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                EXT_PER_ALLOC_SIZE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    p_array_init(&map->extensions, map->extension_pool, initial_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenuint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    const struct mail_index_ext *extensions;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    unsigned int i, size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (!array_is_created(&map->extensions))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return (uint32_t)-1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    extensions = array_get(&map->extensions, &size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    for (i = 0; i < size; i++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (strcmp(extensions[i].name, name) == 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return i;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return (uint32_t)-1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenuint32_t
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenmail_index_map_register_ext(struct mail_index_map *map, const char *name,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                uint32_t hdr_offset, uint32_t hdr_size,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                uint32_t record_offset, uint32_t record_size,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                uint32_t record_align, uint32_t reset_id)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    struct mail_index_ext *ext;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    uint32_t idx, empty_idx = (uint32_t)-1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (!array_is_created(&map->extensions)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                mail_index_map_init_extbufs(map, 5);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        idx = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        idx = array_count(&map->extensions);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext = array_append_space(&map->extensions);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext->name = p_strdup(map->extension_pool, name);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext->hdr_offset = hdr_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext->hdr_size = hdr_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext->record_offset = record_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext->record_size = record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext->record_align = record_align;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext->reset_id = reset_id;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    ext->index_idx = mail_index_ext_register(map->index, name, hdr_size,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                         record_size, record_align);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    /* Update index ext_id -> map ext_id mapping. Fill non-used
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen       ext_ids with (uint32_t)-1 */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    while (array_count(&map->ext_id_map) < ext->index_idx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        array_append(&map->ext_id_map, &empty_idx, 1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return idx;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic bool size_check(size_t *size_left, size_t size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (size > *size_left)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    *size_left -= size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic size_t get_align(size_t name_len)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    size_t size = sizeof(struct mail_index_ext_header) + name_len;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenstatic int mail_index_parse_extensions(struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    const struct mail_index_ext_header *ext_hdr;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    unsigned int i, old_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    const char *name;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    uint32_t ext_id, offset, name_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    size_t size_left;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    /* extension headers always start from 64bit offsets, so if base header
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen       doesn't happen to be 64bit aligned we'll skip some bytes */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* nothing to do, skip allocatations and all */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    old_count = array_count(&index->extensions);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    mail_index_map_init_extbufs(map, old_count + 5);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ext_id = (uint32_t)-1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    for (i = 0; i < old_count; i++)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        array_append(&map->ext_id_map, &ext_id, 1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    for (i = 0; offset < map->hdr.header_size; i++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* Extension header contains:
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen           - struct mail_index_ext_header
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen           - name (not 0-terminated)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen           - 64bit alignment padding
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen           - extension header contents
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen           - 64bit alignment padding
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        size_left = map->hdr.header_size - offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (!size_check(&size_left, sizeof(*ext_hdr)) ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            !size_check(&size_left, ext_hdr->name_size) ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            !size_check(&size_left, get_align(ext_hdr->name_size)) ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            !size_check(&size_left, ext_hdr->hdr_size)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                "Header extension goes outside header",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                index->filepath);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        offset += sizeof(*ext_hdr);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        name_offset = offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        offset += ext_hdr->name_size + get_align(ext_hdr->name_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        t_push();
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                 ext_hdr->name_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                "Duplicate header extension %s",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                index->filepath, name);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            t_pop();
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen            (ext_hdr->record_align == 0 && ext_hdr->record_size != 0) ||
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen            *name == '\0') {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            mail_index_set_error(index, "Corrupted index file %s: "
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                         "Broken header extension %s",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                         index->filepath, *name == '\0' ?
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                         t_strdup_printf("#%d", i) : name);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            t_pop();
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (map->hdr.record_size <
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            ext_hdr->record_offset + ext_hdr->record_size) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                "Record field %s points outside record size "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                "(%u < %u+%u)", index->filepath, name,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                map->hdr.record_size,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                ext_hdr->record_offset, ext_hdr->record_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            t_pop();
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen        if (ext_hdr->record_size > 0 &&
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen            ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen             (map->hdr.record_size % ext_hdr->record_align) != 0)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                "Record field %s alignmentation %u not used",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                index->filepath, name, ext_hdr->record_align);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            t_pop();
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen        mail_index_map_register_ext(map, name,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                        offset, ext_hdr->hdr_size,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                        ext_hdr->record_offset,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                        ext_hdr->record_size,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                        ext_hdr->record_align,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                        ext_hdr->reset_id);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        t_pop();
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic bool mail_index_check_header_compat(struct mail_index *index,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                       const struct mail_index_header *hdr,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                       uoff_t file_size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        enum mail_index_header_compat_flags compat_flags = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#ifndef WORDS_BIGENDIAN
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#endif
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* major version change - handle silently(?) */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        /* we've already complained about it */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        return FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (hdr->compat_flags != compat_flags) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        /* architecture change */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        mail_index_set_error(index, "Rebuilding index file %s: "
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     "CPU architecture changed",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     index->filepath);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        hdr->header_size < hdr->base_header_size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        mail_index_set_error(index, "Corrupted index file %s: "
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     "Corrupted header sizes (base %u, full %u)",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     index->filepath, hdr->base_header_size,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     hdr->header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (hdr->header_size > file_size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        mail_index_set_error(index, "Corrupted index file %s: "
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     "Corrupted header size (%u > %"PRIuUOFF_T")",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     index->filepath, hdr->header_size,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     file_size);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen        return FALSE;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen    }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen    if (hdr->indexid != index->indexid) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen        if (index->indexid != 0) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen            mail_index_set_error(index, "Index file %s: "
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen                         "indexid changed: %u -> %u",
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen                         index->filepath, index->indexid,
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen                         hdr->indexid);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen        }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen        index->indexid = hdr->indexid;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen        mail_transaction_log_indexid_changed(index->log);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenstatic int mail_index_check_header(struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    const struct mail_index_header *hdr = &map->hdr;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    /* following some extra checks that only take a bit of CPU */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     "uid_validity = 0, next_uid = %u",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     index->filepath, hdr->next_uid);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (hdr->record_size < sizeof(struct mail_index_record)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     "record_size too small: %u < %"PRIuSIZE_T,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     index->filepath, hdr->record_size,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     sizeof(struct mail_index_record));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (hdr->next_uid == 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen    if (hdr->seen_messages_count > hdr->messages_count ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        hdr->deleted_messages_count > hdr->messages_count)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen    if (hdr->first_recent_uid == 0 ||
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen        hdr->first_recent_uid > hdr->next_uid ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        hdr->first_unseen_uid_lowwater > hdr->next_uid ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        hdr->first_deleted_uid_lowwater > hdr->next_uid)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->records_count > 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* last message's UID must be smaller than next_uid.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen           also make sure it's not zero. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        const struct mail_index_record *rec;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        rec = MAIL_INDEX_MAP_IDX(map, map->records_count-1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (rec->uid == 0 || rec->uid >= hdr->next_uid)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenstatic void mail_index_map_clear(struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->buffer != NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        i_assert(map->mmap_base == NULL);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        buffer_free(map->buffer);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->buffer = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    } else if (map->mmap_base != NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        i_assert(map->buffer == NULL);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (munmap(map->mmap_base, map->mmap_size) < 0)
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen            mail_index_set_syscall_error(map->index, "munmap()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->mmap_base = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->refcount > 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        memset(&map->hdr, 0, sizeof(map->hdr));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->mmap_size = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->mmap_used_size = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->records = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->records_count = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void mail_index_map_copy_hdr(struct mail_index_map *map,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                    const struct mail_index_header *hdr)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (hdr->base_header_size < sizeof(map->hdr)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* header smaller than ours, make a copy so our newer headers
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen           won't have garbage in them */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        memset(&map->hdr, 0, sizeof(map->hdr));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        memcpy(&map->hdr, hdr, hdr->base_header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->hdr = *hdr;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenstatic int mail_index_mmap(struct mail_index_map *map, uoff_t file_size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    const struct mail_index_header *hdr;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    unsigned int records_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->buffer != NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* we had temporarily used a buffer, eg. for updating index */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        buffer_free(map->buffer);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->buffer = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (file_size > SSIZE_T_MAX) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        /* too large file to map into memory */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        mail_index_set_error(index, "Index file too large: %s",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     index->filepath);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen    map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen                  MAP_PRIVATE, index->fd, 0);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->mmap_base == MAP_FAILED) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        map->mmap_base = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mail_index_set_syscall_error(index, "mmap()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    map->mmap_size = file_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    hdr = map->mmap_base;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->mmap_size >
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        offsetof(struct mail_index_header, major_version) &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* major version change - handle silently */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     "File too small (%"PRIuSIZE_T")",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     index->filepath, map->mmap_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (!mail_index_check_header_compat(index, hdr, map->mmap_size)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* Can't use this file */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    map->mmap_used_size = hdr->header_size +
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        hdr->messages_count * hdr->record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (map->mmap_used_size > map->mmap_size) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        records_count = (map->mmap_size - hdr->header_size) /
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            hdr->record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     "messages_count too large (%u > %u)",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     index->filepath, hdr->messages_count,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     records_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    mail_index_map_copy_hdr(map, hdr);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    map->hdr_base = map->mmap_base;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    map->records_count = map->hdr.messages_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mail_index_read_header(struct mail_index *index,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                  void *buf, size_t buf_size, size_t *pos_r)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    size_t pos;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    int ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    memset(buf, 0, sizeof(struct mail_index_header));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        /* try to read the whole header, but it's not necessarily an error to
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen       read less since the older versions of the index format could be
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen       smaller. Request reading up to buf_size, but accept if we only got
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen       the header. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        pos = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    do {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        ret = pread(index->fd, PTR_OFFSET(buf, pos),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                buf_size - pos, pos);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (ret > 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            pos += ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    } while (ret > 0 && pos < sizeof(struct mail_index_header));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    *pos_r = pos;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    return ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenmail_index_try_read_map(struct mail_index_map *map,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            uoff_t file_size, bool *retry_r, bool try_retry)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    const struct mail_index_header *hdr;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    unsigned char read_buf[4096];
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    const void *buf;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    void *data = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    ssize_t ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    size_t pos, records_size, initial_buf_pos = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    unsigned int records_count, extra;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    i_assert(map->mmap_base == NULL);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    *retry_r = FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    buf = read_buf; hdr = buf;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* major version change - handle silently */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        (ret > 0 || pos >= hdr->base_header_size)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (!mail_index_check_header_compat(index, hdr, file_size)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            /* Can't use this file */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        initial_buf_pos = pos;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (pos > hdr->header_size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            pos = hdr->header_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* place the base header into memory. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        buffer_reset(map->hdr_copy_buf);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        buffer_append(map->hdr_copy_buf, buf, pos);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (pos != hdr->header_size) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            /* @UNSAFE: read the rest of the header into memory */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            data = buffer_append_space_unsafe(map->hdr_copy_buf,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                              hdr->header_size -
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                              pos);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            ret = pread_full(index->fd, data,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                     hdr->header_size - pos, pos);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (ret > 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* header read, read the records now. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        records_size = (size_t)hdr->messages_count * hdr->record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (file_size - hdr->header_size < records_size ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            (hdr->record_size != 0 &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen             records_size / hdr->record_size != hdr->messages_count)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            records_count = (file_size - hdr->header_size) /
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                hdr->record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                "messages_count too large (%u > %u)",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                index->filepath, hdr->messages_count,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                records_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (map->buffer == NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            map->buffer = buffer_create_dynamic(default_pool,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                                records_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* @UNSAFE */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        buffer_set_used_size(map->buffer, 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (initial_buf_pos <= hdr->header_size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            extra = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            extra = initial_buf_pos - hdr->header_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            buffer_append(map->buffer,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                      CONST_PTR_OFFSET(buf, hdr->header_size),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                      extra);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (records_size > extra) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            data = buffer_append_space_unsafe(map->buffer,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                              records_size - extra);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            ret = pread_full(index->fd, data, records_size - extra,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                     hdr->header_size + extra);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (ret < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (errno == ESTALE && try_retry) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            /* a new index file was renamed over this one. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            *retry_r = TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mail_index_set_syscall_error(index, "pread_full()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (ret == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mail_index_set_error(index,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            "Corrupted index file %s: File too small",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            index->filepath);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    map->records = buffer_get_modifiable_data(map->buffer, NULL);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    map->records_count = hdr->messages_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    mail_index_map_copy_hdr(map, hdr);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    map->hdr_base = map->hdr_copy_buf->data;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenstatic int mail_index_read_map(struct mail_index_map *map, uoff_t file_size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    mail_index_sync_lost_handler_t *const *handlers;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    struct stat st;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    unsigned int i, count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    int ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    bool try_retry, retry;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    /* notify all "sync lost" handlers */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    handlers = array_get(&index->sync_lost_handlers, &count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    for (i = 0; i < count; i++)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        (*handlers[i])(index);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    for (i = 0;; i++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (file_size == (uoff_t)-1) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            /* fstat() below failed */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            ret = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            retry = try_retry;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        } else {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen            ret = mail_index_try_read_map(map, file_size,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                              &retry, try_retry);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (ret != 0 || !retry)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* ESTALE - reopen index file */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                if (close(index->fd) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            mail_index_set_syscall_error(index, "close()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        index->fd = -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                ret = mail_index_try_open_only(index);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        if (ret <= 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            if (ret == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                /* the file was lost */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                errno = ENOENT;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                mail_index_set_syscall_error(index, "open()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (fstat(index->fd, &st) == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            file_size = st.st_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            if (errno != ESTALE) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                mail_index_set_syscall_error(index, "fstat()");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            file_size = (uoff_t)-1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    return ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void mail_index_header_init(struct mail_index *index,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                   struct mail_index_header *hdr)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    memset(hdr, 0, sizeof(*hdr));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->base_header_size = sizeof(*hdr);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->header_size = sizeof(*hdr);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->record_size = sizeof(struct mail_index_record);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen#ifndef WORDS_BIGENDIAN
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen#endif
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->indexid = index->indexid;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->log_file_seq = 1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    hdr->next_uid = 1;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen    hdr->first_recent_uid = 1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstruct mail_index_map *mail_index_map_alloc(struct mail_index *index)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    struct mail_index_map tmp_map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    memset(&tmp_map, 0, sizeof(tmp_map));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    mail_index_header_init(index, &tmp_map.hdr);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    tmp_map.index = index;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    tmp_map.hdr_base = &tmp_map.hdr;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    /* a bit kludgy way to do this, but it initializes everything
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen       nicely and correctly */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    return mail_index_map_clone(&tmp_map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mail_index_map_latest_file(struct mail_index *index,
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen                      struct mail_index_map **map)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    struct mail_index_map *new_map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    struct stat st;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    unsigned int lock_id;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    uoff_t file_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    bool use_mmap;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    int ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    ret = mail_index_reopen_if_changed(index);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (ret <= 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (ret < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        /* the index file is lost/broken. let's hope that we can
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen           build it from the transaction log. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    /* the index file is still open, lock it */
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    if (mail_index_lock_shared(index, &lock_id) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen    if (index->nfs_flush)
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen        nfs_flush_attr_cache_fd(index->filepath, index->fd);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (fstat(index->fd, &st) == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        file_size = st.st_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (errno != ESTALE) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            mail_index_set_syscall_error(index, "fstat()");
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen            mail_index_unlock(index, &lock_id);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        file_size = (uoff_t)-1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    /* mmaping seems to be slower than just reading the file, so even if
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen       mmap isn't disabled don't use it unless the file is large enough */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    use_mmap = !index->mmap_disable && file_size != (uoff_t)-1 &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        file_size > MAIL_INDEX_MMAP_MIN_SIZE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    new_map = mail_index_map_alloc(index);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    if (use_mmap) {
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen        new_map->lock_id = lock_id;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen        ret = mail_index_mmap(new_map, file_size);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    } else {
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen        ret = mail_index_read_map(new_map, file_size);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen        mail_index_unlock(index, &lock_id);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (ret > 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        /* make sure the header is ok before using this mapping */
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen        ret = mail_index_check_header(new_map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (ret >= 0)
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen            ret = mail_index_parse_extensions(new_map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        if (ret++ == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            index->fsck = TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (ret <= 0) {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen        mail_index_unmap(&new_map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        return ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen    index->last_read_log_file_seq = new_map->hdr.log_file_seq;
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen    index->last_read_log_file_head_offset =
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen        new_map->hdr.log_file_head_offset;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen    index->last_read_log_file_tail_offset =
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen        new_map->hdr.log_file_tail_offset;
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen    index->last_read_stat = st;
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    mail_index_unmap(map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    *map = new_map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainenint mail_index_map(struct mail_index *index,
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen           enum mail_index_sync_handler_type type)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    int ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen    i_assert(index->lock_type != F_WRLCK);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    i_assert(!index->mapping);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    index->mapping = TRUE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen    if (index->map == NULL)
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen        index->map = mail_index_map_alloc(index);
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    /* first try updating the existing mapping from transaction log. */
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen    if (index->map->hdr.indexid != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        /* we're not creating the index, or opening transaction log.
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen           sync this as a view from transaction log. */
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen        ret = mail_index_sync_map(&index->map, type, FALSE);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    } else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        ret = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (ret == 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        /* try to open and read the latest index. if it fails for
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen           any reason, we'll fallback to updating the existing mapping
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen           from transaction logs (which we'll also do even if the
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen           reopening succeeds) */
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen        (void)mail_index_map_latest_file(index, &index->map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen        /* if we're creating the index file, we don't have any
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen           logs yet */
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen        if (index->log->head != NULL) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen            /* and update the map with the latest changes from
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen               transaction log */
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen            ret = mail_index_sync_map(&index->map, type, TRUE);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    index->mapping = FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    return ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenvoid mail_index_unmap(struct mail_index_map **_map)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    struct mail_index_map *map = *_map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    *_map = NULL;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (--map->refcount > 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        return;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    i_assert(map->refcount == 0);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    mail_index_map_clear(map);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    mail_index_map_unlock(map);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (map->extension_pool != NULL)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        pool_unref(map->extension_pool);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (array_is_created(&map->keyword_idx_map))
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        array_free(&map->keyword_idx_map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    buffer_free(map->hdr_copy_buf);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    i_free(map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainenint mail_index_map_lock(struct mail_index_map *map)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen{
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    if (map->lock_id != 0 || MAIL_INDEX_MAP_IS_IN_MEMORY(map))
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen        return 0;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen    if (mail_index_lock_shared(map->index, &map->lock_id) < 0)
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen        return -1;
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen    mail_index_map_copy_hdr(map, map->mmap_base);
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen    return 0;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen}
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainenvoid mail_index_map_unlock(struct mail_index_map *map)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen{
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    mail_index_unlock(map->index, &map->lock_id);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen}
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void mail_index_map_copy(struct mail_index_map *dest,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                const struct mail_index_map *src)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    size_t size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    /* copy records */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    size = src->records_count * src->hdr.record_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    dest->buffer = buffer_create_dynamic(default_pool, size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    buffer_append(dest->buffer, src->records, size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    dest->records = buffer_get_modifiable_data(dest->buffer, NULL);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    dest->records_count = src->records_count;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen    /* copy header. use src->hdr copy directly, because if we got here
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen       from syncing it has the latest changes. */
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen    dest->hdr = src->hdr;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (dest->hdr_copy_buf != NULL)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        buffer_set_used_size(dest->hdr_copy_buf, 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        dest->hdr_copy_buf =
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen            buffer_create_dynamic(default_pool,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                          dest->hdr.header_size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    }
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen    buffer_append(dest->hdr_copy_buf, &dest->hdr,
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen              I_MIN(sizeof(dest->hdr), src->hdr.base_header_size));
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen    buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size,
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen             CONST_PTR_OFFSET(src->hdr_base, src->hdr.base_header_size),
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen             src->hdr.header_size - src->hdr.base_header_size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstruct mail_index_map *mail_index_map_clone(const struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    struct mail_index_map *mem_map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    struct mail_index_ext *extensions;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    unsigned int i, count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    mem_map = i_new(struct mail_index_map, 1);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen    mem_map->index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    mem_map->refcount = 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    mail_index_map_copy(mem_map, map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    /* if the map is ever written back to disk, we need to keep track of
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen       what has changed. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (map->write_atomic)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        mem_map->write_atomic = TRUE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mem_map->write_seq_first = map->write_seq_first;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mem_map->write_seq_last = map->write_seq_last;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        mem_map->write_base_header = map->write_base_header;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        mem_map->write_ext_header = map->write_ext_header;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    /* copy extensions */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (array_is_created(&map->ext_id_map)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        count = array_count(&map->ext_id_map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        mail_index_map_init_extbufs(mem_map, count + 2);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        array_append_array(&mem_map->extensions, &map->extensions);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        /* fix the name pointers to use our own pool */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        extensions = array_get_modifiable(&mem_map->extensions, &count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        for (i = 0; i < count; i++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            i_assert(extensions[i].record_offset +
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                 extensions[i].record_size <=
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen                 mem_map->hdr.record_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen            extensions[i].name = p_strdup(mem_map->extension_pool,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                              extensions[i].name);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return mem_map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenvoid mail_index_map_move_to_memory(struct mail_index_map *map)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (map->mmap_base == NULL)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        return;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    i_assert(map->lock_id != 0);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    mail_index_map_copy(map, map);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen    mail_index_map_unlock(map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    if (munmap(map->mmap_base, map->mmap_size) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen        i_error("munmap(index map) failed: %m");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen    map->mmap_base = NULL;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenint mail_index_map_get_ext_idx(struct mail_index_map *map,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen                   uint32_t ext_id, uint32_t *idx_r)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    const uint32_t *id;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    if (!array_is_created(&map->ext_id_map) ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        ext_id >= array_count(&map->ext_id_map))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen        return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    id = array_idx(&map->ext_id_map, ext_id);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    *idx_r = *id;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen    return *idx_r != (uint32_t)-1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}