mail-index-map.c revision 8e57335924f5ff57cbd1929ec99764dc267c3312
2875N/A/* Copyright (C) 2003-2007 Timo Sirainen */
2875N/A
2875N/A#include "lib.h"
2875N/A#include "array.h"
2875N/A#include "nfs-workarounds.h"
2875N/A#include "mmap-util.h"
2875N/A#include "read-full.h"
2875N/A#include "mail-index-private.h"
2875N/A#include "mail-index-sync-private.h"
2875N/A
2875N/Astatic void mail_index_map_init_extbufs(struct mail_index_map *map,
2875N/A unsigned int initial_count)
2875N/A{
2875N/A#define EXTENSION_NAME_APPROX_LEN 20
2875N/A#define EXT_GLOBAL_ALLOC_SIZE \
2875N/A ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
2875N/A#define EXT_PER_ALLOC_SIZE \
2875N/A (EXTENSION_NAME_APPROX_LEN + \
2875N/A sizeof(struct mail_index_ext) + sizeof(uint32_t))
2875N/A size_t size;
2875N/A
2875N/A if (map->extension_pool == NULL) {
3014N/A size = EXT_GLOBAL_ALLOC_SIZE +
2875N/A initial_count * EXT_PER_ALLOC_SIZE;
2875N/A map->extension_pool =
2875N/A pool_alloconly_create("map extensions",
2875N/A nearest_power(size));
2875N/A } else {
2875N/A p_clear(map->extension_pool);
2875N/A
3025N/A /* try to use the existing pool's size for initial_count so
2875N/A we don't grow it unneededly */
2875N/A size = p_get_max_easy_alloc_size(map->extension_pool);
2875N/A if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
2875N/A initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
2875N/A EXT_PER_ALLOC_SIZE;
2875N/A }
2875N/A }
2875N/A
2875N/A p_array_init(&map->extensions, map->extension_pool, initial_count);
2875N/A p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
2875N/A}
2875N/A
2875N/Auint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
2875N/A{
2875N/A const struct mail_index_ext *extensions;
2875N/A unsigned int i, size;
3025N/A
2875N/A if (!array_is_created(&map->extensions))
2875N/A return (uint32_t)-1;
2875N/A
2875N/A extensions = array_get(&map->extensions, &size);
2875N/A for (i = 0; i < size; i++) {
2875N/A if (strcmp(extensions[i].name, name) == 0)
2875N/A return i;
2875N/A }
2875N/A return (uint32_t)-1;
2875N/A}
3025N/A
3025N/Auint32_t
3025N/Amail_index_map_register_ext(struct mail_index *index,
3025N/A struct mail_index_map *map, const char *name,
3025N/A uint32_t hdr_offset, uint32_t hdr_size,
2875N/A uint32_t record_offset, uint32_t record_size,
2875N/A uint32_t record_align, uint32_t reset_id)
2875N/A{
2875N/A struct mail_index_ext *ext;
2875N/A uint32_t idx, empty_idx = (uint32_t)-1;
2875N/A
2875N/A if (!array_is_created(&map->extensions)) {
2875N/A mail_index_map_init_extbufs(map, 5);
2875N/A idx = 0;
2875N/A } else {
2875N/A idx = array_count(&map->extensions);
2875N/A }
2875N/A i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);
2875N/A
2875N/A ext = array_append_space(&map->extensions);
2875N/A ext->name = p_strdup(map->extension_pool, name);
2875N/A ext->hdr_offset = hdr_offset;
2875N/A ext->hdr_size = hdr_size;
2875N/A ext->record_offset = record_offset;
2875N/A ext->record_size = record_size;
2875N/A ext->record_align = record_align;
2875N/A ext->reset_id = reset_id;
2875N/A
3014N/A ext->index_idx = mail_index_ext_register(index, name, hdr_size,
3014N/A record_size, record_align);
2875N/A
2875N/A /* Update index ext_id -> map ext_id mapping. Fill non-used
2875N/A ext_ids with (uint32_t)-1 */
2875N/A while (array_count(&map->ext_id_map) < ext->index_idx)
2875N/A array_append(&map->ext_id_map, &empty_idx, 1);
2875N/A array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
2875N/A return idx;
2875N/A}
2875N/A
2875N/Astatic bool size_check(size_t *size_left, size_t size)
2875N/A{
2875N/A if (size > *size_left)
2875N/A return FALSE;
2875N/A *size_left -= size;
2875N/A return TRUE;
2875N/A}
2875N/A
2875N/Astatic size_t get_align(size_t name_len)
2875N/A{
2875N/A size_t size = sizeof(struct mail_index_ext_header) + name_len;
2875N/A return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
2875N/A}
2875N/A
2875N/Astatic int mail_index_parse_extensions(struct mail_index *index,
2875N/A struct mail_index_map *map)
2875N/A{
2875N/A const struct mail_index_ext_header *ext_hdr;
2875N/A unsigned int i, old_count;
2875N/A const char *name;
2875N/A uint32_t ext_id, offset, name_offset;
2875N/A size_t size_left;
2875N/A
2875N/A /* extension headers always start from 64bit offsets, so if base header
2875N/A doesn't happen to be 64bit aligned we'll skip some bytes */
2875N/A offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
2875N/A if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
2875N/A /* nothing to do, skip allocatations and all */
2875N/A return 1;
2875N/A }
2875N/A
2875N/A old_count = array_count(&index->extensions);
2875N/A mail_index_map_init_extbufs(map, old_count + 5);
2875N/A
2875N/A ext_id = (uint32_t)-1;
2875N/A for (i = 0; i < old_count; i++)
2875N/A array_append(&map->ext_id_map, &ext_id, 1);
2875N/A
2875N/A for (i = 0; offset < map->hdr.header_size; i++) {
2875N/A ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
2875N/A
2875N/A /* Extension header contains:
2875N/A - struct mail_index_ext_header
2875N/A - name (not 0-terminated)
2875N/A - 64bit alignment padding
2875N/A - extension header contents
2875N/A - 64bit alignment padding
2875N/A */
2875N/A size_left = map->hdr.header_size - offset;
2875N/A if (!size_check(&size_left, sizeof(*ext_hdr)) ||
2875N/A !size_check(&size_left, ext_hdr->name_size) ||
2875N/A !size_check(&size_left, get_align(ext_hdr->name_size)) ||
2875N/A !size_check(&size_left, ext_hdr->hdr_size)) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "Header extension goes outside header",
2875N/A index->filepath);
2875N/A return -1;
2875N/A }
2875N/A
2875N/A offset += sizeof(*ext_hdr);
2875N/A name_offset = offset;
2875N/A offset += ext_hdr->name_size + get_align(ext_hdr->name_size);
2875N/A
2875N/A t_push();
2875N/A name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
2875N/A ext_hdr->name_size);
2875N/A
2875N/A if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "Duplicate header extension %s",
2875N/A index->filepath, name);
2875N/A t_pop();
2875N/A return -1;
2875N/A }
2875N/A
2875N/A if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
2875N/A ext_hdr->record_align == 0 || *name == '\0') {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "Broken header extension %s",
2875N/A index->filepath, *name == '\0' ?
2875N/A t_strdup_printf("#%d", i) : name);
2875N/A t_pop();
2875N/A return -1;
2875N/A }
2875N/A if (map->hdr.record_size <
2875N/A ext_hdr->record_offset + ext_hdr->record_size) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "Record field %s points outside record size "
2875N/A "(%u < %u+%u)", index->filepath, name,
2875N/A map->hdr.record_size,
2875N/A ext_hdr->record_offset, ext_hdr->record_size);
2875N/A t_pop();
2875N/A return -1;
2875N/A }
2875N/A
2875N/A if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
2875N/A (map->hdr.record_size % ext_hdr->record_align) != 0) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "Record field %s alignmentation %u not used",
2875N/A index->filepath, name, ext_hdr->record_align);
2875N/A t_pop();
2875N/A return -1;
2875N/A }
2875N/A
2875N/A mail_index_map_register_ext(index, map, name,
2875N/A offset, ext_hdr->hdr_size,
2875N/A ext_hdr->record_offset,
2875N/A ext_hdr->record_size,
2875N/A ext_hdr->record_align,
2875N/A ext_hdr->reset_id);
2875N/A t_pop();
2875N/A
2875N/A offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
2875N/A }
2875N/A return 1;
2875N/A}
2875N/A
2875N/Astatic bool mail_index_check_header_compat(struct mail_index *index,
2875N/A const struct mail_index_header *hdr,
2875N/A uoff_t file_size)
2875N/A{
2875N/A enum mail_index_header_compat_flags compat_flags = 0;
2875N/A
2875N/A#ifndef WORDS_BIGENDIAN
2875N/A compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
2875N/A#endif
2875N/A
2875N/A if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2875N/A /* major version change - handle silently(?) */
2875N/A return FALSE;
2875N/A }
2875N/A if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
2875N/A /* we've already complained about it */
2875N/A return FALSE;
2875N/A }
2875N/A
2875N/A if (hdr->compat_flags != compat_flags) {
2875N/A /* architecture change */
2875N/A mail_index_set_error(index, "Rebuilding index file %s: "
2875N/A "CPU architecture changed",
2875N/A index->filepath);
2875N/A return FALSE;
2875N/A }
2875N/A
2875N/A if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
2875N/A hdr->header_size < hdr->base_header_size) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "Corrupted header sizes (base %u, full %u)",
2875N/A index->filepath, hdr->base_header_size,
2875N/A hdr->header_size);
2875N/A return FALSE;
2875N/A }
2875N/A if (hdr->header_size > file_size) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "Corrupted header size (%u > %"PRIuUOFF_T")",
2875N/A index->filepath, hdr->header_size,
2875N/A file_size);
2875N/A return 0;
2875N/A }
2875N/A
2875N/A return TRUE;
2875N/A}
2875N/A
2875N/Astatic int mail_index_check_header(struct mail_index *index,
2875N/A struct mail_index_map *map)
2875N/A{
2875N/A const struct mail_index_header *hdr = &map->hdr;
2875N/A
2875N/A if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1))
2875N/A return -1;
2875N/A
2875N/A /* following some extra checks that only take a bit of CPU */
2875N/A if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "uid_validity = 0, next_uid = %u",
2875N/A index->filepath, hdr->next_uid);
2875N/A return -1;
2875N/A }
2875N/A
2875N/A if (hdr->record_size < sizeof(struct mail_index_record)) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "record_size too small: %u < %"PRIuSIZE_T,
2875N/A index->filepath, hdr->record_size,
2875N/A sizeof(struct mail_index_record));
2875N/A return -1;
2875N/A }
2875N/A
2875N/A if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
2875N/A return 0;
2875N/A
2875N/A if (hdr->next_uid == 0)
2875N/A return 0;
2875N/A
2875N/A if (hdr->recent_messages_count > hdr->messages_count ||
2875N/A hdr->seen_messages_count > hdr->messages_count ||
2875N/A hdr->deleted_messages_count > hdr->messages_count)
2875N/A return 0;
2875N/A if (hdr->first_recent_uid_lowwater > hdr->next_uid ||
2875N/A hdr->first_unseen_uid_lowwater > hdr->next_uid ||
2875N/A hdr->first_deleted_uid_lowwater > hdr->next_uid)
2875N/A return 0;
2875N/A
2875N/A if (map->records_count > 0) {
2875N/A /* last message's UID must be smaller than next_uid.
2875N/A also make sure it's not zero. */
2875N/A const struct mail_index_record *rec;
2875N/A
2875N/A rec = MAIL_INDEX_MAP_IDX(map, map->records_count-1);
2875N/A if (rec->uid == 0 || rec->uid >= hdr->next_uid)
2875N/A return 0;
3025N/A }
3025N/A
3025N/A return 1;
3025N/A}
3025N/A
3025N/Astatic void mail_index_map_clear(struct mail_index *index,
3025N/A struct mail_index_map *map)
3025N/A{
3025N/A if (map->buffer != NULL) {
3025N/A i_assert(map->mmap_base == NULL);
3025N/A buffer_free(map->buffer);
3025N/A map->buffer = NULL;
3025N/A } else if (map->mmap_base != NULL) {
3025N/A i_assert(map->buffer == NULL);
3025N/A if (munmap(map->mmap_base, map->mmap_size) < 0)
3025N/A mail_index_set_syscall_error(index, "munmap()");
3025N/A map->mmap_base = NULL;
3025N/A }
3025N/A
3025N/A if (map->refcount > 0) {
3025N/A memset(&map->hdr, 0, sizeof(map->hdr));
3025N/A map->mmap_size = 0;
3025N/A map->mmap_used_size = 0;
3025N/A map->records = NULL;
3025N/A map->records_count = 0;
3025N/A }
3025N/A}
3025N/A
3025N/Astatic void mail_index_map_copy_hdr(struct mail_index_map *map,
3025N/A const struct mail_index_header *hdr)
3025N/A{
3025N/A if (hdr->base_header_size < sizeof(map->hdr)) {
3025N/A /* header smaller than ours, make a copy so our newer headers
3025N/A won't have garbage in them */
3025N/A memset(&map->hdr, 0, sizeof(map->hdr));
3025N/A memcpy(&map->hdr, hdr, hdr->base_header_size);
3025N/A } else {
3025N/A map->hdr = *hdr;
3025N/A }
3025N/A}
3025N/A
3025N/Astatic int mail_index_mmap(struct mail_index *index, struct mail_index_map *map,
3025N/A uoff_t file_size)
3025N/A{
3025N/A const struct mail_index_header *hdr;
3025N/A unsigned int records_count;
3025N/A
3025N/A if (map->buffer != NULL) {
3025N/A /* we had temporarily used a buffer, eg. for updating index */
3025N/A buffer_free(map->buffer);
3025N/A map->buffer = NULL;
3025N/A }
3025N/A
3025N/A if (file_size > SSIZE_T_MAX) {
3025N/A /* too large file to map into memory */
3025N/A mail_index_set_error(index, "Index file too large: %s",
3025N/A index->filepath);
3025N/A return -1;
3025N/A }
3025N/A
3025N/A map->mmap_base =
3025N/A mmap(NULL, file_size, PROT_READ, MAP_SHARED, index->fd, 0);
3025N/A if (map->mmap_base == MAP_FAILED) {
2875N/A map->mmap_base = NULL;
2875N/A mail_index_set_syscall_error(index, "mmap()");
2875N/A return -1;
2875N/A }
2875N/A map->mmap_size = file_size;
2875N/A
2875N/A hdr = map->mmap_base;
2875N/A if (map->mmap_size >
2875N/A offsetof(struct mail_index_header, major_version) &&
2875N/A hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2875N/A /* major version change - handle silently */
2875N/A return 0;
2875N/A }
2875N/A
2875N/A if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "File too small (%"PRIuSIZE_T")",
2875N/A index->filepath, map->mmap_size);
2875N/A return 0;
2875N/A }
2875N/A
2875N/A if (!mail_index_check_header_compat(index, hdr, map->mmap_size)) {
2875N/A /* Can't use this file */
2875N/A return 0;
2875N/A }
2875N/A
2875N/A map->mmap_used_size = hdr->header_size +
2875N/A hdr->messages_count * hdr->record_size;
2875N/A
2875N/A if (map->mmap_used_size > map->mmap_size) {
2875N/A records_count = (map->mmap_size - hdr->header_size) /
2875N/A hdr->record_size;
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "messages_count too large (%u > %u)",
2875N/A index->filepath, hdr->messages_count,
2875N/A records_count);
2875N/A return 0;
2875N/A }
2875N/A
2875N/A mail_index_map_copy_hdr(map, hdr);
2875N/A
2875N/A map->hdr_base = map->mmap_base;
2875N/A map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
2875N/A map->records_count = map->hdr.messages_count;
2875N/A return 1;
2875N/A}
2875N/A
2875N/Astatic int mail_index_read_header(struct mail_index *index,
2875N/A void *buf, size_t buf_size, size_t *pos_r)
2875N/A{
2875N/A size_t pos;
2875N/A int ret;
2875N/A
2875N/A memset(buf, 0, sizeof(struct mail_index_header));
2875N/A
2875N/A /* try to read the whole header, but it's not necessarily an error to
2875N/A read less since the older versions of the index format could be
2875N/A smaller. Request reading up to buf_size, but accept if we only got
2875N/A the header. */
2875N/A pos = 0;
2875N/A do {
2875N/A ret = pread(index->fd, PTR_OFFSET(buf, pos),
2875N/A buf_size - pos, pos);
2875N/A if (ret > 0)
2875N/A pos += ret;
2875N/A } while (ret > 0 && pos < sizeof(struct mail_index_header));
2875N/A
2875N/A *pos_r = pos;
2875N/A return ret;
2875N/A}
2875N/A
2875N/Astatic int
2875N/Amail_index_try_read_map(struct mail_index *index, struct mail_index_map *map,
2875N/A uoff_t file_size, bool *retry_r, bool try_retry)
2875N/A{
2875N/A const struct mail_index_header *hdr;
2875N/A unsigned char read_buf[4096];
2875N/A const void *buf;
2875N/A void *data = NULL;
2875N/A ssize_t ret;
2875N/A size_t pos, records_size, initial_buf_pos = 0;
2875N/A unsigned int records_count, extra;
2875N/A
2875N/A i_assert(map->mmap_base == NULL);
2875N/A
2875N/A *retry_r = FALSE;
2875N/A ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
2875N/A buf = read_buf; hdr = buf;
2875N/A
2875N/A if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
2875N/A hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2875N/A /* major version change - handle silently */
2875N/A return 0;
2875N/A }
2875N/A
2875N/A if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
2875N/A (ret > 0 || pos >= hdr->base_header_size)) {
2875N/A if (!mail_index_check_header_compat(index, hdr, file_size)) {
2875N/A /* Can't use this file */
2875N/A return 0;
2875N/A }
2875N/A
2875N/A initial_buf_pos = pos;
2875N/A if (pos > hdr->header_size)
2875N/A pos = hdr->header_size;
2875N/A
2875N/A /* place the base header into memory. */
2875N/A buffer_reset(map->hdr_copy_buf);
2875N/A buffer_append(map->hdr_copy_buf, buf, pos);
2875N/A
2875N/A if (pos != hdr->header_size) {
2875N/A /* @UNSAFE: read the rest of the header into memory */
2875N/A data = buffer_append_space_unsafe(map->hdr_copy_buf,
2875N/A hdr->header_size -
2875N/A pos);
2875N/A ret = pread_full(index->fd, data,
2875N/A hdr->header_size - pos, pos);
2875N/A }
2875N/A }
2875N/A
2875N/A if (ret > 0) {
2875N/A /* header read, read the records now. */
2875N/A records_size = (size_t)hdr->messages_count * hdr->record_size;
2875N/A
2875N/A if (file_size - hdr->header_size < records_size ||
2875N/A (hdr->record_size != 0 &&
2875N/A records_size / hdr->record_size != hdr->messages_count)) {
2875N/A records_count = (file_size - hdr->header_size) /
2875N/A hdr->record_size;
2875N/A mail_index_set_error(index, "Corrupted index file %s: "
2875N/A "messages_count too large (%u > %u)",
2875N/A index->filepath, hdr->messages_count,
2875N/A records_count);
2875N/A return 0;
2875N/A }
2893N/A
2893N/A if (map->buffer == NULL) {
2893N/A map->buffer = buffer_create_dynamic(default_pool,
2893N/A records_size);
2875N/A }
2875N/A
2875N/A /* @UNSAFE */
2875N/A buffer_set_used_size(map->buffer, 0);
2875N/A if (initial_buf_pos <= hdr->header_size)
2875N/A extra = 0;
2875N/A else {
2875N/A extra = initial_buf_pos - hdr->header_size;
2875N/A buffer_append(map->buffer,
2875N/A CONST_PTR_OFFSET(buf, hdr->header_size),
2875N/A extra);
2875N/A }
2875N/A if (records_size > extra) {
2875N/A data = buffer_append_space_unsafe(map->buffer,
2875N/A records_size - extra);
2875N/A ret = pread_full(index->fd, data, records_size - extra,
2875N/A hdr->header_size + extra);
2875N/A }
2875N/A }
2875N/A
2875N/A if (ret < 0) {
2875N/A if (errno == ESTALE && try_retry) {
2875N/A /* a new index file was renamed over this one. */
2875N/A *retry_r = TRUE;
2875N/A return 0;
2875N/A }
2875N/A mail_index_set_syscall_error(index, "pread_full()");
2875N/A return -1;
2875N/A }
2875N/A if (ret == 0) {
2875N/A mail_index_set_error(index,
2875N/A "Corrupted index file %s: File too small",
2875N/A index->filepath);
2875N/A return 0;
2875N/A }
2875N/A
2875N/A map->records = buffer_get_modifiable_data(map->buffer, NULL);
2875N/A map->records_count = hdr->messages_count;
2875N/A
2875N/A mail_index_map_copy_hdr(map, hdr);
2875N/A map->hdr_base = map->hdr_copy_buf->data;
2875N/A return 1;
2875N/A}
2875N/A
2875N/Astatic int
2875N/Amail_index_read_map(struct mail_index *index, struct mail_index_map *map,
2896N/A uoff_t file_size)
2875N/A{
2875N/A mail_index_sync_lost_handler_t *const *handlers;
2875N/A struct stat st;
2875N/A unsigned int i, count;
3014N/A int ret;
3014N/A bool try_retry, retry;
3014N/A
3014N/A /* notify all "sync lost" handlers */
3014N/A handlers = array_get(&index->sync_lost_handlers, &count);
3014N/A for (i = 0; i < count; i++)
3014N/A (*handlers[i])(index);
3014N/A
3014N/A for (i = 0;; i++) {
3014N/A try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
3014N/A if (file_size == (uoff_t)-1) {
3014N/A /* fstat() below failed */
3014N/A ret = 0;
3014N/A retry = try_retry;
3014N/A } else {
2875N/A ret = mail_index_try_read_map(index, map, file_size,
2875N/A &retry, try_retry);
2875N/A }
2875N/A if (ret != 0 || !retry)
2875N/A break;
2875N/A
2875N/A /* ESTALE - reopen index file */
2875N/A if (close(index->fd) < 0)
2875N/A mail_index_set_syscall_error(index, "close()");
2875N/A index->fd = -1;
2875N/A
2875N/A ret = mail_index_try_open_only(index);
2875N/A if (ret <= 0) {
2875N/A if (ret == 0) {
2875N/A /* the file was lost */
2875N/A errno = ENOENT;
2875N/A mail_index_set_syscall_error(index, "open()");
2875N/A }
2875N/A return -1;
2875N/A }
2875N/A if (fstat(index->fd, &st) == 0)
2875N/A file_size = st.st_size;
2875N/A else {
2875N/A if (errno != ESTALE) {
2875N/A mail_index_set_syscall_error(index, "fstat()");
2875N/A return -1;
2875N/A }
2875N/A file_size = (uoff_t)-1;
2875N/A }
2875N/A }
2875N/A return ret;
2875N/A}
2875N/A
2875N/Astatic void mail_index_header_init(struct mail_index *index,
2875N/A struct mail_index_header *hdr)
2875N/A{
2875N/A i_assert(index->indexid != 0);
2875N/A i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
2875N/A
2875N/A memset(hdr, 0, sizeof(*hdr));
2875N/A
2875N/A hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
2875N/A hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
2875N/A hdr->base_header_size = sizeof(*hdr);
2875N/A hdr->header_size = sizeof(*hdr);
2875N/A hdr->record_size = sizeof(struct mail_index_record);
2875N/A
2875N/A#ifndef WORDS_BIGENDIAN
2875N/A hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
2875N/A#endif
2875N/A
2875N/A hdr->indexid = index->indexid;
2875N/A hdr->log_file_seq = 1;
2875N/A hdr->next_uid = 1;
2875N/A}
2875N/A
2875N/Astruct mail_index_map *mail_index_map_alloc(struct mail_index *index)
2875N/A{
2875N/A struct mail_index_map tmp_map;
2875N/A
2875N/A memset(&tmp_map, 0, sizeof(tmp_map));
2875N/A mail_index_header_init(index, &tmp_map.hdr);
2875N/A tmp_map.hdr_base = &tmp_map.hdr;
2875N/A
2875N/A /* a bit kludgy way to do this, but it initializes everything
2875N/A nicely and correctly */
2875N/A return mail_index_map_clone(&tmp_map);
2875N/A}
3025N/A
3025N/Astatic int mail_index_map_latest_file(struct mail_index *index,
3025N/A struct mail_index_map **map,
3025N/A unsigned int *lock_id_r)
3025N/A{
3025N/A struct mail_index_map *new_map;
3025N/A struct stat st;
2875N/A uoff_t file_size;
2875N/A bool use_mmap;
2875N/A int ret;
2875N/A
2875N/A ret = mail_index_reopen_if_changed(index);
2875N/A if (ret <= 0) {
2875N/A if (ret < 0)
2875N/A return -1;
2875N/A
2875N/A /* the index file is lost/broken. let's hope that we can
2875N/A build it from the transaction log. */
2875N/A return 0;
2875N/A }
2875N/A
2875N/A /* the index file is still open, lock it */
2875N/A if (mail_index_lock_shared(index, lock_id_r) < 0)
2875N/A return -1;
2875N/A
2875N/A if (fstat(index->fd, &st) == 0)
2875N/A file_size = st.st_size;
2875N/A else {
2875N/A if (errno != ESTALE) {
2875N/A mail_index_set_syscall_error(index, "fstat()");
2875N/A return -1;
2875N/A }
2875N/A file_size = (uoff_t)-1;
2875N/A }
2875N/A
2875N/A /* mmaping seems to be slower than just reading the file, so even if
2875N/A mmap isn't disabled don't use it unless the file is large enough */
2875N/A use_mmap = !index->mmap_disable && file_size != (uoff_t)-1 &&
2875N/A file_size > MAIL_INDEX_MMAP_MIN_SIZE;
2875N/A
2875N/A new_map = mail_index_map_alloc(index);
2875N/A ret = use_mmap ? mail_index_mmap(index, new_map, file_size) :
2875N/A mail_index_read_map(index, new_map, file_size);
2875N/A if (ret > 0) {
2875N/A /* make sure the header is ok before using this mapping */
2875N/A ret = mail_index_check_header(index, new_map);
2875N/A if (ret >= 0)
2875N/A ret = mail_index_parse_extensions(index, new_map);
2875N/A if (ret++ == 0)
2875N/A index->fsck = TRUE;
2875N/A }
2875N/A if (ret <= 0) {
2875N/A mail_index_unmap(index, &new_map);
2875N/A return ret;
2875N/A }
2875N/A
2875N/A index->last_read_log_file_seq = new_map->hdr.log_file_seq;
2875N/A index->last_read_log_file_head_offset =
2875N/A new_map->hdr.log_file_head_offset;
2875N/A index->last_read_log_file_tail_offset =
2875N/A new_map->hdr.log_file_tail_offset;
2875N/A index->last_read_stat = st;
3025N/A
3025N/A mail_index_unmap(index, map);
3025N/A *map = new_map;
3025N/A return 1;
3014N/A}
3014N/A
3014N/Aint mail_index_map(struct mail_index *index,
3014N/A enum mail_index_sync_handler_type type,
3014N/A unsigned int *lock_id_r)
3014N/A{
2875N/A unsigned int lock_id = 0;
2875N/A int ret;
2875N/A
2875N/A i_assert(index->lock_type != F_WRLCK);
3025N/A i_assert(!index->mapping);
3014N/A
2875N/A *lock_id_r = 0;
2875N/A index->mapping = TRUE;
2875N/A
2875N/A if (index->map == NULL)
2875N/A index->map = mail_index_map_alloc(index);
2875N/A
2875N/A /* first try updating the existing mapping from transaction log. */
2875N/A if (index->map->hdr.indexid != 0) {
2875N/A /* we're not creating the index, or opening transaction log.
2875N/A sync this as a view from transaction log. */
2875N/A ret = mail_index_sync_map(index, &index->map, type, FALSE);
2875N/A } else {
2875N/A ret = 0;
3014N/A }
3014N/A
3025N/A if (ret == 0) {
3025N/A /* try to open and read the latest index. if it fails for
3014N/A any reason, we'll fallback to updating the existing mapping
3014N/A from transaction logs (which we'll also do even if the
3014N/A reopening succeeds) */
3014N/A (void)mail_index_map_latest_file(index, &index->map, &lock_id);
2875N/A
3014N/A /* and update the map with the latest changes from
2875N/A transaction log */
3025N/A ret = mail_index_sync_map(index, &index->map, type, TRUE);
2875N/A
3014N/A /* we need the lock only if we didn't move the map to memory */
3025N/A if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map))
2875N/A *lock_id_r = lock_id;
2875N/A else
3014N/A mail_index_unlock(index, lock_id);
2875N/A }
3014N/A
2875N/A index->mapping = FALSE;
2875N/A return ret;
2875N/A}
2875N/A
2875N/Avoid mail_index_unmap(struct mail_index *index, struct mail_index_map **_map)
2875N/A{
2875N/A struct mail_index_map *map = *_map;
2875N/A
2875N/A *_map = NULL;
2875N/A if (--map->refcount > 0)
2875N/A return;
2875N/A
2875N/A i_assert(map->refcount == 0);
2875N/A mail_index_map_clear(index, map);
2875N/A if (map->extension_pool != NULL)
2875N/A pool_unref(map->extension_pool);
2875N/A if (array_is_created(&map->keyword_idx_map))
2875N/A array_free(&map->keyword_idx_map);
2875N/A buffer_free(map->hdr_copy_buf);
2875N/A i_free(map);
2875N/A}
2875N/A
2875N/Astatic void mail_index_map_copy(struct mail_index_map *dest,
2875N/A const struct mail_index_map *src)
2875N/A{
2875N/A const struct mail_index_header *src_hdr;
2875N/A size_t size;
2875N/A
2875N/A src_hdr = src->mmap_base != NULL ? src->mmap_base : src->hdr_base;
3014N/A
2875N/A /* copy records */
3014N/A size = src->records_count * src->hdr.record_size;
2875N/A dest->buffer = buffer_create_dynamic(default_pool, size);
2875N/A buffer_append(dest->buffer, src->records, size);
2875N/A
2875N/A dest->records = buffer_get_modifiable_data(dest->buffer, NULL);
2875N/A dest->records_count = src->records_count;
3014N/A
2875N/A if (src->mmap_base == NULL)
3014N/A dest->hdr = src->hdr;
2875N/A else {
2875N/A /* refresh the header */
2875N/A memcpy(&dest->hdr, src_hdr, src->hdr.base_header_size);
2875N/A /* fix base header size if needed */
2875N/A if (dest->hdr.base_header_size < sizeof(dest->hdr)) {
2875N/A dest->hdr.base_header_size = sizeof(dest->hdr);
2875N/A dest->hdr.header_size +=
2875N/A sizeof(dest->hdr) - dest->hdr.base_header_size;
2875N/A }
2875N/A }
2925N/A
2875N/A /* copy header */
2875N/A if (dest->hdr_copy_buf != NULL)
2875N/A buffer_set_used_size(dest->hdr_copy_buf, 0);
3014N/A else {
3025N/A dest->hdr_copy_buf =
3025N/A buffer_create_dynamic(default_pool,
3025N/A dest->hdr.header_size);
3025N/A }
2875N/A buffer_append(dest->hdr_copy_buf, &dest->hdr, sizeof(dest->hdr));
2875N/A buffer_append(dest->hdr_copy_buf,
2875N/A CONST_PTR_OFFSET(src_hdr, src_hdr->base_header_size),
2875N/A src_hdr->header_size - src_hdr->base_header_size);
2875N/A dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
2875N/A}
2875N/A
2875N/Astruct mail_index_map *mail_index_map_clone(const struct mail_index_map *map)
2875N/A{
2875N/A struct mail_index_map *mem_map;
2925N/A struct mail_index_ext *extensions;
2925N/A unsigned int i, count;
2875N/A
2875N/A mem_map = i_new(struct mail_index_map, 1);
2875N/A mem_map->refcount = 1;
2875N/A
2875N/A mail_index_map_copy(mem_map, map);
2875N/A
2875N/A /* if the map is ever written back to disk, we need to keep track of
2875N/A what has changed. */
2875N/A if (map->write_atomic)
2875N/A mem_map->write_atomic = TRUE;
2875N/A else {
2875N/A mem_map->write_seq_first = map->write_seq_first;
2875N/A mem_map->write_seq_last = map->write_seq_last;
2875N/A mem_map->write_base_header = map->write_base_header;
2875N/A mem_map->write_ext_header = map->write_ext_header;
2875N/A }
2875N/A
2875N/A /* copy extensions */
2875N/A if (array_is_created(&map->ext_id_map)) {
2893N/A count = array_count(&map->ext_id_map);
2875N/A mail_index_map_init_extbufs(mem_map, count + 2);
2875N/A
2875N/A array_append_array(&mem_map->extensions, &map->extensions);
2875N/A array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
2875N/A
2875N/A /* fix the name pointers to use our own pool */
2875N/A extensions = array_get_modifiable(&mem_map->extensions, &count);
2875N/A for (i = 0; i < count; i++) {
2875N/A i_assert(extensions[i].record_offset +
2875N/A extensions[i].record_size <=
2875N/A mem_map->hdr.record_size);
2875N/A extensions[i].name = p_strdup(mem_map->extension_pool,
2875N/A extensions[i].name);
2875N/A }
2875N/A }
2875N/A
2875N/A return mem_map;
2875N/A}
2875N/A
2875N/Avoid mail_index_map_move_to_memory(struct mail_index_map *map)
2875N/A{
2875N/A if (map->mmap_base == NULL)
2875N/A return;
2875N/A
2875N/A mail_index_map_copy(map, map);
2875N/A
2875N/A if (munmap(map->mmap_base, map->mmap_size) < 0)
2875N/A i_error("munmap(index map) failed: %m");
2875N/A map->mmap_base = NULL;
2875N/A}
2875N/A
2875N/Aint mail_index_map_get_ext_idx(struct mail_index_map *map,
2875N/A uint32_t ext_id, uint32_t *idx_r)
2875N/A{
2875N/A const uint32_t *id;
2875N/A
2875N/A if (!array_is_created(&map->ext_id_map) ||
2875N/A ext_id >= array_count(&map->ext_id_map))
2875N/A return 0;
2875N/A
2875N/A id = array_idx(&map->ext_id_map, ext_id);
2875N/A *idx_r = *id;
2875N/A return *idx_r != (uint32_t)-1;
2875N/A}
2875N/A