mail-index-map.c revision 3e559edb905bb57d1f1fbfda3431b339f45ea791
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync/* Copyright (c) 2003-2007 Dovecot authors, see the included COPYING file */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "lib.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "array.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "str-sanitize.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "nfs-workarounds.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "mmap-util.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "read-full.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "mail-index-private.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "mail-index-sync-private.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#include "mail-transaction-log-private.h"
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int initial_count)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#define EXTENSION_NAME_APPROX_LEN 20
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#define EXT_GLOBAL_ALLOC_SIZE \
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#define EXT_PER_ALLOC_SIZE \
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync (EXTENSION_NAME_APPROX_LEN + \
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync sizeof(struct mail_index_ext) + sizeof(uint32_t))
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync size_t size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (map->extension_pool == NULL) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync size = EXT_GLOBAL_ALLOC_SIZE +
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync initial_count * EXT_PER_ALLOC_SIZE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync map->extension_pool =
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync pool_alloconly_create("map extensions",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync nearest_power(size));
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync } else {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync p_clear(map->extension_pool);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* try to use the existing pool's size for initial_count so
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync we don't grow it unneededly */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync size = p_get_max_easy_alloc_size(map->extension_pool);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync EXT_PER_ALLOC_SIZE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync p_array_init(&map->extensions, map->extension_pool, initial_count);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncbool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uint32_t *idx_r)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_ext *extensions;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int i, size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (array_is_created(&map->extensions)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync extensions = array_get(&map->extensions, &size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync for (i = 0; i < size; i++) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (strcmp(extensions[i].name, name) == 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (idx_r != NULL)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *idx_r = i;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return TRUE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return FALSE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncunsigned int mail_index_map_ext_hdr_offset(unsigned int name_len)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync size_t size = sizeof(struct mail_index_ext_header) + name_len;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return MAIL_INDEX_HEADER_SIZE_ALIGN(size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncuint32_t
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncmail_index_map_register_ext(struct mail_index_map *map, const char *name,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uint32_t ext_offset, uint32_t hdr_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uint32_t record_offset, uint32_t record_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uint32_t record_align, uint32_t reset_id)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync struct mail_index_ext *ext;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uint32_t idx, empty_idx = (uint32_t)-1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (!array_is_created(&map->extensions)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_map_init_extbufs(map, 5);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync idx = 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync } else {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync idx = array_count(&map->extensions);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync i_assert(!mail_index_map_lookup_ext(map, name, NULL));
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext = array_append_space(&map->extensions);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->name = p_strdup(map->extension_pool, name);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->ext_offset = ext_offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->hdr_offset = ext_offset +
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_map_ext_hdr_offset(strlen(name));
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->hdr_size = hdr_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->record_offset = record_offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->record_size = record_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->record_align = record_align;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->reset_id = reset_id;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext->index_idx = mail_index_ext_register(map->index, name, hdr_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync record_size, record_align);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* Update index ext_id -> map ext_id mapping. Fill non-used
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_ids with (uint32_t)-1 */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync while (array_count(&map->ext_id_map) < ext->index_idx)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync array_append(&map->ext_id_map, &empty_idx, 1);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return idx;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncint mail_index_map_ext_get_next(struct mail_index_map *map,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int *offset_p,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_ext_header **ext_hdr_r,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const char **name_r)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_ext_header *ext_hdr;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int offset, name_offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync offset = *offset_p;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *name_r = "";
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* Extension header contains:
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync - struct mail_index_ext_header
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync - name (not 0-terminated)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync - 64bit alignment padding
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync - extension header contents
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync - 64bit alignment padding
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync name_offset = offset + sizeof(*ext_hdr);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (offset + sizeof(*ext_hdr) >= map->hdr.header_size)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync offset += mail_index_map_ext_hdr_offset(ext_hdr->name_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (offset > map->hdr.header_size)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *name_r = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr->name_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (strcmp(*name_r, str_sanitize(*name_r, -1)) != 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* we allow only plain ASCII names, so this extension
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync is most likely broken */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *name_r = "";
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* finally make sure that the hdr_size is small enough.
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync do this last so that we could return a usable name. */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (offset > map->hdr.header_size)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *offset_p = offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *ext_hdr_r = ext_hdr;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncint mail_index_map_ext_hdr_check(const struct mail_index_header *hdr,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_ext_header *ext_hdr,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const char *name, const char **error_r)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync (ext_hdr->record_align == 0 && ext_hdr->record_size != 0)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *error_r = "Invalid field values";
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (*name == '\0') {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *error_r = "Broken name";
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (ext_hdr->record_offset + ext_hdr->record_size > hdr->record_size) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *error_r = t_strdup_printf("Record field points "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "outside record size (%u+%u > %u)",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr->record_offset,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr->record_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->record_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (ext_hdr->record_size > 0 &&
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync (hdr->record_size % ext_hdr->record_align) != 0)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *error_r = t_strdup_printf("Record field alignmentation %u "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "not used", ext_hdr->record_align);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncstatic int mail_index_map_parse_extensions(struct mail_index_map *map)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync struct mail_index *index = map->index;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_ext_header *ext_hdr;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int i, old_count, offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const char *name, *error;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uint32_t ext_id, ext_offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* extension headers always start from 64bit offsets, so if base header
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync doesn't happen to be 64bit aligned we'll skip some bytes */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* nothing to do, skip allocatations and all */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync old_count = array_count(&index->extensions);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_map_init_extbufs(map, old_count + 5);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_id = (uint32_t)-1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync for (i = 0; i < old_count; i++)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync array_append(&map->ext_id_map, &ext_id, 1);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync for (i = 0; offset < map->hdr.header_size; i++) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_offset = offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync t_push();
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (mail_index_map_ext_get_next(map, &offset,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync &ext_hdr, &name) < 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Header extension #%d (%s) goes outside header",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, i, name);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync t_pop();
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (mail_index_map_ext_hdr_check(&map->hdr, ext_hdr,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync name, &error) < 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Broken extension #%d (%s): %s",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, i, name, error);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync t_pop();
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (mail_index_map_lookup_ext(map, name, NULL)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Duplicate header extension %s",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, name);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync t_pop();
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_map_register_ext(map, name, ext_offset,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr->hdr_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr->record_offset,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr->record_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr->record_align,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext_hdr->reset_id);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync t_pop();
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncint mail_index_map_parse_keywords(struct mail_index_map *map)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync struct mail_index *index = map->index;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_ext *ext;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_keyword_header *kw_hdr;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_keyword_header_rec *kw_rec;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const char *name;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int i, name_area_end_offset, old_count;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uint32_t idx;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (!mail_index_map_lookup_ext(map, "keywords", &idx)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (array_is_created(&map->keyword_idx_map))
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync array_clear(&map->keyword_idx_map);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ext = array_idx(&map->extensions, idx);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* Extension header contains:
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync - struct mail_index_keyword_header
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync - struct mail_index_keyword_header_rec * keywords_count
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync - const char names[] * keywords_count
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync i_assert(ext->hdr_offset < map->hdr.header_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync kw_rec = (const void *)(kw_hdr + 1);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync name = (const char *)(kw_rec + kw_hdr->keywords_count);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync array_count(&map->keyword_idx_map);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* Keywords can only be added into same mapping. Removing requires a
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync new mapping (recreating the index file) */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (kw_hdr->keywords_count == old_count) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* nothing changed */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* make sure the header is valid */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (kw_hdr->keywords_count < old_count) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Keywords removed unexpectedly",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "keywords_count larger than header size",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync for (i = 0; i < kw_hdr->keywords_count; i++) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (kw_rec[i].name_offset > name_area_end_offset) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "name_offset points outside allocated header",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (name[name_area_end_offset-1] != '\0') {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Keyword header doesn't end with NUL",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* create file -> index mapping */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (!array_is_created(&map->keyword_idx_map))
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#ifdef DEBUG
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* Check that existing headers are still the same. It's behind DEBUG
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync since it's pretty useless waste of CPU normally. */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const char *keyword = name + kw_rec[i].name_offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const unsigned int *old_idx;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int idx;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync old_idx = array_idx(&map->keyword_idx_map, i);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (!mail_index_keyword_lookup(index, keyword, &idx) ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync idx != *old_idx) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Keywords changed unexpectedly",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#endif
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* Register the newly seen keywords */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync i = array_count(&map->keyword_idx_map);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync for (; i < kw_hdr->keywords_count; i++) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const char *keyword = name + kw_rec[i].name_offset;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int idx;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (*keyword == '\0') {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Empty keyword name in header",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_keyword_lookup_or_create(index, keyword, &idx);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync array_append(&map->keyword_idx_map, &idx, 1);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncstatic bool mail_index_check_header_compat(struct mail_index *index,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_header *hdr,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uoff_t file_size)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync enum mail_index_header_compat_flags compat_flags = 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#ifndef WORDS_BIGENDIAN
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync#endif
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* major version change - handle silently(?) */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return FALSE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* we've already complained about it */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return FALSE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->compat_flags != compat_flags) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* architecture change */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Rebuilding index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "CPU architecture changed",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return FALSE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->header_size < hdr->base_header_size) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Corrupted header sizes (base %u, full %u)",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, hdr->base_header_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->header_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return FALSE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->header_size > file_size) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "Corrupted header size (%u > %"PRIuUOFF_T")",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, hdr->header_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync file_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return FALSE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->indexid != index->indexid) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (index->indexid != 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "indexid changed: %u -> %u",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, index->indexid,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->indexid);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->indexid = hdr->indexid;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_transaction_log_indexid_changed(index->log);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return TRUE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncint mail_index_map_check_header(struct mail_index_map *map)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync struct mail_index *index = map->index;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_header *hdr = &map->hdr;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1))
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* following some extra checks that only take a bit of CPU */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->record_size < sizeof(struct mail_index_record)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "record_size too small: %u < %"PRIuSIZE_T,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, hdr->record_size,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync sizeof(struct mail_index_record));
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->uid_validity == 0 && hdr->next_uid != 1)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->next_uid == 0)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->messages_count > map->rec_map->records_count)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->seen_messages_count > hdr->messages_count ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->deleted_messages_count > hdr->messages_count)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->first_recent_uid == 0 && hdr->minor_version == 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* upgrade silently from v1.0 */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync map->hdr.first_recent_uid = 1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->first_recent_uid == 0 ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->first_recent_uid > hdr->next_uid ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->first_unseen_uid_lowwater > hdr->next_uid ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->first_deleted_uid_lowwater > hdr->next_uid)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->messages_count > 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* last message's UID must be smaller than next_uid.
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync also make sure it's not zero. */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_record *rec;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec = MAIL_INDEX_MAP_IDX(map, hdr->messages_count-1);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (rec->uid == 0 || rec->uid >= hdr->next_uid)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncstatic void mail_index_map_copy_hdr(struct mail_index_map *map,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_header *hdr)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (hdr->base_header_size < sizeof(map->hdr)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* header smaller than ours, make a copy so our newer headers
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync won't have garbage in them */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync memset(&map->hdr, 0, sizeof(map->hdr));
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync memcpy(&map->hdr, hdr, hdr->base_header_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync } else {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync map->hdr = *hdr;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* FIXME: backwards compatibility, remove later. In case this index is
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync accessed with Dovecot v1.0, avoid recent message counter errors. */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync map->hdr.unused_old_recent_messages_count = 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncstatic int mail_index_mmap(struct mail_index_map *map, uoff_t file_size)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync struct mail_index *index = map->index;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync struct mail_index_record_map *rec_map = map->rec_map;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_header *hdr;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync i_assert(rec_map->mmap_base == NULL);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync buffer_free(&rec_map->buffer);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (file_size > SSIZE_T_MAX) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* too large file to map into memory */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Index file too large: %s",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync MAP_PRIVATE, index->fd, 0);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (rec_map->mmap_base == MAP_FAILED) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->mmap_base = NULL;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_syscall_error(index, "mmap()");
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->mmap_size = file_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr = rec_map->mmap_base;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (rec_map->mmap_size >
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync offsetof(struct mail_index_header, major_version) &&
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* major version change - handle silently */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (rec_map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "File too small (%"PRIuSIZE_T")",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, rec_map->mmap_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* Can't use this file */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->mmap_used_size = hdr->header_size +
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->messages_count * hdr->record_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (rec_map->mmap_used_size <= rec_map->mmap_size)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->records_count = hdr->messages_count;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync else {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->records_count =
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync (rec_map->mmap_size - hdr->header_size) /
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->record_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->mmap_used_size = hdr->header_size +
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->records_count * hdr->record_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "messages_count too large (%u > %u)",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, hdr->messages_count,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->records_count);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_map_copy_hdr(map, hdr);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync map->hdr_base = rec_map->mmap_base;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync rec_map->records = PTR_OFFSET(rec_map->mmap_base, map->hdr.header_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncstatic int mail_index_read_header(struct mail_index *index,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync void *buf, size_t buf_size, size_t *pos_r)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync size_t pos;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync int ret;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync memset(buf, 0, sizeof(struct mail_index_header));
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* try to read the whole header, but it's not necessarily an error to
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync read less since the older versions of the index format could be
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync smaller. Request reading up to buf_size, but accept if we only got
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync the header. */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync pos = 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync do {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ret = pread(index->fd, PTR_OFFSET(buf, pos),
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync buf_size - pos, pos);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (ret > 0)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync pos += ret;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync } while (ret > 0 && pos < sizeof(struct mail_index_header));
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *pos_r = pos;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return ret;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync}
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncstatic int
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsyncmail_index_try_read_map(struct mail_index_map *map,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync uoff_t file_size, bool *retry_r, bool try_retry)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync{
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync struct mail_index *index = map->index;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const struct mail_index_header *hdr;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned char read_buf[4096];
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync const void *buf;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync void *data = NULL;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ssize_t ret;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync size_t pos, records_size, initial_buf_pos = 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync unsigned int records_count = 0, extra;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync i_assert(map->rec_map->mmap_base == NULL);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *retry_r = FALSE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync buf = read_buf; hdr = buf;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* major version change - handle silently */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync (ret > 0 || pos >= hdr->base_header_size)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (!mail_index_check_header_compat(index, hdr, file_size)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* Can't use this file */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync initial_buf_pos = pos;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (pos > hdr->header_size)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync pos = hdr->header_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* place the base header into memory. */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync buffer_reset(map->hdr_copy_buf);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync buffer_append(map->hdr_copy_buf, buf, pos);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (pos != hdr->header_size) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* @UNSAFE: read the rest of the header into memory */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync data = buffer_append_space_unsafe(map->hdr_copy_buf,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->header_size -
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync pos);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ret = pread_full(index->fd, data,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->header_size - pos, pos);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (ret > 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* header read, read the records now. */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync records_size = (size_t)hdr->messages_count * hdr->record_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync records_count = hdr->messages_count;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (file_size - hdr->header_size < records_size ||
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync (hdr->record_size != 0 &&
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync records_size / hdr->record_size != hdr->messages_count)) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync records_count = (file_size - hdr->header_size) /
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->record_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync records_size = (size_t)records_count * hdr->record_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index, "Corrupted index file %s: "
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync "messages_count too large (%u > %u)",
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync index->filepath, hdr->messages_count,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync records_count);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (map->rec_map->buffer == NULL) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync map->rec_map->buffer =
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync buffer_create_dynamic(default_pool,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync records_size);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* @UNSAFE */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync buffer_set_used_size(map->rec_map->buffer, 0);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (initial_buf_pos <= hdr->header_size)
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync extra = 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync else {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync extra = initial_buf_pos - hdr->header_size;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync buffer_append(map->rec_map->buffer,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync CONST_PTR_OFFSET(buf, hdr->header_size),
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync extra);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (records_size > extra) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync data = buffer_append_space_unsafe(map->rec_map->buffer,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync records_size - extra);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync ret = pread_full(index->fd, data, records_size - extra,
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync hdr->header_size + extra);
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (ret < 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (errno == ESTALE && try_retry) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync /* a new index file was renamed over this one. */
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync *retry_r = TRUE;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return 0;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_syscall_error(index, "pread_full()");
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync return -1;
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync }
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync if (ret == 0) {
a734c64bff58bda2fa48c2795453e092167b0ff7vboxsync mail_index_set_error(index,
"Corrupted index file %s: File too small",
index->filepath);
return 0;
}
map->rec_map->records =
buffer_get_modifiable_data(map->rec_map->buffer, NULL);
map->rec_map->records_count = records_count;
mail_index_map_copy_hdr(map, hdr);
map->hdr_base = map->hdr_copy_buf->data;
return 1;
}
static int mail_index_read_map(struct mail_index_map *map, uoff_t file_size,
unsigned int *lock_id)
{
struct mail_index *index = map->index;
mail_index_sync_lost_handler_t *const *handlers;
struct stat st;
unsigned int i, count;
int ret;
bool try_retry, retry;
/* notify all "sync lost" handlers */
handlers = array_get(&index->sync_lost_handlers, &count);
for (i = 0; i < count; i++)
(*handlers[i])(index);
for (i = 0;; i++) {
try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
if (file_size == (uoff_t)-1) {
/* fstat() below failed */
ret = 0;
retry = try_retry;
} else {
ret = mail_index_try_read_map(map, file_size,
&retry, try_retry);
}
if (ret != 0 || !retry)
break;
/* ESTALE - reopen index file */
mail_index_close_file(index);
*lock_id = 0;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
if (mail_index_lock_shared(index, lock_id) < 0)
return -1;
if (fstat(index->fd, &st) == 0)
file_size = st.st_size;
else {
if (errno != ESTALE) {
mail_index_set_syscall_error(index, "fstat()");
return -1;
}
file_size = (uoff_t)-1;
}
}
return ret;
}
static void mail_index_header_init(struct mail_index *index,
struct mail_index_header *hdr)
{
i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
memset(hdr, 0, sizeof(*hdr));
hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
hdr->base_header_size = sizeof(*hdr);
hdr->header_size = sizeof(*hdr);
hdr->record_size = sizeof(struct mail_index_record);
#ifndef WORDS_BIGENDIAN
hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
hdr->indexid = index->indexid;
hdr->log_file_seq = 1;
hdr->next_uid = 1;
hdr->first_recent_uid = 1;
}
struct mail_index_map *mail_index_map_alloc(struct mail_index *index)
{
struct mail_index_map tmp_map;
memset(&tmp_map, 0, sizeof(tmp_map));
mail_index_header_init(index, &tmp_map.hdr);
tmp_map.index = index;
tmp_map.hdr_base = &tmp_map.hdr;
/* a bit kludgy way to do this, but it initializes everything
nicely and correctly */
return mail_index_map_clone(&tmp_map);
}
static int mail_index_map_latest_file(struct mail_index *index)
{
struct mail_index_map *old_map, *new_map;
struct stat st;
unsigned int lock_id;
uoff_t file_size;
bool use_mmap;
int ret, try;
ret = mail_index_reopen_if_changed(index);
if (ret <= 0) {
if (ret < 0)
return -1;
/* the index file is lost/broken. let's hope that we can
build it from the transaction log. */
return 0;
}
/* the index file is still open, lock it */
if (mail_index_lock_shared(index, &lock_id) < 0)
return -1;
if (index->nfs_flush)
nfs_flush_attr_cache_fd(index->filepath, index->fd);
if (fstat(index->fd, &st) == 0)
file_size = st.st_size;
else {
if (errno != ESTALE) {
mail_index_set_syscall_error(index, "fstat()");
mail_index_unlock(index, &lock_id);
return -1;
}
file_size = (uoff_t)-1;
}
/* mmaping seems to be slower than just reading the file, so even if
mmap isn't disabled don't use it unless the file is large enough */
use_mmap = !index->mmap_disable && file_size != (uoff_t)-1 &&
file_size > MAIL_INDEX_MMAP_MIN_SIZE;
new_map = mail_index_map_alloc(index);
if (use_mmap) {
new_map->rec_map->lock_id = lock_id;
ret = mail_index_mmap(new_map, file_size);
} else {
ret = mail_index_read_map(new_map, file_size, &lock_id);
mail_index_unlock(index, &lock_id);
}
for (try = 0; ret > 0; try++) {
/* make sure the header is ok before using this mapping */
ret = mail_index_map_check_header(new_map);
if (ret > 0) {
if (mail_index_map_parse_extensions(new_map) < 0)
ret = 0;
else if (mail_index_map_parse_keywords(new_map) < 0)
ret = 0;
}
if (ret != 0 || try == 2)
break;
/* fsck and try again */
old_map = index->map;
index->map = new_map;
if (mail_index_fsck(index) < 0) {
ret = -1;
break;
}
/* fsck replaced the map */
new_map = index->map;
index->map = old_map;
}
if (ret <= 0) {
mail_index_unmap(&new_map);
return ret;
}
i_assert(new_map->rec_map->records != NULL);
index->last_read_log_file_seq = new_map->hdr.log_file_seq;
index->last_read_log_file_head_offset =
new_map->hdr.log_file_head_offset;
index->last_read_log_file_tail_offset =
new_map->hdr.log_file_tail_offset;
index->last_read_stat = st;
mail_index_unmap(&index->map);
index->map = new_map;
return 1;
}
int mail_index_map(struct mail_index *index,
enum mail_index_sync_handler_type type)
{
int ret;
i_assert(index->lock_type != F_WRLCK);
i_assert(!index->mapping);
index->mapping = TRUE;
if (index->map == NULL)
index->map = mail_index_map_alloc(index);
/* first try updating the existing mapping from transaction log. */
if (index->map->hdr.indexid != 0 && index->indexid != 0) {
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
ret = mail_index_sync_map(&index->map, type, FALSE);
} else {
ret = 0;
}
if (ret == 0) {
/* try to open and read the latest index. if it fails for
any reason, we'll fallback to updating the existing mapping
from transaction logs (which we'll also do even if the
reopening succeeds) */
(void)mail_index_map_latest_file(index);
/* if we're creating the index file, we don't have any
logs yet */
if (index->log->head != NULL && index->indexid != 0) {
/* and update the map with the latest changes from
transaction log */
ret = mail_index_sync_map(&index->map, type, TRUE);
}
}
index->mapping = FALSE;
return ret;
}
static void mail_index_record_map_free(struct mail_index_map *map,
struct mail_index_record_map *rec_map)
{
if (rec_map->lock_id != 0)
mail_index_unlock(map->index, &rec_map->lock_id);
if (rec_map->buffer != NULL) {
i_assert(rec_map->mmap_base == NULL);
buffer_free(&rec_map->buffer);
} else if (rec_map->mmap_base != NULL) {
i_assert(rec_map->buffer == NULL);
if (munmap(rec_map->mmap_base, rec_map->mmap_size) < 0)
mail_index_set_syscall_error(map->index, "munmap()");
rec_map->mmap_base = NULL;
}
array_free(&rec_map->maps);
i_free(rec_map);
}
static void mail_index_record_map_unlink(struct mail_index_map *map)
{
struct mail_index_map *const *maps;
unsigned int i, count;
maps = array_get(&map->rec_map->maps, &count);
for (i = 0; i < count; i++) {
if (maps[i] == map) {
array_delete(&map->rec_map->maps, i, 1);
if (i == 0 && count == 1)
mail_index_record_map_free(map, map->rec_map);
return;
}
}
i_unreached();
}
void mail_index_unmap(struct mail_index_map **_map)
{
struct mail_index_map *map = *_map;
*_map = NULL;
if (--map->refcount > 0)
return;
i_assert(map->refcount == 0);
mail_index_record_map_unlink(map);
if (map->extension_pool != NULL)
pool_unref(&map->extension_pool);
if (array_is_created(&map->keyword_idx_map))
array_free(&map->keyword_idx_map);
buffer_free(&map->hdr_copy_buf);
i_free(map);
}
static void mail_index_map_copy_records(struct mail_index_record_map *dest,
const struct mail_index_record_map *src,
unsigned int record_size)
{
size_t size;
size = src->records_count * record_size;
dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024));
buffer_append(dest->buffer, src->records, size);
dest->records = buffer_get_modifiable_data(dest->buffer, NULL);
dest->records_count = src->records_count;
/* if the map is ever written back to disk, we need to keep track of
what has changed. */
dest->write_seq_first = src->write_seq_first;
dest->write_seq_last = src->write_seq_last;
}
static void mail_index_map_copy_header(struct mail_index_map *dest,
const struct mail_index_map *src)
{
/* use src->hdr copy directly, because if we got here
from syncing it has the latest changes. */
dest->hdr = src->hdr;
if (dest->hdr_copy_buf != NULL) {
if (src == dest)
return;
buffer_set_used_size(dest->hdr_copy_buf, 0);
} else {
dest->hdr_copy_buf =
buffer_create_dynamic(default_pool,
dest->hdr.header_size);
}
buffer_append(dest->hdr_copy_buf, &dest->hdr,
I_MIN(sizeof(dest->hdr), src->hdr.base_header_size));
if (src != dest) {
buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size,
CONST_PTR_OFFSET(src->hdr_base,
src->hdr.base_header_size),
src->hdr.header_size - src->hdr.base_header_size);
}
dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
i_assert(dest->hdr_copy_buf->used == dest->hdr.header_size);
}
static struct mail_index_record_map *
mail_index_record_map_alloc(struct mail_index_map *map)
{
struct mail_index_record_map *rec_map;
rec_map = i_new(struct mail_index_record_map, 1);
i_array_init(&rec_map->maps, 4);
array_append(&rec_map->maps, &map, 1);
return rec_map;
}
struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map)
{
struct mail_index_map *mem_map;
struct mail_index_ext *extensions;
unsigned int i, count;
mem_map = i_new(struct mail_index_map, 1);
mem_map->index = map->index;
mem_map->refcount = 1;
if (map->rec_map == NULL) {
mem_map->rec_map = mail_index_record_map_alloc(mem_map);
mem_map->rec_map->buffer =
buffer_create_dynamic(default_pool, 1024);
} else {
mem_map->rec_map = map->rec_map;
array_append(&mem_map->rec_map->maps, &mem_map, 1);
}
mail_index_map_copy_header(mem_map, map);
mem_map->write_atomic = map->write_atomic;
mem_map->write_base_header = map->write_base_header;
mem_map->write_ext_header = map->write_ext_header;
/* copy extensions */
if (array_is_created(&map->ext_id_map)) {
count = array_count(&map->ext_id_map);
mail_index_map_init_extbufs(mem_map, count + 2);
array_append_array(&mem_map->extensions, &map->extensions);
array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
/* fix the name pointers to use our own pool */
extensions = array_get_modifiable(&mem_map->extensions, &count);
for (i = 0; i < count; i++) {
i_assert(extensions[i].record_offset +
extensions[i].record_size <=
mem_map->hdr.record_size);
extensions[i].name = p_strdup(mem_map->extension_pool,
extensions[i].name);
}
}
/* copy keyword map */
if (array_is_created(&map->keyword_idx_map)) {
i_array_init(&mem_map->keyword_idx_map,
array_count(&map->keyword_idx_map) + 4);
array_append_array(&mem_map->keyword_idx_map,
&map->keyword_idx_map);
}
return mem_map;
}
void mail_index_record_map_move_to_private(struct mail_index_map *map)
{
struct mail_index_record_map *new_map;
if (array_count(&map->rec_map->maps) == 1)
return;
new_map = mail_index_record_map_alloc(map);
mail_index_map_copy_records(new_map, map->rec_map,
map->hdr.record_size);
mail_index_record_map_unlink(map);
map->rec_map = new_map;
}
void mail_index_map_move_to_memory(struct mail_index_map *map)
{
struct mail_index_record_map *new_map;
if (map->rec_map->mmap_base == NULL)
return;
i_assert(map->rec_map->lock_id != 0);
new_map = array_count(&map->rec_map->maps) == 1 ? map->rec_map :
mail_index_record_map_alloc(map);
mail_index_map_copy_records(new_map, map->rec_map,
map->hdr.record_size);
mail_index_map_copy_header(map, map);
if (new_map != map->rec_map) {
mail_index_record_map_unlink(map);
map->rec_map = new_map;
} else {
mail_index_unlock(map->index, &new_map->lock_id);
if (munmap(new_map->mmap_base, new_map->mmap_size) < 0)
mail_index_set_syscall_error(map->index, "munmap()");
new_map->mmap_base = NULL;
}
}
bool mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r)
{
const uint32_t *id;
if (!array_is_created(&map->ext_id_map) ||
ext_id >= array_count(&map->ext_id_map))
return FALSE;
id = array_idx(&map->ext_id_map, ext_id);
*idx_r = *id;
return *idx_r != (uint32_t)-1;
}