mail-index-map.c revision 3e559edb905bb57d1f1fbfda3431b339f45ea791
1516N/A/* Copyright (c) 2003-2007 Dovecot authors, see the included COPYING file */
39N/A
39N/A#include "lib.h"
39N/A#include "array.h"
39N/A#include "str-sanitize.h"
39N/A#include "nfs-workarounds.h"
39N/A#include "mmap-util.h"
39N/A#include "read-full.h"
39N/A#include "mail-index-private.h"
39N/A#include "mail-index-sync-private.h"
39N/A#include "mail-transaction-log-private.h"
39N/A
39N/Astatic void mail_index_map_init_extbufs(struct mail_index_map *map,
39N/A unsigned int initial_count)
39N/A{
39N/A#define EXTENSION_NAME_APPROX_LEN 20
39N/A#define EXT_GLOBAL_ALLOC_SIZE \
39N/A ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
39N/A#define EXT_PER_ALLOC_SIZE \
39N/A (EXTENSION_NAME_APPROX_LEN + \
39N/A sizeof(struct mail_index_ext) + sizeof(uint32_t))
926N/A size_t size;
926N/A
2854N/A if (map->extension_pool == NULL) {
926N/A size = EXT_GLOBAL_ALLOC_SIZE +
39N/A initial_count * EXT_PER_ALLOC_SIZE;
2453N/A map->extension_pool =
342N/A pool_alloconly_create("map extensions",
1516N/A nearest_power(size));
1636N/A } else {
1386N/A p_clear(map->extension_pool);
2639N/A
2639N/A /* try to use the existing pool's size for initial_count so
39N/A we don't grow it unneededly */
51N/A size = p_get_max_easy_alloc_size(map->extension_pool);
2073N/A if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
2144N/A initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
1066N/A EXT_PER_ALLOC_SIZE;
1231N/A }
2453N/A }
1352N/A
1890N/A p_array_init(&map->extensions, map->extension_pool, initial_count);
296N/A p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
39N/A}
2690N/A
2690N/Abool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name,
2690N/A uint32_t *idx_r)
2690N/A{
2690N/A const struct mail_index_ext *extensions;
2690N/A unsigned int i, size;
2690N/A
2690N/A if (array_is_created(&map->extensions)) {
2690N/A extensions = array_get(&map->extensions, &size);
2690N/A for (i = 0; i < size; i++) {
2690N/A if (strcmp(extensions[i].name, name) == 0) {
2690N/A if (idx_r != NULL)
2690N/A *idx_r = i;
2690N/A return TRUE;
2690N/A }
2690N/A }
2690N/A }
2690N/A return FALSE;
2690N/A}
2690N/A
2690N/Aunsigned int mail_index_map_ext_hdr_offset(unsigned int name_len)
2690N/A{
2690N/A size_t size = sizeof(struct mail_index_ext_header) + name_len;
2690N/A return MAIL_INDEX_HEADER_SIZE_ALIGN(size);
2690N/A}
2690N/A
2690N/Auint32_t
2690N/Amail_index_map_register_ext(struct mail_index_map *map, const char *name,
2690N/A uint32_t ext_offset, uint32_t hdr_size,
2690N/A uint32_t record_offset, uint32_t record_size,
2690N/A uint32_t record_align, uint32_t reset_id)
2690N/A{
2690N/A struct mail_index_ext *ext;
2690N/A uint32_t idx, empty_idx = (uint32_t)-1;
2690N/A
2690N/A if (!array_is_created(&map->extensions)) {
2690N/A mail_index_map_init_extbufs(map, 5);
2690N/A idx = 0;
1713N/A } else {
39N/A idx = array_count(&map->extensions);
39N/A }
39N/A i_assert(!mail_index_map_lookup_ext(map, name, NULL));
39N/A
39N/A ext = array_append_space(&map->extensions);
205N/A ext->name = p_strdup(map->extension_pool, name);
39N/A ext->ext_offset = ext_offset;
205N/A ext->hdr_offset = ext_offset +
39N/A mail_index_map_ext_hdr_offset(strlen(name));
39N/A ext->hdr_size = hdr_size;
39N/A ext->record_offset = record_offset;
39N/A ext->record_size = record_size;
39N/A ext->record_align = record_align;
39N/A ext->reset_id = reset_id;
39N/A
39N/A ext->index_idx = mail_index_ext_register(map->index, name, hdr_size,
39N/A record_size, record_align);
39N/A
39N/A /* Update index ext_id -> map ext_id mapping. Fill non-used
39N/A ext_ids with (uint32_t)-1 */
39N/A while (array_count(&map->ext_id_map) < ext->index_idx)
39N/A array_append(&map->ext_id_map, &empty_idx, 1);
39N/A array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
39N/A return idx;
39N/A}
39N/A
39N/Aint mail_index_map_ext_get_next(struct mail_index_map *map,
39N/A unsigned int *offset_p,
39N/A const struct mail_index_ext_header **ext_hdr_r,
39N/A const char **name_r)
48N/A{
48N/A const struct mail_index_ext_header *ext_hdr;
48N/A unsigned int offset, name_offset;
48N/A
48N/A offset = *offset_p;
59N/A *name_r = "";
48N/A
2073N/A /* Extension header contains:
2073N/A - struct mail_index_ext_header
39N/A - name (not 0-terminated)
2639N/A - 64bit alignment padding
2639N/A - extension header contents
2639N/A - 64bit alignment padding
39N/A */
567N/A name_offset = offset + sizeof(*ext_hdr);
838N/A ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
1890N/A if (offset + sizeof(*ext_hdr) >= map->hdr.header_size)
2608N/A return -1;
39N/A
39N/A offset += mail_index_map_ext_hdr_offset(ext_hdr->name_size);
39N/A if (offset > map->hdr.header_size)
1431N/A return -1;
1431N/A
39N/A *name_r = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
227N/A ext_hdr->name_size);
315N/A if (strcmp(*name_r, str_sanitize(*name_r, -1)) != 0) {
315N/A /* we allow only plain ASCII names, so this extension
39N/A is most likely broken */
1352N/A *name_r = "";
1352N/A }
1352N/A
1352N/A /* finally make sure that the hdr_size is small enough.
1431N/A do this last so that we could return a usable name. */
1431N/A offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
429N/A if (offset > map->hdr.header_size)
315N/A return -1;
1352N/A
429N/A *offset_p = offset;
1352N/A *ext_hdr_r = ext_hdr;
1352N/A return 0;
39N/A}
926N/A
926N/Aint mail_index_map_ext_hdr_check(const struct mail_index_header *hdr,
203N/A const struct mail_index_ext_header *ext_hdr,
203N/A const char *name, const char **error_r)
203N/A{
203N/A if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
1045N/A (ext_hdr->record_align == 0 && ext_hdr->record_size != 0)) {
1045N/A *error_r = "Invalid field values";
72N/A return -1;
72N/A }
59N/A if (*name == '\0') {
2054N/A *error_r = "Broken name";
1045N/A return -1;
1045N/A }
1045N/A
1045N/A if (ext_hdr->record_offset + ext_hdr->record_size > hdr->record_size) {
1713N/A *error_r = t_strdup_printf("Record field points "
1045N/A "outside record size (%u+%u > %u)",
1045N/A ext_hdr->record_offset,
1045N/A ext_hdr->record_size,
2084N/A hdr->record_size);
2084N/A return -1;
2639N/A }
2084N/A
2854N/A if (ext_hdr->record_size > 0 &&
2084N/A ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
2842N/A (hdr->record_size % ext_hdr->record_align) != 0)) {
2842N/A *error_r = t_strdup_printf("Record field alignmentation %u "
2842N/A "not used", ext_hdr->record_align);
2842N/A return -1;
2842N/A }
2842N/A return 0;
2842N/A}
2842N/A
2854N/Astatic int mail_index_map_parse_extensions(struct mail_index_map *map)
2854N/A{
2842N/A struct mail_index *index = map->index;
2842N/A const struct mail_index_ext_header *ext_hdr;
2842N/A unsigned int i, old_count, offset;
2842N/A const char *name, *error;
2842N/A uint32_t ext_id, ext_offset;
2842N/A
2842N/A /* extension headers always start from 64bit offsets, so if base header
2842N/A doesn't happen to be 64bit aligned we'll skip some bytes */
2842N/A offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
2842N/A if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
59N/A /* nothing to do, skip allocatations and all */
2639N/A return 0;
2639N/A }
59N/A
72N/A old_count = array_count(&index->extensions);
72N/A mail_index_map_init_extbufs(map, old_count + 5);
72N/A
72N/A ext_id = (uint32_t)-1;
72N/A for (i = 0; i < old_count; i++)
2144N/A array_append(&map->ext_id_map, &ext_id, 1);
72N/A
59N/A for (i = 0; offset < map->hdr.header_size; i++) {
72N/A ext_offset = offset;
72N/A
72N/A t_push();
72N/A if (mail_index_map_ext_get_next(map, &offset,
59N/A &ext_hdr, &name) < 0) {
72N/A mail_index_set_error(index, "Corrupted index file %s: "
2639N/A "Header extension #%d (%s) goes outside header",
2639N/A index->filepath, i, name);
2639N/A t_pop();
2639N/A return -1;
2639N/A }
59N/A
1713N/A if (mail_index_map_ext_hdr_check(&map->hdr, ext_hdr,
59N/A name, &error) < 0) {
838N/A mail_index_set_error(index, "Corrupted index file %s: "
2240N/A "Broken extension #%d (%s): %s",
838N/A index->filepath, i, name, error);
838N/A t_pop();
838N/A return -1;
926N/A }
838N/A if (mail_index_map_lookup_ext(map, name, NULL)) {
838N/A mail_index_set_error(index, "Corrupted index file %s: "
2240N/A "Duplicate header extension %s",
2240N/A index->filepath, name);
2284N/A t_pop();
2240N/A return -1;
838N/A }
2240N/A
2240N/A mail_index_map_register_ext(map, name, ext_offset,
2240N/A ext_hdr->hdr_size,
2240N/A ext_hdr->record_offset,
2240N/A ext_hdr->record_size,
2240N/A ext_hdr->record_align,
2240N/A ext_hdr->reset_id);
2240N/A t_pop();
2240N/A }
2317N/A return 0;
2240N/A}
2240N/A
2240N/Aint mail_index_map_parse_keywords(struct mail_index_map *map)
2240N/A{
2240N/A struct mail_index *index = map->index;
2240N/A const struct mail_index_ext *ext;
2240N/A const struct mail_index_keyword_header *kw_hdr;
2240N/A const struct mail_index_keyword_header_rec *kw_rec;
2240N/A const char *name;
2240N/A unsigned int i, name_area_end_offset, old_count;
2240N/A uint32_t idx;
2284N/A
2284N/A if (!mail_index_map_lookup_ext(map, "keywords", &idx)) {
2284N/A if (array_is_created(&map->keyword_idx_map))
2284N/A array_clear(&map->keyword_idx_map);
2240N/A return 0;
2240N/A }
2284N/A ext = array_idx(&map->extensions, idx);
2284N/A
2284N/A /* Extension header contains:
838N/A - struct mail_index_keyword_header
838N/A - struct mail_index_keyword_header_rec * keywords_count
838N/A - const char names[] * keywords_count
838N/A */
838N/A i_assert(ext->hdr_offset < map->hdr.header_size);
838N/A kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
838N/A kw_rec = (const void *)(kw_hdr + 1);
926N/A name = (const char *)(kw_rec + kw_hdr->keywords_count);
838N/A
838N/A old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
838N/A array_count(&map->keyword_idx_map);
838N/A
838N/A /* Keywords can only be added into same mapping. Removing requires a
838N/A new mapping (recreating the index file) */
838N/A if (kw_hdr->keywords_count == old_count) {
845N/A /* nothing changed */
845N/A return 0;
926N/A }
838N/A
845N/A /* make sure the header is valid */
845N/A if (kw_hdr->keywords_count < old_count) {
845N/A mail_index_set_error(index, "Corrupted index file %s: "
838N/A "Keywords removed unexpectedly",
845N/A index->filepath);
845N/A return -1;
838N/A }
926N/A
203N/A if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
315N/A mail_index_set_error(index, "Corrupted index file %s: "
838N/A "keywords_count larger than header size",
203N/A index->filepath);
838N/A return -1;
48N/A }
48N/A
48N/A name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
48N/A for (i = 0; i < kw_hdr->keywords_count; i++) {
48N/A if (kw_rec[i].name_offset > name_area_end_offset) {
838N/A mail_index_set_error(index, "Corrupted index file %s: "
203N/A "name_offset points outside allocated header",
48N/A index->filepath);
203N/A return -1;
72N/A }
181N/A }
72N/A if (name[name_area_end_offset-1] != '\0') {
181N/A mail_index_set_error(index, "Corrupted index file %s: "
46N/A "Keyword header doesn't end with NUL",
181N/A index->filepath);
237N/A return -1;
46N/A }
2453N/A
2453N/A /* create file -> index mapping */
2453N/A if (!array_is_created(&map->keyword_idx_map))
2453N/A i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
2339N/A
2453N/A#ifdef DEBUG
2453N/A /* Check that existing headers are still the same. It's behind DEBUG
2453N/A since it's pretty useless waste of CPU normally. */
2453N/A for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
2453N/A const char *keyword = name + kw_rec[i].name_offset;
2339N/A const unsigned int *old_idx;
2339N/A unsigned int idx;
2339N/A
2339N/A old_idx = array_idx(&map->keyword_idx_map, i);
2339N/A if (!mail_index_keyword_lookup(index, keyword, &idx) ||
2339N/A idx != *old_idx) {
2453N/A mail_index_set_error(index, "Corrupted index file %s: "
2453N/A "Keywords changed unexpectedly",
2453N/A index->filepath);
2453N/A return -1;
2453N/A }
2453N/A }
2453N/A#endif
2453N/A /* Register the newly seen keywords */
2453N/A i = array_count(&map->keyword_idx_map);
2453N/A for (; i < kw_hdr->keywords_count; i++) {
2453N/A const char *keyword = name + kw_rec[i].name_offset;
2453N/A unsigned int idx;
2339N/A
2453N/A if (*keyword == '\0') {
2453N/A mail_index_set_error(index, "Corrupted index file %s: "
2453N/A "Empty keyword name in header",
2453N/A index->filepath);
2453N/A return -1;
2453N/A }
2453N/A mail_index_keyword_lookup_or_create(index, keyword, &idx);
2453N/A array_append(&map->keyword_idx_map, &idx, 1);
2453N/A }
2837N/A return 0;
2453N/A}
2453N/A
2453N/Astatic bool mail_index_check_header_compat(struct mail_index *index,
2453N/A const struct mail_index_header *hdr,
2453N/A uoff_t file_size)
2453N/A{
2453N/A enum mail_index_header_compat_flags compat_flags = 0;
2453N/A
2453N/A#ifndef WORDS_BIGENDIAN
2453N/A compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
2453N/A#endif
2453N/A
2453N/A if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2453N/A /* major version change - handle silently(?) */
2453N/A return FALSE;
2453N/A }
2453N/A if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
2453N/A /* we've already complained about it */
2339N/A return FALSE;
2339N/A }
2339N/A
2339N/A if (hdr->compat_flags != compat_flags) {
2339N/A /* architecture change */
2453N/A mail_index_set_error(index, "Rebuilding index file %s: "
2453N/A "CPU architecture changed",
2453N/A index->filepath);
2453N/A return FALSE;
2453N/A }
2339N/A
2453N/A if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
2453N/A hdr->header_size < hdr->base_header_size) {
2453N/A mail_index_set_error(index, "Corrupted index file %s: "
2453N/A "Corrupted header sizes (base %u, full %u)",
2339N/A index->filepath, hdr->base_header_size,
2453N/A hdr->header_size);
2453N/A return FALSE;
2453N/A }
2453N/A if (hdr->header_size > file_size) {
2453N/A mail_index_set_error(index, "Corrupted index file %s: "
2339N/A "Corrupted header size (%u > %"PRIuUOFF_T")",
2339N/A index->filepath, hdr->header_size,
2339N/A file_size);
2339N/A return FALSE;
2339N/A }
2608N/A
2608N/A if (hdr->indexid != index->indexid) {
2608N/A if (index->indexid != 0) {
2339N/A mail_index_set_error(index, "Index file %s: "
2453N/A "indexid changed: %u -> %u",
2453N/A index->filepath, index->indexid,
2339N/A hdr->indexid);
2453N/A }
2639N/A index->indexid = hdr->indexid;
2540N/A mail_transaction_log_indexid_changed(index->log);
2339N/A }
2339N/A
2339N/A return TRUE;
2339N/A}
2339N/A
2639N/Aint mail_index_map_check_header(struct mail_index_map *map)
2339N/A{
2339N/A struct mail_index *index = map->index;
2339N/A const struct mail_index_header *hdr = &map->hdr;
2339N/A
2453N/A if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1))
2453N/A return -1;
2453N/A
2453N/A /* following some extra checks that only take a bit of CPU */
2453N/A if (hdr->record_size < sizeof(struct mail_index_record)) {
2453N/A mail_index_set_error(index, "Corrupted index file %s: "
2453N/A "record_size too small: %u < %"PRIuSIZE_T,
2453N/A index->filepath, hdr->record_size,
2608N/A sizeof(struct mail_index_record));
2608N/A return -1;
2608N/A }
2453N/A
2453N/A if (hdr->uid_validity == 0 && hdr->next_uid != 1)
2453N/A return 0;
2453N/A if (hdr->next_uid == 0)
2453N/A return 0;
2639N/A if (hdr->messages_count > map->rec_map->records_count)
2540N/A return 0;
2453N/A
2453N/A if (hdr->seen_messages_count > hdr->messages_count ||
2453N/A hdr->deleted_messages_count > hdr->messages_count)
2453N/A return 0;
2453N/A if (hdr->first_recent_uid == 0 && hdr->minor_version == 0) {
2453N/A /* upgrade silently from v1.0 */
2639N/A map->hdr.first_recent_uid = 1;
2453N/A }
2453N/A if (hdr->first_recent_uid == 0 ||
2453N/A hdr->first_recent_uid > hdr->next_uid ||
2453N/A hdr->first_unseen_uid_lowwater > hdr->next_uid ||
2453N/A hdr->first_deleted_uid_lowwater > hdr->next_uid)
2453N/A return 0;
2453N/A
2453N/A if (hdr->messages_count > 0) {
2453N/A /* last message's UID must be smaller than next_uid.
2453N/A also make sure it's not zero. */
2453N/A const struct mail_index_record *rec;
2453N/A
2453N/A rec = MAIL_INDEX_MAP_IDX(map, hdr->messages_count-1);
2453N/A if (rec->uid == 0 || rec->uid >= hdr->next_uid)
2453N/A return 0;
2453N/A }
2453N/A
2453N/A return 1;
2453N/A}
2453N/A
838N/Astatic void mail_index_map_copy_hdr(struct mail_index_map *map,
838N/A const struct mail_index_header *hdr)
2608N/A{
2608N/A if (hdr->base_header_size < sizeof(map->hdr)) {
2608N/A /* header smaller than ours, make a copy so our newer headers
2608N/A won't have garbage in them */
838N/A memset(&map->hdr, 0, sizeof(map->hdr));
838N/A memcpy(&map->hdr, hdr, hdr->base_header_size);
838N/A } else {
838N/A map->hdr = *hdr;
838N/A }
838N/A
342N/A /* FIXME: backwards compatibility, remove later. In case this index is
926N/A accessed with Dovecot v1.0, avoid recent message counter errors. */
838N/A map->hdr.unused_old_recent_messages_count = 0;
838N/A}
2608N/A
2608N/Astatic int mail_index_mmap(struct mail_index_map *map, uoff_t file_size)
2608N/A{
2608N/A struct mail_index *index = map->index;
926N/A struct mail_index_record_map *rec_map = map->rec_map;
838N/A const struct mail_index_header *hdr;
838N/A
838N/A i_assert(rec_map->mmap_base == NULL);
838N/A
838N/A buffer_free(&rec_map->buffer);
926N/A if (file_size > SSIZE_T_MAX) {
2453N/A /* too large file to map into memory */
2453N/A mail_index_set_error(index, "Index file too large: %s",
2453N/A index->filepath);
2608N/A return -1;
2453N/A }
2453N/A
2453N/A rec_map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
2453N/A MAP_PRIVATE, index->fd, 0);
2453N/A if (rec_map->mmap_base == MAP_FAILED) {
926N/A rec_map->mmap_base = NULL;
2843N/A mail_index_set_syscall_error(index, "mmap()");
615N/A return -1;
615N/A }
615N/A rec_map->mmap_size = file_size;
615N/A
926N/A hdr = rec_map->mmap_base;
615N/A if (rec_map->mmap_size >
615N/A offsetof(struct mail_index_header, major_version) &&
838N/A hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
111N/A /* major version change - handle silently */
111N/A return 0;
111N/A }
111N/A
111N/A if (rec_map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
111N/A mail_index_set_error(index, "Corrupted index file %s: "
111N/A "File too small (%"PRIuSIZE_T")",
111N/A index->filepath, rec_map->mmap_size);
113N/A return 0;
926N/A }
838N/A
926N/A if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size)) {
113N/A /* Can't use this file */
113N/A return 0;
113N/A }
113N/A
113N/A rec_map->mmap_used_size = hdr->header_size +
113N/A hdr->messages_count * hdr->record_size;
113N/A
113N/A if (rec_map->mmap_used_size <= rec_map->mmap_size)
113N/A rec_map->records_count = hdr->messages_count;
111N/A else {
1500N/A rec_map->records_count =
1432N/A (rec_map->mmap_size - hdr->header_size) /
1542N/A hdr->record_size;
1970N/A rec_map->mmap_used_size = hdr->header_size +
2073N/A rec_map->records_count * hdr->record_size;
2073N/A mail_index_set_error(index, "Corrupted index file %s: "
2073N/A "messages_count too large (%u > %u)",
2073N/A index->filepath, hdr->messages_count,
2073N/A rec_map->records_count);
2073N/A }
1542N/A
146N/A mail_index_map_copy_hdr(map, hdr);
1432N/A
1432N/A map->hdr_base = rec_map->mmap_base;
50N/A rec_map->records = PTR_OFFSET(rec_map->mmap_base, map->hdr.header_size);
1432N/A return 1;
1432N/A}
1432N/A
51N/Astatic int mail_index_read_header(struct mail_index *index,
1636N/A void *buf, size_t buf_size, size_t *pos_r)
1432N/A{
1507N/A size_t pos;
51N/A int ret;
1500N/A
591N/A memset(buf, 0, sizeof(struct mail_index_header));
1970N/A
1970N/A /* try to read the whole header, but it's not necessarily an error to
1970N/A read less since the older versions of the index format could be
591N/A smaller. Request reading up to buf_size, but accept if we only got
1542N/A the header. */
1970N/A pos = 0;
1970N/A do {
1970N/A ret = pread(index->fd, PTR_OFFSET(buf, pos),
2073N/A buf_size - pos, pos);
1500N/A if (ret > 0)
2073N/A pos += ret;
2073N/A } while (ret > 0 && pos < sizeof(struct mail_index_header));
1713N/A
1713N/A *pos_r = pos;
2073N/A return ret;
2073N/A}
2073N/A
1713N/Astatic int
2608N/Amail_index_try_read_map(struct mail_index_map *map,
2608N/A uoff_t file_size, bool *retry_r, bool try_retry)
2613N/A{
2284N/A struct mail_index *index = map->index;
2073N/A const struct mail_index_header *hdr;
2073N/A unsigned char read_buf[4096];
1713N/A const void *buf;
2073N/A void *data = NULL;
2073N/A ssize_t ret;
2073N/A size_t pos, records_size, initial_buf_pos = 0;
1713N/A unsigned int records_count = 0, extra;
1713N/A
2073N/A i_assert(map->rec_map->mmap_base == NULL);
2073N/A
2073N/A *retry_r = FALSE;
1500N/A ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
1500N/A buf = read_buf; hdr = buf;
2639N/A
2639N/A if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
1500N/A hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2613N/A /* major version change - handle silently */
51N/A return 0;
1500N/A }
1500N/A
1500N/A if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
1500N/A (ret > 0 || pos >= hdr->base_header_size)) {
1500N/A if (!mail_index_check_header_compat(index, hdr, file_size)) {
1500N/A /* Can't use this file */
1500N/A return 0;
2073N/A }
2073N/A
2073N/A initial_buf_pos = pos;
2073N/A if (pos > hdr->header_size)
2073N/A pos = hdr->header_size;
2073N/A
1890N/A /* place the base header into memory. */
1500N/A buffer_reset(map->hdr_copy_buf);
2073N/A buffer_append(map->hdr_copy_buf, buf, pos);
2073N/A
2073N/A if (pos != hdr->header_size) {
1500N/A /* @UNSAFE: read the rest of the header into memory */
1500N/A data = buffer_append_space_unsafe(map->hdr_copy_buf,
1500N/A hdr->header_size -
1500N/A pos);
926N/A ret = pread_full(index->fd, data,
1500N/A hdr->header_size - pos, pos);
2026N/A }
2608N/A }
2608N/A
2608N/A if (ret > 0) {
2608N/A /* header read, read the records now. */
1500N/A records_size = (size_t)hdr->messages_count * hdr->record_size;
2026N/A records_count = hdr->messages_count;
2026N/A
2026N/A if (file_size - hdr->header_size < records_size ||
2026N/A (hdr->record_size != 0 &&
2073N/A records_size / hdr->record_size != hdr->messages_count)) {
2026N/A records_count = (file_size - hdr->header_size) /
2026N/A hdr->record_size;
1500N/A records_size = (size_t)records_count * hdr->record_size;
1500N/A mail_index_set_error(index, "Corrupted index file %s: "
1500N/A "messages_count too large (%u > %u)",
1500N/A index->filepath, hdr->messages_count,
1500N/A records_count);
123N/A }
1500N/A
1500N/A if (map->rec_map->buffer == NULL) {
877N/A map->rec_map->buffer =
2639N/A buffer_create_dynamic(default_pool,
2639N/A records_size);
1500N/A }
2639N/A
2639N/A /* @UNSAFE */
2639N/A buffer_set_used_size(map->rec_map->buffer, 0);
2639N/A if (initial_buf_pos <= hdr->header_size)
2639N/A extra = 0;
2639N/A else {
2639N/A extra = initial_buf_pos - hdr->header_size;
2639N/A buffer_append(map->rec_map->buffer,
2639N/A CONST_PTR_OFFSET(buf, hdr->header_size),
1500N/A extra);
2639N/A }
2639N/A if (records_size > extra) {
2639N/A data = buffer_append_space_unsafe(map->rec_map->buffer,
2639N/A records_size - extra);
1500N/A ret = pread_full(index->fd, data, records_size - extra,
2639N/A hdr->header_size + extra);
2639N/A }
2639N/A }
2639N/A
2639N/A if (ret < 0) {
1500N/A if (errno == ESTALE && try_retry) {
1500N/A /* a new index file was renamed over this one. */
2639N/A *retry_r = TRUE;
2639N/A return 0;
2639N/A }
2639N/A mail_index_set_syscall_error(index, "pread_full()");
838N/A return -1;
1500N/A }
2639N/A if (ret == 0) {
1500N/A mail_index_set_error(index,
39N/A "Corrupted index file %s: File too small",
956N/A index->filepath);
956N/A return 0;
956N/A }
956N/A
1431N/A map->rec_map->records =
1431N/A buffer_get_modifiable_data(map->rec_map->buffer, NULL);
956N/A map->rec_map->records_count = records_count;
956N/A
956N/A mail_index_map_copy_hdr(map, hdr);
956N/A map->hdr_base = map->hdr_copy_buf->data;
956N/A return 1;
956N/A}
941N/A
1007N/Astatic int mail_index_read_map(struct mail_index_map *map, uoff_t file_size,
1007N/A unsigned int *lock_id)
1007N/A{
1007N/A struct mail_index *index = map->index;
1007N/A mail_index_sync_lost_handler_t *const *handlers;
1007N/A struct stat st;
1007N/A unsigned int i, count;
1007N/A int ret;
1007N/A bool try_retry, retry;
1007N/A
1007N/A /* notify all "sync lost" handlers */
1007N/A handlers = array_get(&index->sync_lost_handlers, &count);
1007N/A for (i = 0; i < count; i++)
1007N/A (*handlers[i])(index);
1100N/A
2704N/A for (i = 0;; i++) {
2704N/A try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
2704N/A if (file_size == (uoff_t)-1) {
2704N/A /* fstat() below failed */
2704N/A ret = 0;
2704N/A retry = try_retry;
2704N/A } else {
2704N/A ret = mail_index_try_read_map(map, file_size,
941N/A &retry, try_retry);
941N/A }
144N/A if (ret != 0 || !retry)
941N/A break;
1100N/A
1100N/A /* ESTALE - reopen index file */
1100N/A mail_index_close_file(index);
1100N/A *lock_id = 0;
1100N/A
1100N/A ret = mail_index_try_open_only(index);
1100N/A if (ret <= 0) {
1100N/A if (ret == 0) {
1100N/A /* the file was lost */
1100N/A errno = ENOENT;
941N/A mail_index_set_syscall_error(index, "open()");
941N/A }
941N/A return -1;
941N/A }
941N/A if (mail_index_lock_shared(index, lock_id) < 0)
941N/A return -1;
941N/A
941N/A if (fstat(index->fd, &st) == 0)
941N/A file_size = st.st_size;
941N/A else {
941N/A if (errno != ESTALE) {
941N/A mail_index_set_syscall_error(index, "fstat()");
941N/A return -1;
941N/A }
941N/A file_size = (uoff_t)-1;
429N/A }
941N/A }
941N/A return ret;
941N/A}
941N/A
941N/Astatic void mail_index_header_init(struct mail_index *index,
429N/A struct mail_index_header *hdr)
941N/A{
941N/A i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
941N/A
941N/A memset(hdr, 0, sizeof(*hdr));
941N/A
1007N/A hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
1007N/A hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
1234N/A hdr->base_header_size = sizeof(*hdr);
1007N/A hdr->header_size = sizeof(*hdr);
1007N/A hdr->record_size = sizeof(struct mail_index_record);
1007N/A
2639N/A#ifndef WORDS_BIGENDIAN
2639N/A hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
1007N/A#endif
1007N/A
1007N/A hdr->indexid = index->indexid;
1007N/A hdr->log_file_seq = 1;
1007N/A hdr->next_uid = 1;
1007N/A hdr->first_recent_uid = 1;
1007N/A}
1007N/A
1007N/Astruct mail_index_map *mail_index_map_alloc(struct mail_index *index)
1007N/A{
1007N/A struct mail_index_map tmp_map;
1007N/A
1007N/A memset(&tmp_map, 0, sizeof(tmp_map));
1007N/A mail_index_header_init(index, &tmp_map.hdr);
1007N/A tmp_map.index = index;
1007N/A tmp_map.hdr_base = &tmp_map.hdr;
1007N/A
1007N/A /* a bit kludgy way to do this, but it initializes everything
1007N/A nicely and correctly */
1007N/A return mail_index_map_clone(&tmp_map);
1007N/A}
1007N/A
1007N/Astatic int mail_index_map_latest_file(struct mail_index *index)
1007N/A{
941N/A struct mail_index_map *old_map, *new_map;
941N/A struct stat st;
941N/A unsigned int lock_id;
144N/A uoff_t file_size;
144N/A bool use_mmap;
1472N/A int ret, try;
1472N/A
1472N/A ret = mail_index_reopen_if_changed(index);
1472N/A if (ret <= 0) {
1352N/A if (ret < 0)
1516N/A return -1;
1890N/A
1890N/A /* the index file is lost/broken. let's hope that we can
1890N/A build it from the transaction log. */
1890N/A return 0;
1890N/A }
1352N/A
1472N/A /* the index file is still open, lock it */
1352N/A if (mail_index_lock_shared(index, &lock_id) < 0)
1352N/A return -1;
1352N/A
1352N/A if (index->nfs_flush)
1352N/A nfs_flush_attr_cache_fd(index->filepath, index->fd);
1352N/A
1352N/A if (fstat(index->fd, &st) == 0)
2073N/A file_size = st.st_size;
1352N/A else {
429N/A if (errno != ESTALE) {
429N/A mail_index_set_syscall_error(index, "fstat()");
144N/A mail_index_unlock(index, &lock_id);
1386N/A return -1;
1386N/A }
1386N/A file_size = (uoff_t)-1;
1636N/A }
1636N/A
1636N/A /* mmaping seems to be slower than just reading the file, so even if
1636N/A mmap isn't disabled don't use it unless the file is large enough */
2073N/A use_mmap = !index->mmap_disable && file_size != (uoff_t)-1 &&
1636N/A file_size > MAIL_INDEX_MMAP_MIN_SIZE;
2073N/A
1636N/A new_map = mail_index_map_alloc(index);
1636N/A if (use_mmap) {
1636N/A new_map->rec_map->lock_id = lock_id;
1386N/A ret = mail_index_mmap(new_map, file_size);
1636N/A } else {
1636N/A ret = mail_index_read_map(new_map, file_size, &lock_id);
1636N/A mail_index_unlock(index, &lock_id);
1636N/A }
2073N/A
1636N/A for (try = 0; ret > 0; try++) {
2284N/A /* make sure the header is ok before using this mapping */
1636N/A ret = mail_index_map_check_header(new_map);
1636N/A if (ret > 0) {
1636N/A if (mail_index_map_parse_extensions(new_map) < 0)
1386N/A ret = 0;
315N/A else if (mail_index_map_parse_keywords(new_map) < 0)
315N/A ret = 0;
315N/A }
315N/A if (ret != 0 || try == 2)
315N/A break;
315N/A
315N/A /* fsck and try again */
315N/A old_map = index->map;
1636N/A index->map = new_map;
1636N/A if (mail_index_fsck(index) < 0) {
1636N/A ret = -1;
1636N/A break;
1636N/A }
1636N/A
2073N/A /* fsck replaced the map */
1636N/A new_map = index->map;
2073N/A index->map = old_map;
1636N/A }
1636N/A if (ret <= 0) {
144N/A mail_index_unmap(&new_map);
838N/A return ret;
838N/A }
838N/A i_assert(new_map->rec_map->records != NULL);
926N/A
926N/A index->last_read_log_file_seq = new_map->hdr.log_file_seq;
926N/A index->last_read_log_file_head_offset =
926N/A new_map->hdr.log_file_head_offset;
838N/A index->last_read_log_file_tail_offset =
1231N/A new_map->hdr.log_file_tail_offset;
1231N/A index->last_read_stat = st;
2091N/A
1231N/A mail_index_unmap(&index->map);
1231N/A index->map = new_map;
1231N/A return 1;
1231N/A}
1231N/A
205N/Aint mail_index_map(struct mail_index *index,
257N/A enum mail_index_sync_handler_type type)
257N/A{
257N/A int ret;
205N/A
205N/A i_assert(index->lock_type != F_WRLCK);
1461N/A i_assert(!index->mapping);
1461N/A
1461N/A index->mapping = TRUE;
1461N/A
1461N/A if (index->map == NULL)
1461N/A index->map = mail_index_map_alloc(index);
1461N/A
1461N/A /* first try updating the existing mapping from transaction log. */
1461N/A if (index->map->hdr.indexid != 0 && index->indexid != 0) {
1461N/A /* we're not creating the index, or opening transaction log.
1461N/A sync this as a view from transaction log. */
1461N/A ret = mail_index_sync_map(&index->map, type, FALSE);
1044N/A } else {
1044N/A ret = 0;
1044N/A }
1044N/A
1044N/A if (ret == 0) {
1044N/A /* try to open and read the latest index. if it fails for
1044N/A any reason, we'll fallback to updating the existing mapping
1044N/A from transaction logs (which we'll also do even if the
1044N/A reopening succeeds) */
1044N/A (void)mail_index_map_latest_file(index);
2286N/A
1044N/A /* if we're creating the index file, we don't have any
1044N/A logs yet */
2639N/A if (index->log->head != NULL && index->indexid != 0) {
2639N/A /* and update the map with the latest changes from
2639N/A transaction log */
2639N/A ret = mail_index_sync_map(&index->map, type, TRUE);
2639N/A }
2639N/A }
2639N/A
2639N/A index->mapping = FALSE;
2639N/A return ret;
2639N/A}
2639N/A
2639N/Astatic void mail_index_record_map_free(struct mail_index_map *map,
2639N/A struct mail_index_record_map *rec_map)
2639N/A{
2639N/A if (rec_map->lock_id != 0)
2639N/A mail_index_unlock(map->index, &rec_map->lock_id);
2639N/A
2639N/A if (rec_map->buffer != NULL) {
2639N/A i_assert(rec_map->mmap_base == NULL);
2639N/A buffer_free(&rec_map->buffer);
2639N/A } else if (rec_map->mmap_base != NULL) {
2639N/A i_assert(rec_map->buffer == NULL);
2639N/A if (munmap(rec_map->mmap_base, rec_map->mmap_size) < 0)
2639N/A mail_index_set_syscall_error(map->index, "munmap()");
2639N/A rec_map->mmap_base = NULL;
2639N/A }
2639N/A array_free(&rec_map->maps);
2639N/A i_free(rec_map);
2639N/A}
2639N/A
2639N/Astatic void mail_index_record_map_unlink(struct mail_index_map *map)
2639N/A{
2639N/A struct mail_index_map *const *maps;
2639N/A unsigned int i, count;
2639N/A
2639N/A maps = array_get(&map->rec_map->maps, &count);
2639N/A for (i = 0; i < count; i++) {
2639N/A if (maps[i] == map) {
205N/A array_delete(&map->rec_map->maps, i, 1);
838N/A if (i == 0 && count == 1)
838N/A mail_index_record_map_free(map, map->rec_map);
205N/A return;
296N/A }
296N/A }
838N/A i_unreached();
296N/A}
296N/A
296N/Avoid mail_index_unmap(struct mail_index_map **_map)
296N/A{
296N/A struct mail_index_map *map = *_map;
296N/A
296N/A *_map = NULL;
1461N/A if (--map->refcount > 0)
296N/A return;
205N/A
838N/A i_assert(map->refcount == 0);
205N/A mail_index_record_map_unlink(map);
2639N/A
2639N/A if (map->extension_pool != NULL)
2639N/A pool_unref(&map->extension_pool);
48N/A if (array_is_created(&map->keyword_idx_map))
956N/A array_free(&map->keyword_idx_map);
2054N/A buffer_free(&map->hdr_copy_buf);
2054N/A i_free(map);
2054N/A}
2054N/A
2054N/Astatic void mail_index_map_copy_records(struct mail_index_record_map *dest,
2054N/A const struct mail_index_record_map *src,
2054N/A unsigned int record_size)
956N/A{
2073N/A size_t size;
2073N/A
2073N/A size = src->records_count * record_size;
2073N/A dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024));
2073N/A buffer_append(dest->buffer, src->records, size);
1386N/A
2073N/A dest->records = buffer_get_modifiable_data(dest->buffer, NULL);
2073N/A dest->records_count = src->records_count;
2073N/A
2073N/A /* if the map is ever written back to disk, we need to keep track of
2073N/A what has changed. */
2073N/A dest->write_seq_first = src->write_seq_first;
2073N/A dest->write_seq_last = src->write_seq_last;
2073N/A}
956N/A
2608N/Astatic void mail_index_map_copy_header(struct mail_index_map *dest,
2608N/A const struct mail_index_map *src)
2608N/A{
956N/A /* use src->hdr copy directly, because if we got here
2073N/A from syncing it has the latest changes. */
2073N/A dest->hdr = src->hdr;
2073N/A if (dest->hdr_copy_buf != NULL) {
2144N/A if (src == dest)
2144N/A return;
2144N/A
2073N/A buffer_set_used_size(dest->hdr_copy_buf, 0);
2073N/A } else {
2073N/A dest->hdr_copy_buf =
2073N/A buffer_create_dynamic(default_pool,
2073N/A dest->hdr.header_size);
2608N/A }
2608N/A buffer_append(dest->hdr_copy_buf, &dest->hdr,
2631N/A I_MIN(sizeof(dest->hdr), src->hdr.base_header_size));
2073N/A if (src != dest) {
956N/A buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size,
956N/A CONST_PTR_OFFSET(src->hdr_base,
2073N/A src->hdr.base_header_size),
2814N/A src->hdr.header_size - src->hdr.base_header_size);
956N/A }
956N/A dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
956N/A i_assert(dest->hdr_copy_buf->used == dest->hdr.header_size);
956N/A}
2073N/A
956N/Astatic struct mail_index_record_map *
967N/Amail_index_record_map_alloc(struct mail_index_map *map)
967N/A{
2631N/A struct mail_index_record_map *rec_map;
2073N/A
956N/A rec_map = i_new(struct mail_index_record_map, 1);
956N/A i_array_init(&rec_map->maps, 4);
956N/A array_append(&rec_map->maps, &map, 1);
2073N/A return rec_map;
1507N/A}
956N/A
956N/Astruct mail_index_map *mail_index_map_clone(const struct mail_index_map *map)
956N/A{
956N/A struct mail_index_map *mem_map;
967N/A struct mail_index_ext *extensions;
967N/A unsigned int i, count;
2631N/A
956N/A mem_map = i_new(struct mail_index_map, 1);
956N/A mem_map->index = map->index;
2631N/A mem_map->refcount = 1;
2631N/A if (map->rec_map == NULL) {
956N/A mem_map->rec_map = mail_index_record_map_alloc(mem_map);
2073N/A mem_map->rec_map->buffer =
2073N/A buffer_create_dynamic(default_pool, 1024);
2073N/A } else {
956N/A mem_map->rec_map = map->rec_map;
956N/A array_append(&mem_map->rec_map->maps, &mem_map, 1);
2073N/A }
956N/A
956N/A mail_index_map_copy_header(mem_map, map);
967N/A
967N/A mem_map->write_atomic = map->write_atomic;
967N/A mem_map->write_base_header = map->write_base_header;
967N/A mem_map->write_ext_header = map->write_ext_header;
967N/A
2639N/A /* copy extensions */
2639N/A if (array_is_created(&map->ext_id_map)) {
1507N/A count = array_count(&map->ext_id_map);
967N/A mail_index_map_init_extbufs(mem_map, count + 2);
1507N/A
956N/A array_append_array(&mem_map->extensions, &map->extensions);
1507N/A array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
956N/A
956N/A /* fix the name pointers to use our own pool */
956N/A extensions = array_get_modifiable(&mem_map->extensions, &count);
956N/A for (i = 0; i < count; i++) {
956N/A i_assert(extensions[i].record_offset +
967N/A extensions[i].record_size <=
956N/A mem_map->hdr.record_size);
956N/A extensions[i].name = p_strdup(mem_map->extension_pool,
2073N/A extensions[i].name);
956N/A }
967N/A }
2073N/A
956N/A /* copy keyword map */
1672N/A if (array_is_created(&map->keyword_idx_map)) {
967N/A i_array_init(&mem_map->keyword_idx_map,
956N/A array_count(&map->keyword_idx_map) + 4);
956N/A array_append_array(&mem_map->keyword_idx_map,
956N/A &map->keyword_idx_map);
1507N/A }
956N/A
956N/A return mem_map;
956N/A}
956N/A
2073N/Avoid mail_index_record_map_move_to_private(struct mail_index_map *map)
1386N/A{
2144N/A struct mail_index_record_map *new_map;
2144N/A
2144N/A if (array_count(&map->rec_map->maps) == 1)
956N/A return;
1507N/A
956N/A new_map = mail_index_record_map_alloc(map);
1386N/A mail_index_map_copy_records(new_map, map->rec_map,
1386N/A map->hdr.record_size);
1386N/A
1386N/A mail_index_record_map_unlink(map);
1386N/A map->rec_map = new_map;
956N/A}
956N/A
956N/Avoid mail_index_map_move_to_memory(struct mail_index_map *map)
1507N/A{
2073N/A struct mail_index_record_map *new_map;
1386N/A
2453N/A if (map->rec_map->mmap_base == NULL)
2453N/A return;
2453N/A
2453N/A i_assert(map->rec_map->lock_id != 0);
2453N/A
2453N/A new_map = array_count(&map->rec_map->maps) == 1 ? map->rec_map :
2453N/A mail_index_record_map_alloc(map);
2453N/A
2453N/A mail_index_map_copy_records(new_map, map->rec_map,
2453N/A map->hdr.record_size);
956N/A mail_index_map_copy_header(map, map);
2453N/A
2453N/A if (new_map != map->rec_map) {
2453N/A mail_index_record_map_unlink(map);
956N/A map->rec_map = new_map;
1352N/A } else {
2073N/A mail_index_unlock(map->index, &new_map->lock_id);
2073N/A if (munmap(new_map->mmap_base, new_map->mmap_size) < 0)
2073N/A mail_index_set_syscall_error(map->index, "munmap()");
2073N/A new_map->mmap_base = NULL;
2035N/A }
2035N/A}
2073N/A
2035N/Abool mail_index_map_get_ext_idx(struct mail_index_map *map,
2035N/A uint32_t ext_id, uint32_t *idx_r)
2035N/A{
2035N/A const uint32_t *id;
2073N/A
2035N/A if (!array_is_created(&map->ext_id_map) ||
2035N/A ext_id >= array_count(&map->ext_id_map))
2035N/A return FALSE;
2035N/A
2475N/A id = array_idx(&map->ext_id_map, ext_id);
2475N/A *idx_r = *id;
2475N/A return *idx_r != (uint32_t)-1;
2475N/A}
2035N/A