mail-index-map.c revision 7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071
76b43e4417bab52e913da39b5f5bc2a130d3f149Timo Sirainen/* Copyright (c) 2003-2008 Dovecot authors, see the included COPYING file */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "lib.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "array.h"
ffd9a1898a18fadfc5dce399162c25d50548f905Timo Sirainen#include "str-sanitize.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "nfs-workarounds.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mmap-util.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "read-full.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mail-index-private.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mail-index-sync-private.h"
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen#include "mail-index-modseq.h"
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen#include "mail-transaction-log-private.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int initial_count)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define EXTENSION_NAME_APPROX_LEN 20
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define EXT_GLOBAL_ALLOC_SIZE \
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define EXT_PER_ALLOC_SIZE \
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (EXTENSION_NAME_APPROX_LEN + \
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sizeof(struct mail_index_ext) + sizeof(uint32_t))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size_t size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (map->extension_pool == NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size = EXT_GLOBAL_ALLOC_SIZE +
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen initial_count * EXT_PER_ALLOC_SIZE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->extension_pool =
221351ed85c839e0b03d82c47654c3d17202e3dbTimo Sirainen pool_alloconly_create(MEMPOOL_GROWING"map extensions",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen nearest_power(size));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen p_clear(map->extension_pool);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* try to use the existing pool's size for initial_count so
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen we don't grow it unneededly */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size = p_get_max_easy_alloc_size(map->extension_pool);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen EXT_PER_ALLOC_SIZE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen p_array_init(&map->extensions, map->extension_pool, initial_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainenbool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name,
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen uint32_t *idx_r)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_ext *extensions;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int i, size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen if (array_is_created(&map->extensions)) {
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen extensions = array_get(&map->extensions, &size);
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen for (i = 0; i < size; i++) {
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen if (strcmp(extensions[i].name, name) == 0) {
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen if (idx_r != NULL)
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen *idx_r = i;
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen return TRUE;
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen }
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
8ababf3e7b15f793370d1dedf85825d38b42633fTimo Sirainenunsigned int mail_index_map_ext_hdr_offset(unsigned int name_len)
cfa9359fbd6a967ccdcd553c5e483a093885ab6fTimo Sirainen{
cfa9359fbd6a967ccdcd553c5e483a093885ab6fTimo Sirainen size_t size = sizeof(struct mail_index_ext_header) + name_len;
cfa9359fbd6a967ccdcd553c5e483a093885ab6fTimo Sirainen return MAIL_INDEX_HEADER_SIZE_ALIGN(size);
cfa9359fbd6a967ccdcd553c5e483a093885ab6fTimo Sirainen}
cfa9359fbd6a967ccdcd553c5e483a093885ab6fTimo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenuint32_t
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainenmail_index_map_register_ext(struct mail_index_map *map,
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen const char *name, uint32_t ext_offset,
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen const struct mail_index_ext_header *ext_hdr)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_ext *ext;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t idx, empty_idx = (uint32_t)-1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!array_is_created(&map->extensions)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_map_init_extbufs(map, 5);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen idx = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen idx = array_count(&map->extensions);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen i_assert(!mail_index_map_lookup_ext(map, name, NULL));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ext = array_append_space(&map->extensions);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ext->name = p_strdup(map->extension_pool, name);
cfa9359fbd6a967ccdcd553c5e483a093885ab6fTimo Sirainen ext->ext_offset = ext_offset;
8ababf3e7b15f793370d1dedf85825d38b42633fTimo Sirainen ext->hdr_offset = ext_offset +
8ababf3e7b15f793370d1dedf85825d38b42633fTimo Sirainen mail_index_map_ext_hdr_offset(strlen(name));
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext->hdr_size = ext_hdr->hdr_size;
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext->record_offset = ext_hdr->record_offset;
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext->record_size = ext_hdr->record_size;
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext->record_align = ext_hdr->record_align;
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext->reset_id = ext_hdr->reset_id;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext->index_idx = mail_index_ext_register(map->index, name,
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext_hdr->hdr_size,
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext_hdr->record_size,
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen ext_hdr->record_align);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* Update index ext_id -> map ext_id mapping. Fill non-used
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ext_ids with (uint32_t)-1 */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen while (array_count(&map->ext_id_map) < ext->index_idx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen array_append(&map->ext_id_map, &empty_idx, 1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return idx;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainenint mail_index_map_ext_get_next(struct mail_index_map *map,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen unsigned int *offset_p,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen const struct mail_index_ext_header **ext_hdr_r,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen const char **name_r)
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen{
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen const struct mail_index_ext_header *ext_hdr;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen unsigned int offset, name_offset;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen offset = *offset_p;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen *name_r = "";
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen /* Extension header contains:
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen - struct mail_index_ext_header
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen - name (not 0-terminated)
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen - 64bit alignment padding
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen - extension header contents
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen - 64bit alignment padding
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen */
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen name_offset = offset + sizeof(*ext_hdr);
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (offset + sizeof(*ext_hdr) >= map->hdr.header_size)
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return -1;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
8ababf3e7b15f793370d1dedf85825d38b42633fTimo Sirainen offset += mail_index_map_ext_hdr_offset(ext_hdr->name_size);
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (offset > map->hdr.header_size)
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return -1;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen *name_r = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen ext_hdr->name_size);
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (strcmp(*name_r, str_sanitize(*name_r, -1)) != 0) {
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen /* we allow only plain ASCII names, so this extension
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen is most likely broken */
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen *name_r = "";
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen }
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen /* finally make sure that the hdr_size is small enough.
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen do this last so that we could return a usable name. */
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (offset > map->hdr.header_size)
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return -1;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen *offset_p = offset;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen *ext_hdr_r = ext_hdr;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return 0;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen}
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainenint mail_index_map_ext_hdr_check(const struct mail_index_header *hdr,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen const struct mail_index_ext_header *ext_hdr,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen const char *name, const char **error_r)
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen{
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen (ext_hdr->record_align == 0 && ext_hdr->record_size != 0)) {
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen *error_r = "Invalid field values";
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return -1;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen }
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (*name == '\0') {
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen *error_r = "Broken name";
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return -1;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen }
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (ext_hdr->record_offset + ext_hdr->record_size > hdr->record_size) {
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen *error_r = t_strdup_printf("Record field points "
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen "outside record size (%u+%u > %u)",
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen ext_hdr->record_offset,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen ext_hdr->record_size,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen hdr->record_size);
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return -1;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen }
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (ext_hdr->record_size > 0 &&
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen (ext_hdr->record_offset % ext_hdr->record_align) != 0) {
9573f4283f9118315b0865998d43223fedee0246Timo Sirainen *error_r = t_strdup_printf("Record field alignment %u "
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen "not used", ext_hdr->record_align);
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return -1;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen }
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen /* if we get here from extension introduction, record_offset=0 and
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen hdr->record_size hasn't been updated yet */
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen if (ext_hdr->record_offset != 0 &&
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen (hdr->record_size % ext_hdr->record_align) != 0) {
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen *error_r = t_strdup_printf("Record size not aligned by %u "
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen "as required by extension",
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen ext_hdr->record_align);
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen return -1;
6c9699d37fbe4d8af3682ee7f341ede8d54faa87Timo Sirainen }
ed50658501d9ae8c85f6264831056b1debed11c3Timo Sirainen if (ext_hdr->hdr_size > MAIL_INDEX_EXT_HEADER_MAX_SIZE) {
ed50658501d9ae8c85f6264831056b1debed11c3Timo Sirainen *error_r = t_strdup_printf("Headersize too large (%u)",
ed50658501d9ae8c85f6264831056b1debed11c3Timo Sirainen ext_hdr->hdr_size);
ed50658501d9ae8c85f6264831056b1debed11c3Timo Sirainen return -1;
ed50658501d9ae8c85f6264831056b1debed11c3Timo Sirainen }
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen return 0;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen}
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainenstatic int mail_index_map_parse_extensions(struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_ext_header *ext_hdr;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen unsigned int i, old_count, offset;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen const char *name, *error;
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen uint32_t ext_id, ext_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* extension headers always start from 64bit offsets, so if base header
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen doesn't happen to be 64bit aligned we'll skip some bytes */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* nothing to do, skip allocatations and all */
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen old_count = array_count(&index->extensions);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_map_init_extbufs(map, old_count + 5);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ext_id = (uint32_t)-1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (i = 0; i < old_count; i++)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen array_append(&map->ext_id_map, &ext_id, 1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0; offset < map->hdr.header_size; i++) {
cfa9359fbd6a967ccdcd553c5e483a093885ab6fTimo Sirainen ext_offset = offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (mail_index_map_ext_get_next(map, &offset,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen &ext_hdr, &name) < 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen "Header extension #%d (%s) goes outside header",
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen index->filepath, i, name);
ffd9a1898a18fadfc5dce399162c25d50548f905Timo Sirainen return -1;
ffd9a1898a18fadfc5dce399162c25d50548f905Timo Sirainen }
ffd9a1898a18fadfc5dce399162c25d50548f905Timo Sirainen
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (mail_index_map_ext_hdr_check(&map->hdr, ext_hdr,
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen name, &error) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen "Broken extension #%d (%s): %s",
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen index->filepath, i, name, error);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (mail_index_map_lookup_ext(map, name, NULL)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen "Duplicate header extension %s",
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen index->filepath, name);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3675a7e9bd3775ba13fe8bc93915902513a0f1a4Timo Sirainen mail_index_map_register_ext(map, name, ext_offset, ext_hdr);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainenint mail_index_map_parse_keywords(struct mail_index_map *map)
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen{
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen struct mail_index *index = map->index;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen const struct mail_index_ext *ext;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen const struct mail_index_keyword_header *kw_hdr;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen const struct mail_index_keyword_header_rec *kw_rec;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen const char *name;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen unsigned int i, name_area_end_offset, old_count;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen uint32_t idx;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if (!mail_index_map_lookup_ext(map, "keywords", &idx)) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if (array_is_created(&map->keyword_idx_map))
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen array_clear(&map->keyword_idx_map);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return 0;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen ext = array_idx(&map->extensions, idx);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen /* Extension header contains:
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen - struct mail_index_keyword_header
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen - struct mail_index_keyword_header_rec * keywords_count
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen - const char names[] * keywords_count
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen */
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen i_assert(ext->hdr_offset < map->hdr.header_size);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen kw_rec = (const void *)(kw_hdr + 1);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen name = (const char *)(kw_rec + kw_hdr->keywords_count);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen array_count(&map->keyword_idx_map);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen /* Keywords can only be added into same mapping. Removing requires a
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen new mapping (recreating the index file) */
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if (kw_hdr->keywords_count == old_count) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen /* nothing changed */
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return 0;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen /* make sure the header is valid */
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if (kw_hdr->keywords_count < old_count) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen "Keywords removed unexpectedly",
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen index->filepath);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return -1;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen "keywords_count larger than header size",
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen index->filepath);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return -1;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen for (i = 0; i < kw_hdr->keywords_count; i++) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if (kw_rec[i].name_offset > name_area_end_offset) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen "name_offset points outside allocated header",
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen index->filepath);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return -1;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if (name[name_area_end_offset-1] != '\0') {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen "Keyword header doesn't end with NUL",
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen index->filepath);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return -1;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen /* create file -> index mapping */
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if (!array_is_created(&map->keyword_idx_map))
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen#ifdef DEBUG
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen /* Check that existing headers are still the same. It's behind DEBUG
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen since it's pretty useless waste of CPU normally. */
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen const char *keyword = name + kw_rec[i].name_offset;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen const unsigned int *old_idx;
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen unsigned int kw_idx;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen old_idx = array_idx(&map->keyword_idx_map, i);
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen if (!mail_index_keyword_lookup(index, keyword, &kw_idx) ||
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen kw_idx != *old_idx) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen "Keywords changed unexpectedly",
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen index->filepath);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return -1;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen#endif
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen /* Register the newly seen keywords */
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen i = array_count(&map->keyword_idx_map);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen for (; i < kw_hdr->keywords_count; i++) {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen const char *keyword = name + kw_rec[i].name_offset;
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen unsigned int kw_idx;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen if (*keyword == '\0') {
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen "Empty keyword name in header",
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen index->filepath);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return -1;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen mail_index_keyword_lookup_or_create(index, keyword, &kw_idx);
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen array_append(&map->keyword_idx_map, &kw_idx, 1);
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen }
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen return 0;
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen}
2bd2ca14d18819b77201ab2b46910d9ae6858abeTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic bool mail_index_check_header_compat(struct mail_index *index,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_index_header *hdr,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uoff_t file_size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen enum mail_index_header_compat_flags compat_flags = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#ifndef WORDS_BIGENDIAN
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#endif
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* major version change - handle silently(?) */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* we've already complained about it */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->compat_flags != compat_flags) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* architecture change */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_set_error(index, "Rebuilding index file %s: "
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "CPU architecture changed",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen index->filepath);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->header_size < hdr->base_header_size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "Corrupted header sizes (base %u, full %u)",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen index->filepath, hdr->base_header_size,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (hdr->header_size > file_size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "Corrupted header size (%u > %"PRIuUOFF_T")",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen index->filepath, hdr->header_size,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file_size);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen return FALSE;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen if (hdr->indexid != index->indexid) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen if (index->indexid != 0) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen mail_index_set_error(index, "Index file %s: "
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen "indexid changed: %u -> %u",
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen index->filepath, index->indexid,
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen hdr->indexid);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen index->indexid = hdr->indexid;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen mail_transaction_log_indexid_changed(index->log);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainenstatic void mail_index_map_clear_recent_flags(struct mail_index_map *map)
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen{
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen struct mail_index_record *rec;
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen unsigned int i;
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen for (i = 0; i < map->hdr.messages_count; i++) {
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen rec = MAIL_INDEX_MAP_IDX(map, i);
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen rec->flags &= ~MAIL_RECENT;
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen }
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen}
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainenint mail_index_map_check_header(struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_header *hdr = &map->hdr;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* following some extra checks that only take a bit of CPU */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->record_size < sizeof(struct mail_index_record)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "record_size too small: %u < %"PRIuSIZE_T,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index->filepath, hdr->record_size,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sizeof(struct mail_index_record));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
7eff76882487ef39803446f75a709a97a3cafc53Timo Sirainen if (hdr->uid_validity == 0 && hdr->next_uid != 1)
7eff76882487ef39803446f75a709a97a3cafc53Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->next_uid == 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen if (hdr->messages_count > map->rec_map->records_count)
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen if (hdr->seen_messages_count > hdr->messages_count ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->deleted_messages_count > hdr->messages_count)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen switch (hdr->minor_version) {
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen case 0:
3e559edb905bb57d1f1fbfda3431b339f45ea791Timo Sirainen /* upgrade silently from v1.0 */
608ae99c5b03989df263d72e49aa83e1f9d8a50eTimo Sirainen map->hdr.unused_old_recent_messages_count = 0;
608ae99c5b03989df263d72e49aa83e1f9d8a50eTimo Sirainen if (hdr->first_recent_uid == 0)
608ae99c5b03989df263d72e49aa83e1f9d8a50eTimo Sirainen map->hdr.first_recent_uid = 1;
608ae99c5b03989df263d72e49aa83e1f9d8a50eTimo Sirainen index->need_recreate = TRUE;
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen /* fall through */
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen case 1:
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen /* pre-v1.1.rc6: make sure the \Recent flags are gone */
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen mail_index_map_clear_recent_flags(map);
50031a6b36a6051512bd18f39e4bbabe54acf565Timo Sirainen map->hdr.minor_version = MAIL_INDEX_MINOR_VERSION;
3e559edb905bb57d1f1fbfda3431b339f45ea791Timo Sirainen }
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen if (hdr->first_recent_uid == 0 ||
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen hdr->first_recent_uid > hdr->next_uid ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->first_unseen_uid_lowwater > hdr->next_uid ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->first_deleted_uid_lowwater > hdr->next_uid)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (hdr->messages_count > 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* last message's UID must be smaller than next_uid.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen also make sure it's not zero. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_record *rec;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec = MAIL_INDEX_MAP_IDX(map, hdr->messages_count-1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (rec->uid == 0 || rec->uid >= hdr->next_uid)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void mail_index_map_copy_hdr(struct mail_index_map *map,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_header *hdr)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->base_header_size < sizeof(map->hdr)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* header smaller than ours, make a copy so our newer headers
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen won't have garbage in them */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memcpy(&map->hdr, hdr, hdr->base_header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr = *hdr;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
91add6e26ed2b91e15276c96ffe02d9f71476d71Timo Sirainen
91add6e26ed2b91e15276c96ffe02d9f71476d71Timo Sirainen /* FIXME: backwards compatibility, remove later. In case this index is
91add6e26ed2b91e15276c96ffe02d9f71476d71Timo Sirainen accessed with Dovecot v1.0, avoid recent message counter errors. */
91add6e26ed2b91e15276c96ffe02d9f71476d71Timo Sirainen map->hdr.unused_old_recent_messages_count = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenstatic int mail_index_mmap(struct mail_index_map *map, uoff_t file_size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mail_index *index = map->index;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_record_map *rec_map = map->rec_map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_header *hdr;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(rec_map->mmap_base == NULL);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen buffer_free(&rec_map->buffer);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (file_size > SSIZE_T_MAX) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* too large file to map into memory */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_set_error(index, "Index file too large: %s",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen index->filepath);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec_map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen MAP_PRIVATE, index->fd, 0);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (rec_map->mmap_base == MAP_FAILED) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec_map->mmap_base = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_syscall_error(index, "mmap()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec_map->mmap_size = file_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen hdr = rec_map->mmap_base;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (rec_map->mmap_size >
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen offsetof(struct mail_index_header, major_version) &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* major version change - handle silently */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (rec_map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "File too small (%"PRIuSIZE_T")",
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen index->filepath, rec_map->mmap_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* Can't use this file */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec_map->mmap_used_size = hdr->header_size +
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->messages_count * hdr->record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen if (rec_map->mmap_used_size <= rec_map->mmap_size)
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen rec_map->records_count = hdr->messages_count;
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen else {
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen rec_map->records_count =
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen (rec_map->mmap_size - hdr->header_size) /
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->record_size;
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen rec_map->mmap_used_size = hdr->header_size +
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen rec_map->records_count * hdr->record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "messages_count too large (%u > %u)",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index->filepath, hdr->messages_count,
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen rec_map->records_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_map_copy_hdr(map, hdr);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->hdr_base = rec_map->mmap_base;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec_map->records = PTR_OFFSET(rec_map->mmap_base, map->hdr.header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mail_index_read_header(struct mail_index *index,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen void *buf, size_t buf_size, size_t *pos_r)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen size_t pos;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen int ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen memset(buf, 0, sizeof(struct mail_index_header));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* try to read the whole header, but it's not necessarily an error to
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen read less since the older versions of the index format could be
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen smaller. Request reading up to buf_size, but accept if we only got
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen the header. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen pos = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen do {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = pread(index->fd, PTR_OFFSET(buf, pos),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buf_size - pos, pos);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret > 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen pos += ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen } while (ret > 0 && pos < sizeof(struct mail_index_header));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen *pos_r = pos;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenmail_index_try_read_map(struct mail_index_map *map,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uoff_t file_size, bool *retry_r, bool try_retry)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_header *hdr;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned char read_buf[4096];
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const void *buf;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen void *data = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ssize_t ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen size_t pos, records_size, initial_buf_pos = 0;
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen unsigned int records_count = 0, extra;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(map->rec_map->mmap_base == NULL);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *retry_r = FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buf = read_buf; hdr = buf;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* major version change - handle silently */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (ret > 0 || pos >= hdr->base_header_size)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!mail_index_check_header_compat(index, hdr, file_size)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* Can't use this file */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen initial_buf_pos = pos;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (pos > hdr->header_size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen pos = hdr->header_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* place the base header into memory. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen buffer_reset(map->hdr_copy_buf);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen buffer_append(map->hdr_copy_buf, buf, pos);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (pos != hdr->header_size) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* @UNSAFE: read the rest of the header into memory */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen data = buffer_append_space_unsafe(map->hdr_copy_buf,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->header_size -
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen pos);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ret = pread_full(index->fd, data,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->header_size - pos, pos);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret > 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* header read, read the records now. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen records_size = (size_t)hdr->messages_count * hdr->record_size;
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen records_count = hdr->messages_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (file_size - hdr->header_size < records_size ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (hdr->record_size != 0 &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen records_size / hdr->record_size != hdr->messages_count)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen records_count = (file_size - hdr->header_size) /
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->record_size;
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen records_size = (size_t)records_count * hdr->record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "messages_count too large (%u > %u)",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index->filepath, hdr->messages_count,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen records_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (map->rec_map->buffer == NULL) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map->buffer =
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen buffer_create_dynamic(default_pool,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen records_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* @UNSAFE */
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen buffer_set_used_size(map->rec_map->buffer, 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (initial_buf_pos <= hdr->header_size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen extra = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen extra = initial_buf_pos - hdr->header_size;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen buffer_append(map->rec_map->buffer,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen CONST_PTR_OFFSET(buf, hdr->header_size),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen extra);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (records_size > extra) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen data = buffer_append_space_unsafe(map->rec_map->buffer,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen records_size - extra);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = pread_full(index->fd, data, records_size - extra,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->header_size + extra);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (errno == ESTALE && try_retry) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* a new index file was renamed over this one. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *retry_r = TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_syscall_error(index, "pread_full()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_error(index,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "Corrupted index file %s: File too small",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index->filepath);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map->records =
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen buffer_get_modifiable_data(map->rec_map->buffer, NULL);
2aa3ac8bfb8fea49fd0ab6d56afcb3af3375d22cTimo Sirainen map->rec_map->records_count = records_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_map_copy_hdr(map, hdr);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr_base = map->hdr_copy_buf->data;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
c2feb7d13482d0f60691cd71d06d42a80df99397Timo Sirainenstatic int mail_index_read_map(struct mail_index_map *map, uoff_t file_size,
c2feb7d13482d0f60691cd71d06d42a80df99397Timo Sirainen unsigned int *lock_id)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_lost_handler_t *const *handlers;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct stat st;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int i, count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen int ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen bool try_retry, retry;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* notify all "sync lost" handlers */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen handlers = array_get(&index->sync_lost_handlers, &count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (i = 0; i < count; i++)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (*handlers[i])(index);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (i = 0;; i++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (file_size == (uoff_t)-1) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* fstat() below failed */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen retry = try_retry;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen } else {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen ret = mail_index_try_read_map(map, file_size,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen &retry, try_retry);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret != 0 || !retry)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* ESTALE - reopen index file */
c2feb7d13482d0f60691cd71d06d42a80df99397Timo Sirainen mail_index_close_file(index);
c2feb7d13482d0f60691cd71d06d42a80df99397Timo Sirainen *lock_id = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ret = mail_index_try_open_only(index);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret <= 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* the file was lost */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen errno = ENOENT;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_set_syscall_error(index, "open()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
c2feb7d13482d0f60691cd71d06d42a80df99397Timo Sirainen if (mail_index_lock_shared(index, lock_id) < 0)
c2feb7d13482d0f60691cd71d06d42a80df99397Timo Sirainen return -1;
c2feb7d13482d0f60691cd71d06d42a80df99397Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (fstat(index->fd, &st) == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file_size = st.st_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (errno != ESTALE) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_set_syscall_error(index, "fstat()");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file_size = (uoff_t)-1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void mail_index_header_init(struct mail_index *index,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_header *hdr)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen memset(hdr, 0, sizeof(*hdr));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->base_header_size = sizeof(*hdr);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->header_size = sizeof(*hdr);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->record_size = sizeof(struct mail_index_record);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen#ifndef WORDS_BIGENDIAN
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen#endif
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->indexid = index->indexid;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->log_file_seq = 1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen hdr->next_uid = 1;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen hdr->first_recent_uid = 1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstruct mail_index_map *mail_index_map_alloc(struct mail_index *index)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_map tmp_map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen memset(&tmp_map, 0, sizeof(tmp_map));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_header_init(index, &tmp_map.hdr);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen tmp_map.index = index;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen tmp_map.hdr_base = &tmp_map.hdr;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* a bit kludgy way to do this, but it initializes everything
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen nicely and correctly */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return mail_index_map_clone(&tmp_map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen/* returns -1 = error, 0 = index files are unusable,
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen 1 = index files are usable or at least repairable */
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainenstatic int mail_index_map_latest_file(struct mail_index *index)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen struct mail_index_map *old_map, *new_map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct stat st;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen unsigned int lock_id;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uoff_t file_size;
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen bool use_mmap, unusable = FALSE;
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen int ret, try;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_index_reopen_if_changed(index);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret <= 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* the index file is lost/broken. let's hope that we can
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen build it from the transaction log. */
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen return 1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* the index file is still open, lock it */
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if (mail_index_lock_shared(index, &lock_id) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen if (index->nfs_flush)
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen nfs_flush_attr_cache_fd_locked(index->filepath, index->fd);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (fstat(index->fd, &st) == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file_size = st.st_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (errno != ESTALE) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_set_syscall_error(index, "fstat()");
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen mail_index_unlock(index, &lock_id);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file_size = (uoff_t)-1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* mmaping seems to be slower than just reading the file, so even if
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mmap isn't disabled don't use it unless the file is large enough */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen use_mmap = !index->mmap_disable && file_size != (uoff_t)-1 &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file_size > MAIL_INDEX_MMAP_MIN_SIZE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen new_map = mail_index_map_alloc(index);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if (use_mmap) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen new_map->rec_map->lock_id = lock_id;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen ret = mail_index_mmap(new_map, file_size);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen } else {
c2feb7d13482d0f60691cd71d06d42a80df99397Timo Sirainen ret = mail_index_read_map(new_map, file_size, &lock_id);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen mail_index_unlock(index, &lock_id);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen }
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen if (ret == 0) {
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen /* the index files are unusable */
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen unusable = TRUE;
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen }
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen
3b80595fcf2001cf7b2fcc6290823e38f4a142fcTimo Sirainen for (try = 0; ret > 0; try++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* make sure the header is ok before using this mapping */
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen ret = mail_index_map_check_header(new_map);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen if (ret > 0) T_BEGIN {
746d35bf3dba3ae5ddbcecb9732f60d5e9de77efTimo Sirainen if (mail_index_map_parse_extensions(new_map) < 0)
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen ret = 0;
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen else if (mail_index_map_parse_keywords(new_map) < 0)
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen ret = 0;
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen } T_END;
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen if (ret != 0 || try == 2) {
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen if (ret < 0) {
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen unusable = TRUE;
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen ret = 0;
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen }
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen break;
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen }
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen /* fsck and try again */
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen old_map = index->map;
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen index->map = new_map;
180503c08ebd58174c4f7f1652d716266877e2a3Timo Sirainen if (mail_index_fsck(index) < 0) {
180503c08ebd58174c4f7f1652d716266877e2a3Timo Sirainen ret = -1;
180503c08ebd58174c4f7f1652d716266877e2a3Timo Sirainen break;
180503c08ebd58174c4f7f1652d716266877e2a3Timo Sirainen }
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen
180503c08ebd58174c4f7f1652d716266877e2a3Timo Sirainen /* fsck replaced the map */
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen new_map = index->map;
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen index->map = old_map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret <= 0) {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen mail_index_unmap(&new_map);
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen return ret < 0 ? -1 : (unusable ? 0 : 1);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(new_map->rec_map->records != NULL);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen index->last_read_log_file_seq = new_map->hdr.log_file_seq;
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen index->last_read_log_file_head_offset =
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen new_map->hdr.log_file_head_offset;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen index->last_read_log_file_tail_offset =
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen new_map->hdr.log_file_tail_offset;
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen index->last_read_stat = st;
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen mail_index_unmap(&index->map);
1fd0d511885c30028aba388588151acf4ee85e75Timo Sirainen index->map = new_map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainenint mail_index_map(struct mail_index *index,
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen enum mail_index_sync_handler_type type)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen int ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen i_assert(index->lock_type != F_WRLCK);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(!index->mapping);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen index->mapping = TRUE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen if (index->map == NULL)
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen index->map = mail_index_map_alloc(index);
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* first try updating the existing mapping from transaction log. */
38efb29cb9453917c1b642095a37d0e901f56eeeTimo Sirainen if (index->map->hdr.indexid != 0 && index->indexid != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* we're not creating the index, or opening transaction log.
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync this as a view from transaction log. */
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen ret = mail_index_sync_map(&index->map, type, FALSE);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen } else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret == 0) {
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen /* try to open and read the latest index. if it fails, we'll
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen fallback to updating the existing mapping from transaction
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen logs (which we'll also do even if the reopening succeeds).
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen if index files are unusable (e.g. major version change)
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen don't even try to use the transaction log. */
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen if (mail_index_map_latest_file(index) == 0) {
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen /* make sure we don't try to open the file again */
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen if (unlink(index->filepath) < 0 && errno != ENOENT)
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen mail_index_set_syscall_error(index, "unlink()");
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen } else {
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen /* if we're creating the index file, we don't have any
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen logs yet */
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen if (index->log->head != NULL && index->indexid != 0) {
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen /* and update the map with the latest changes
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen from transaction log */
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen ret = mail_index_sync_map(&index->map, type,
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen TRUE);
9a0c37cb8588cd287106374eda22a7ea3e2742b2Timo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen index->mapping = FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenstatic void mail_index_record_map_free(struct mail_index_map *map,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_record_map *rec_map)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
88187ee880b4829443e0d55ea7d145d9d5880217Timo Sirainen if (rec_map->lock_id != 0)
88187ee880b4829443e0d55ea7d145d9d5880217Timo Sirainen mail_index_unlock(map->index, &rec_map->lock_id);
88187ee880b4829443e0d55ea7d145d9d5880217Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (rec_map->buffer != NULL) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(rec_map->mmap_base == NULL);
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen buffer_free(&rec_map->buffer);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen } else if (rec_map->mmap_base != NULL) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(rec_map->buffer == NULL);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (munmap(rec_map->mmap_base, rec_map->mmap_size) < 0)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_set_syscall_error(map->index, "munmap()");
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec_map->mmap_base = NULL;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen array_free(&rec_map->maps);
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen if (rec_map->modseq != NULL)
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen mail_index_map_modseq_free(&rec_map->modseq);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_free(rec_map);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenstatic void mail_index_record_map_unlink(struct mail_index_map *map)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_map *const *maps;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen unsigned int i, count;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen maps = array_get(&map->rec_map->maps, &count);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen for (i = 0; i < count; i++) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (maps[i] == map) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen array_delete(&map->rec_map->maps, i, 1);
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen if (i == 0 && count == 1) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_record_map_free(map, map->rec_map);
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen map->rec_map = NULL;
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen return;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_unreached();
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenvoid mail_index_unmap(struct mail_index_map **_map)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_map *map = *_map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen *_map = NULL;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (--map->refcount > 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(map->refcount == 0);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_record_map_unlink(map);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (map->extension_pool != NULL)
648d24583c1574441c4fa0331a90bd4d6e7996c5Timo Sirainen pool_unref(&map->extension_pool);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (array_is_created(&map->keyword_idx_map))
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen array_free(&map->keyword_idx_map);
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen buffer_free(&map->hdr_copy_buf);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_free(map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenstatic void mail_index_map_copy_records(struct mail_index_record_map *dest,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_index_record_map *src,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen unsigned int record_size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen size_t size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen size = src->records_count * record_size;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_append(dest->buffer, src->records, size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dest->records = buffer_get_modifiable_data(dest->buffer, NULL);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dest->records_count = src->records_count;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* if the map is ever written back to disk, we need to keep track of
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen what has changed. */
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen dest->write_seq_first = src->write_seq_first;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen dest->write_seq_last = src->write_seq_last;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenstatic void mail_index_map_copy_header(struct mail_index_map *dest,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_index_map *src)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* use src->hdr copy directly, because if we got here
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen from syncing it has the latest changes. */
a85d0d2b75e5effadf4d3c2167af0ff9ae95904aTimo Sirainen if (src != dest)
a85d0d2b75e5effadf4d3c2167af0ff9ae95904aTimo Sirainen dest->hdr = src->hdr;
5787e39e2be32f657b8c98fee8bac794aa852cf8Timo Sirainen if (dest->hdr_copy_buf != NULL) {
5787e39e2be32f657b8c98fee8bac794aa852cf8Timo Sirainen if (src == dest)
5787e39e2be32f657b8c98fee8bac794aa852cf8Timo Sirainen return;
5787e39e2be32f657b8c98fee8bac794aa852cf8Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_set_used_size(dest->hdr_copy_buf, 0);
5787e39e2be32f657b8c98fee8bac794aa852cf8Timo Sirainen } else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dest->hdr_copy_buf =
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_create_dynamic(default_pool,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dest->hdr.header_size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen buffer_append(dest->hdr_copy_buf, &dest->hdr,
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen I_MIN(sizeof(dest->hdr), src->hdr.base_header_size));
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen if (src != dest) {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size,
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen CONST_PTR_OFFSET(src->hdr_base,
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen src->hdr.base_header_size),
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen src->hdr.header_size - src->hdr.base_header_size);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
5787e39e2be32f657b8c98fee8bac794aa852cf8Timo Sirainen i_assert(dest->hdr_copy_buf->used == dest->hdr.header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenstatic struct mail_index_record_map *
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenmail_index_record_map_alloc(struct mail_index_map *map)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_record_map *rec_map;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec_map = i_new(struct mail_index_record_map, 1);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_array_init(&rec_map->maps, 4);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen array_append(&rec_map->maps, &map, 1);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen return rec_map;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstruct mail_index_map *mail_index_map_clone(const struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *mem_map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_ext *extensions;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int i, count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mem_map = i_new(struct mail_index_map, 1);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen mem_map->index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mem_map->refcount = 1;
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen if (map->rec_map == NULL) {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mem_map->rec_map = mail_index_record_map_alloc(mem_map);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mem_map->rec_map->buffer =
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen buffer_create_dynamic(default_pool, 1024);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen } else {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mem_map->rec_map = map->rec_map;
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen array_append(&mem_map->rec_map->maps, &mem_map, 1);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_map_copy_header(mem_map, map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mem_map->write_atomic = map->write_atomic;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mem_map->write_base_header = map->write_base_header;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mem_map->write_ext_header = map->write_ext_header;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* copy extensions */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (array_is_created(&map->ext_id_map)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen count = array_count(&map->ext_id_map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_map_init_extbufs(mem_map, count + 2);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen array_append_array(&mem_map->extensions, &map->extensions);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* fix the name pointers to use our own pool */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen extensions = array_get_modifiable(&mem_map->extensions, &count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (i = 0; i < count; i++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(extensions[i].record_offset +
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen extensions[i].record_size <=
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mem_map->hdr.record_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen extensions[i].name = p_strdup(mem_map->extension_pool,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen extensions[i].name);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
d051664df497582e1eb75a9f238d04b65e858db8Timo Sirainen /* copy keyword map */
d051664df497582e1eb75a9f238d04b65e858db8Timo Sirainen if (array_is_created(&map->keyword_idx_map)) {
d051664df497582e1eb75a9f238d04b65e858db8Timo Sirainen i_array_init(&mem_map->keyword_idx_map,
d051664df497582e1eb75a9f238d04b65e858db8Timo Sirainen array_count(&map->keyword_idx_map) + 4);
d051664df497582e1eb75a9f238d04b65e858db8Timo Sirainen array_append_array(&mem_map->keyword_idx_map,
d051664df497582e1eb75a9f238d04b65e858db8Timo Sirainen &map->keyword_idx_map);
d051664df497582e1eb75a9f238d04b65e858db8Timo Sirainen }
d051664df497582e1eb75a9f238d04b65e858db8Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return mem_map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainenvoid mail_index_record_map_move_to_private(struct mail_index_map *map)
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen{
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen struct mail_index_record_map *new_map;
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen const struct mail_index_record *rec;
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen if (array_count(&map->rec_map->maps) > 1) {
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen new_map = mail_index_record_map_alloc(map);
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen mail_index_map_copy_records(new_map, map->rec_map,
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen map->hdr.record_size);
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen mail_index_record_map_unlink(map);
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen map->rec_map = new_map;
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen } else {
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen new_map = map->rec_map;
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen }
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen if (map->rec_map->modseq != NULL) {
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen new_map->modseq =
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen mail_index_map_modseq_clone(map->rec_map->modseq);
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen }
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen if (new_map->records_count != map->hdr.messages_count) {
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen new_map->records_count = map->hdr.messages_count;
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen if (new_map->records_count == 0)
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen new_map->last_appended_uid = 0;
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen else {
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen rec = MAIL_INDEX_MAP_IDX(map, new_map->records_count-1);
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen new_map->last_appended_uid = rec->uid;
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen }
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen buffer_set_used_size(new_map->buffer, new_map->records_count *
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen map->hdr.record_size);
d0ab5936be1cd971007fc2241e0be120c442cb84Timo Sirainen }
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen}
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenvoid mail_index_map_move_to_memory(struct mail_index_map *map)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_record_map *new_map;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (map->rec_map->mmap_base == NULL)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(map->rec_map->lock_id != 0);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen if (array_count(&map->rec_map->maps) == 1)
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen new_map = map->rec_map;
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen else {
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen new_map = mail_index_record_map_alloc(map);
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen new_map->modseq = map->rec_map->modseq == NULL ? NULL :
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen mail_index_map_modseq_clone(map->rec_map->modseq);
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_map_copy_records(new_map, map->rec_map,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->hdr.record_size);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_map_copy_header(map, map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (new_map != map->rec_map) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_record_map_unlink(map);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map = new_map;
88187ee880b4829443e0d55ea7d145d9d5880217Timo Sirainen } else {
88187ee880b4829443e0d55ea7d145d9d5880217Timo Sirainen mail_index_unlock(map->index, &new_map->lock_id);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (munmap(new_map->mmap_base, new_map->mmap_size) < 0)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_set_syscall_error(map->index, "munmap()");
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen new_map->mmap_base = NULL;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
12053b7b4b57dbd2790057426d1633988eedad56Timo Sirainenbool mail_index_map_get_ext_idx(struct mail_index_map *map,
12053b7b4b57dbd2790057426d1633988eedad56Timo Sirainen uint32_t ext_id, uint32_t *idx_r)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const uint32_t *id;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!array_is_created(&map->ext_id_map) ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ext_id >= array_count(&map->ext_id_map))
12053b7b4b57dbd2790057426d1633988eedad56Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen id = array_idx(&map->ext_id_map, ext_id);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *idx_r = *id;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return *idx_r != (uint32_t)-1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}