mail-index-map.c revision 746d35bf3dba3ae5ddbcecb9732f60d5e9de77ef
2454dfa32c93c20a8522c6ed42fe057baaac9f9aStephan Bosch/* Copyright (C) 2003-2007 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
16f816d3f3c32ae3351834253f52ddd0212bcbf3Timo Sirainen#include "array.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "str-sanitize.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "nfs-workarounds.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "mmap-util.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "read-full.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-private.h"
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen#include "mail-index-sync-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-transaction-log-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e9594e86dc601b72c1636f2b901dcfbf4ffaf47fAki Tuomistatic void mail_index_map_init_extbufs(struct mail_index_map *map,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int initial_count)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#define EXTENSION_NAME_APPROX_LEN 20
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen#define EXT_GLOBAL_ALLOC_SIZE \
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#define EXT_PER_ALLOC_SIZE \
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (EXTENSION_NAME_APPROX_LEN + \
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen sizeof(struct mail_index_ext) + sizeof(uint32_t))
e8fd7988ec183fb6c104aed19a61f1a096c51d34Timo Sirainen size_t size;
e8fd7988ec183fb6c104aed19a61f1a096c51d34Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (map->extension_pool == NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen size = EXT_GLOBAL_ALLOC_SIZE +
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen initial_count * EXT_PER_ALLOC_SIZE;
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen map->extension_pool =
fafa6393128960c70a9979af1c23cea41027fdd1Josef 'Jeff' Sipek pool_alloconly_create("map extensions",
0b2c958d1cdcbeb46c2ce7ada0917b304ad89dc1Timo Sirainen nearest_power(size));
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen } else {
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen p_clear(map->extension_pool);
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* try to use the existing pool's size for initial_count so
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen we don't grow it unneededly */
ad850190d946d34966a56838cfdb216e021b5b5fTimo Sirainen size = p_get_max_easy_alloc_size(map->extension_pool);
ad850190d946d34966a56838cfdb216e021b5b5fTimo Sirainen if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen EXT_PER_ALLOC_SIZE;
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen }
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen }
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen p_array_init(&map->extensions, map->extension_pool, initial_count);
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainenbool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen uint32_t *idx_r)
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen{
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen const struct mail_index_ext *extensions;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen unsigned int i, size;
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (array_is_created(&map->extensions)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen extensions = array_get(&map->extensions, &size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (i = 0; i < size; i++) {
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi if (strcmp(extensions[i].name, name) == 0) {
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi if (idx_r != NULL)
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi *idx_r = i;
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi return TRUE;
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi }
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi }
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi }
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi return FALSE;
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi}
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomistatic size_t get_ext_size(size_t name_len)
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi{
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi size_t size = sizeof(struct mail_index_ext_header) + name_len;
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi return MAIL_INDEX_HEADER_SIZE_ALIGN(size);
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi}
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomiuint32_t
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomimail_index_map_register_ext(struct mail_index_map *map, const char *name,
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi uint32_t ext_offset, uint32_t hdr_size,
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi uint32_t record_offset, uint32_t record_size,
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi uint32_t record_align, uint32_t reset_id)
846624bdec0d5deec6cf22071158221391ec5f2dAki Tuomi{
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct mail_index_ext *ext;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t idx, empty_idx = (uint32_t)-1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (!array_is_created(&map->extensions)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_map_init_extbufs(map, 5);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen idx = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen idx = array_count(&map->extensions);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
5bd1c1d4fe3265d5e6b6054044fd6d78e42c9d0aTimo Sirainen i_assert(!mail_index_map_lookup_ext(map, name, NULL));
5bd1c1d4fe3265d5e6b6054044fd6d78e42c9d0aTimo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen ext = array_append_space(&map->extensions);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext->name = p_strdup(map->extension_pool, name);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext->ext_offset = ext_offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext->hdr_offset = ext_offset + get_ext_size(strlen(name));
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen ext->hdr_size = hdr_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext->record_offset = record_offset;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen ext->record_size = record_size;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen ext->record_align = record_align;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen ext->reset_id = reset_id;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext->index_idx = mail_index_ext_register(map->index, name, hdr_size,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen record_size, record_align);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* Update index ext_id -> map ext_id mapping. Fill non-used
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext_ids with (uint32_t)-1 */
f330b7d9e14255fc06bc82908d9bc5a12cccb424Timo Sirainen while (array_count(&map->ext_id_map) < ext->index_idx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen array_append(&map->ext_id_map, &empty_idx, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen return idx;
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen}
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainenint mail_index_map_ext_get_next(struct mail_index_map *map,
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen unsigned int *offset_p,
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen const struct mail_index_ext_header **ext_hdr_r,
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen const char **name_r)
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen{
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen const struct mail_index_ext_header *ext_hdr;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen unsigned int offset, name_offset;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen offset = *offset_p;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen *name_r = "";
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen /* Extension header contains:
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen - struct mail_index_ext_header
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen - name (not 0-terminated)
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen - 64bit alignment padding
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen - extension header contents
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen - 64bit alignment padding
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen */
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen name_offset = offset + sizeof(*ext_hdr);
f330b7d9e14255fc06bc82908d9bc5a12cccb424Timo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen if (offset + sizeof(*ext_hdr) >= map->hdr.header_size)
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen return -1;
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen offset += get_ext_size(ext_hdr->name_size);
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen if (offset > map->hdr.header_size)
f330b7d9e14255fc06bc82908d9bc5a12cccb424Timo Sirainen return -1;
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen *name_r = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen ext_hdr->name_size);
f330b7d9e14255fc06bc82908d9bc5a12cccb424Timo Sirainen if (strcmp(*name_r, str_sanitize(*name_r, -1)) != 0) {
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen /* we allow only plain ASCII names, so this extension
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen is most likely broken */
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen *name_r = "";
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen }
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen /* finally make sure that the hdr_size is small enough.
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen do this last so that we could return a usable name. */
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen if (offset > map->hdr.header_size)
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen return -1;
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen
f330b7d9e14255fc06bc82908d9bc5a12cccb424Timo Sirainen *offset_p = offset;
f330b7d9e14255fc06bc82908d9bc5a12cccb424Timo Sirainen *ext_hdr_r = ext_hdr;
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen return 0;
f330b7d9e14255fc06bc82908d9bc5a12cccb424Timo Sirainen}
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainenint mail_index_map_ext_hdr_check(const struct mail_index_header *hdr,
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen const struct mail_index_ext_header *ext_hdr,
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen const char *name, const char **error_r)
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen{
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen (ext_hdr->record_align == 0 && ext_hdr->record_size != 0)) {
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen *error_r = "Invalid field values";
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen return -1;
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen }
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen if (*name == '\0') {
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen *error_r = "Broken name";
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen return -1;
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen }
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen if (ext_hdr->record_offset + ext_hdr->record_size > hdr->record_size) {
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen *error_r = t_strdup_printf("Record field points "
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen "outside record size (%u+%u > %u)",
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen ext_hdr->record_offset,
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen ext_hdr->record_size,
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen hdr->record_size);
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen return -1;
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen }
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen if (ext_hdr->record_size > 0 &&
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen (hdr->record_size % ext_hdr->record_align) != 0)) {
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen *error_r = t_strdup_printf("Record field alignmentation %u "
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen "not used", ext_hdr->record_align);
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen return -1;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen }
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen return 0;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen}
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainenstatic int mail_index_map_parse_extensions(struct mail_index_map *map)
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen{
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen struct mail_index *index = map->index;
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen const struct mail_index_ext_header *ext_hdr;
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen unsigned int i, old_count, offset;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen const char *name, *error;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen uint32_t ext_id, ext_offset;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen
85da8c055280cd45553b6b335e9fb226d6e2801eTimo Sirainen /* extension headers always start from 64bit offsets, so if base header
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen doesn't happen to be 64bit aligned we'll skip some bytes */
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen /* nothing to do, skip allocatations and all */
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen return 0;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen }
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen old_count = array_count(&index->extensions);
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen mail_index_map_init_extbufs(map, old_count + 5);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen ext_id = (uint32_t)-1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen for (i = 0; i < old_count; i++)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen array_append(&map->ext_id_map, &ext_id, 1);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen for (i = 0; offset < map->hdr.header_size; i++) {
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen ext_offset = offset;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen t_push();
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (mail_index_map_ext_get_next(map, &offset,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen &ext_hdr, &name) < 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen "Header extension #%d (%s) goes outside header",
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen index->filepath, i, name);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen t_pop();
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (mail_index_map_ext_hdr_check(&map->hdr, ext_hdr,
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen name, &error) < 0) {
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen "Broken extension #%d (%s): %s",
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen index->filepath, i, name, error);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen t_pop();
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen return -1;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (mail_index_map_lookup_ext(map, name, NULL)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "Duplicate header extension %s",
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen index->filepath, name);
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen t_pop();
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen return -1;
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen }
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen mail_index_map_register_ext(map, name, ext_offset,
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen ext_hdr->hdr_size,
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen ext_hdr->record_offset,
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen ext_hdr->record_size,
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen ext_hdr->record_align,
620b5ed41650da63b0ba15c489f9f312231d5d9bTimo Sirainen ext_hdr->reset_id);
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen t_pop();
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen }
67bbcd664bebce9a507a49c67273be4814d07c97Timo Sirainen return 0;
67bbcd664bebce9a507a49c67273be4814d07c97Timo Sirainen}
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainenint mail_index_map_parse_keywords(struct mail_index_map *map)
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen{
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen struct mail_index *index = map->index;
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen const struct mail_index_ext *ext;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen const struct mail_index_keyword_header *kw_hdr;
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen const struct mail_index_keyword_header_rec *kw_rec;
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen const char *name;
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen unsigned int i, name_area_end_offset, old_count;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen uint32_t idx;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen if (!mail_index_map_lookup_ext(map, "keywords", &idx)) {
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen if (array_is_created(&map->keyword_idx_map))
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen array_clear(&map->keyword_idx_map);
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen return 0;
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen }
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen ext = array_idx(&map->extensions, idx);
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen /* Extension header contains:
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen - struct mail_index_keyword_header
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen - struct mail_index_keyword_header_rec * keywords_count
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen - const char names[] * keywords_count
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen */
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen i_assert(ext->hdr_offset < map->hdr.header_size);
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen kw_rec = (const void *)(kw_hdr + 1);
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen name = (const char *)(kw_rec + kw_hdr->keywords_count);
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
67bbcd664bebce9a507a49c67273be4814d07c97Timo Sirainen array_count(&map->keyword_idx_map);
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen /* Keywords can only be added into same mapping. Removing requires a
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen new mapping (recreating the index file) */
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen if (kw_hdr->keywords_count == old_count) {
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen /* nothing changed */
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen return 0;
67bbcd664bebce9a507a49c67273be4814d07c97Timo Sirainen }
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen /* make sure the header is valid */
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen if (kw_hdr->keywords_count < old_count) {
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen "Keywords removed unexpectedly",
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen index->filepath);
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen return -1;
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen }
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
67bbcd664bebce9a507a49c67273be4814d07c97Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen "keywords_count larger than header size",
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen index->filepath);
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen return -1;
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen }
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen for (i = 0; i < kw_hdr->keywords_count; i++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (kw_rec[i].name_offset > name_area_end_offset) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "name_offset points outside allocated header",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen index->filepath);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
920b9f0fdfa5a5d7763e05736601a31bcb291a53Timo Sirainen if (name[name_area_end_offset-1] != '\0') {
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen "Keyword header doesn't end with NUL",
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen index->filepath);
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen return -1;
e192a3b1ca8ae857e7d87298ea507d32977ba570Timo Sirainen }
e192a3b1ca8ae857e7d87298ea507d32977ba570Timo Sirainen
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen /* create file -> index mapping */
e192a3b1ca8ae857e7d87298ea507d32977ba570Timo Sirainen if (!array_is_created(&map->keyword_idx_map))
811f2e26d9782d9cb99fdf82e18ffa0a77564fe2Timo Sirainen i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen
f501ad38c51cf1d8f4f84313922c785e6ae6e81fTimo Sirainen#ifdef DEBUG
c4877db8b6559846f4b58be8e42422dc734c193fTimo Sirainen /* Check that existing headers are still the same. It's behind DEBUG
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen since it's pretty useless waste of CPU normally. */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
f501ad38c51cf1d8f4f84313922c785e6ae6e81fTimo Sirainen const unsigned int *old_idx;
8b9342aa96b2f297e23afb261f9f7dd859800952Timo Sirainen unsigned int idx;
8b9342aa96b2f297e23afb261f9f7dd859800952Timo Sirainen
8b9342aa96b2f297e23afb261f9f7dd859800952Timo Sirainen old_idx = array_idx(&map->keyword_idx_map, i);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (!mail_index_keyword_lookup(index, keyword, &idx) ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen idx != *old_idx) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen "Keywords changed unexpectedly",
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen index->filepath);
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen return -1;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen }
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen#endif
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen /* Register the newly seen keywords */
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen i = array_count(&map->keyword_idx_map);
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen for (; i < kw_hdr->keywords_count; i++) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen const char *keyword = name + kw_rec[i].name_offset;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen unsigned int idx;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen if (*keyword == '\0') {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
9af06b76539445d2d84d6e1bcb91685b6abeb4e0Timo Sirainen "Empty keyword name in header",
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen index->filepath);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen return -1;
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen }
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen mail_index_keyword_lookup_or_create(index, keyword, &idx);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen array_append(&map->keyword_idx_map, &idx, 1);
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen return 0;
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen}
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainenstatic bool mail_index_check_header_compat(struct mail_index *index,
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen const struct mail_index_header *hdr,
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen uoff_t file_size)
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen{
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen enum mail_index_header_compat_flags compat_flags = 0;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
9af06b76539445d2d84d6e1bcb91685b6abeb4e0Timo Sirainen#ifndef WORDS_BIGENDIAN
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen#endif
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen /* major version change - handle silently(?) */
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen return FALSE;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen /* we've already complained about it */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return FALSE;
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (hdr->compat_flags != compat_flags) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* architecture change */
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen mail_index_set_error(index, "Rebuilding index file %s: "
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen "CPU architecture changed",
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen index->filepath);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen return FALSE;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
bb86f8f22f2561438ce710d2113f04a4d0082b50Timo Sirainen if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen hdr->header_size < hdr->base_header_size) {
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen "Corrupted header sizes (base %u, full %u)",
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen index->filepath, hdr->base_header_size,
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen hdr->header_size);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen return FALSE;
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen }
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen if (hdr->header_size > file_size) {
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen "Corrupted header size (%u > %"PRIuUOFF_T")",
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen index->filepath, hdr->header_size,
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen file_size);
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen return FALSE;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen if (hdr->indexid != index->indexid) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen if (index->indexid != 0) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen mail_index_set_error(index, "Index file %s: "
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen "indexid changed: %u -> %u",
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen index->filepath, index->indexid,
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen hdr->indexid);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen index->indexid = hdr->indexid;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen mail_transaction_log_indexid_changed(index->log);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen return TRUE;
dad1d7b721e80a7e6c0282ace93aef86312fa579Timo Sirainen}
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainenint mail_index_map_check_header(struct mail_index_map *map)
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen{
dad1d7b721e80a7e6c0282ace93aef86312fa579Timo Sirainen struct mail_index *index = map->index;
dad1d7b721e80a7e6c0282ace93aef86312fa579Timo Sirainen const struct mail_index_header *hdr = &map->hdr;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen if (!mail_index_check_header_compat(index, hdr, (uoff_t)-1))
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen return -1;
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* following some extra checks that only take a bit of CPU */
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen if (hdr->record_size < sizeof(struct mail_index_record)) {
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen "record_size too small: %u < %"PRIuSIZE_T,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen index->filepath, hdr->record_size,
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen sizeof(struct mail_index_record));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen if (hdr->uid_validity == 0 && hdr->next_uid != 1)
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen return 0;
97144a346898fb62f9fae44fa5c076986553c66bTimo Sirainen if (hdr->next_uid == 0)
97144a346898fb62f9fae44fa5c076986553c66bTimo Sirainen return 0;
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen if (hdr->messages_count > map->rec_map->records_count)
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen return 0;
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen if (hdr->seen_messages_count > hdr->messages_count ||
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen hdr->deleted_messages_count > hdr->messages_count)
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen return 0;
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen if (hdr->first_recent_uid == 0 ||
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen hdr->first_recent_uid > hdr->next_uid ||
97144a346898fb62f9fae44fa5c076986553c66bTimo Sirainen hdr->first_unseen_uid_lowwater > hdr->next_uid ||
97144a346898fb62f9fae44fa5c076986553c66bTimo Sirainen hdr->first_deleted_uid_lowwater > hdr->next_uid)
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen return 0;
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen if (hdr->messages_count > 0) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen /* last message's UID must be smaller than next_uid.
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen also make sure it's not zero. */
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen const struct mail_index_record *rec;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen rec = MAIL_INDEX_MAP_IDX(map, hdr->messages_count-1);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (rec->uid == 0 || rec->uid >= hdr->next_uid)
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen return 0;
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen }
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen return 1;
211c638d81d382517d196ad47565e0d85012c927klemens}
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainenstatic void mail_index_map_copy_hdr(struct mail_index_map *map,
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen const struct mail_index_header *hdr)
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen{
9af06b76539445d2d84d6e1bcb91685b6abeb4e0Timo Sirainen if (hdr->base_header_size < sizeof(map->hdr)) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* header smaller than ours, make a copy so our newer headers
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen won't have garbage in them */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen memcpy(&map->hdr, hdr, hdr->base_header_size);
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen } else {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen map->hdr = *hdr;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen }
9af06b76539445d2d84d6e1bcb91685b6abeb4e0Timo Sirainen
9af06b76539445d2d84d6e1bcb91685b6abeb4e0Timo Sirainen /* FIXME: backwards compatibility, remove later. In case this index is
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen accessed with Dovecot v1.0, avoid recent message counter errors. */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen map->hdr.unused_old_recent_messages_count = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mail_index_mmap(struct mail_index_map *map, uoff_t file_size)
de9b2ee7878a73346ba0eee34798abb22ffcfcb6Timo Sirainen{
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen struct mail_index *index = map->index;
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen struct mail_index_record_map *rec_map = map->rec_map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mail_index_header *hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(rec_map->mmap_base == NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_free(rec_map->buffer);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen rec_map->buffer = NULL;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen if (file_size > SSIZE_T_MAX) {
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen /* too large file to map into memory */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen mail_index_set_error(index, "Index file too large: %s",
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen index->filepath);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rec_map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MAP_PRIVATE, index->fd, 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (rec_map->mmap_base == MAP_FAILED) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rec_map->mmap_base = NULL;
5bd1c1d4fe3265d5e6b6054044fd6d78e42c9d0aTimo Sirainen mail_index_set_syscall_error(index, "mmap()");
5bd1c1d4fe3265d5e6b6054044fd6d78e42c9d0aTimo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen rec_map->mmap_size = file_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr = rec_map->mmap_base;
e9594e86dc601b72c1636f2b901dcfbf4ffaf47fAki Tuomi if (rec_map->mmap_size >
e9594e86dc601b72c1636f2b901dcfbf4ffaf47fAki Tuomi offsetof(struct mail_index_header, major_version) &&
e9594e86dc601b72c1636f2b901dcfbf4ffaf47fAki Tuomi hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
e9594e86dc601b72c1636f2b901dcfbf4ffaf47fAki Tuomi /* major version change - handle silently */
e9594e86dc601b72c1636f2b901dcfbf4ffaf47fAki Tuomi return 0;
7707c4b35b868eda75b585c863d97726ff23f80fMartti Rannanjärvi }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (rec_map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen "File too small (%"PRIuSIZE_T")",
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen index->filepath, rec_map->mmap_size);
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size)) {
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen /* Can't use this file */
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen return 0;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen }
46d91e9ea8bf41e56c5436c064372171c5876d81Timo Sirainen
46d91e9ea8bf41e56c5436c064372171c5876d81Timo Sirainen rec_map->mmap_used_size = hdr->header_size +
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen hdr->messages_count * hdr->record_size;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen if (rec_map->mmap_used_size <= rec_map->mmap_size)
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen rec_map->records_count = hdr->messages_count;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen else {
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen rec_map->records_count =
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen (rec_map->mmap_size - hdr->header_size) /
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen hdr->record_size;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen rec_map->mmap_used_size = hdr->header_size +
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rec_map->records_count * hdr->record_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen "messages_count too large (%u > %u)",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen index->filepath, hdr->messages_count,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rec_map->records_count);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_map_copy_hdr(map, hdr);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen map->hdr_base = rec_map->mmap_base;
a835194f9a9dae88528367a791cbc282589f6c01Timo Sirainen rec_map->records = PTR_OFFSET(rec_map->mmap_base, map->hdr.header_size);
0b878c6a17c608fcd8b52a5762ed2c6a5cf4700aTimo Sirainen return 1;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen}
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mail_index_read_header(struct mail_index *index,
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen void *buf, size_t buf_size, size_t *pos_r)
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen{
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen size_t pos;
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen int ret;
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen memset(buf, 0, sizeof(struct mail_index_header));
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen /* try to read the whole header, but it's not necessarily an error to
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen read less since the older versions of the index format could be
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen smaller. Request reading up to buf_size, but accept if we only got
17018da24e7dbb419c5047c316caadcb2fc5364aTimo Sirainen the header. */
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen pos = 0;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen do {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = pread(index->fd, PTR_OFFSET(buf, pos),
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen buf_size - pos, pos);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen if (ret > 0)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen pos += ret;
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen } while (ret > 0 && pos < sizeof(struct mail_index_header));
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen *pos_r = pos;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainenstatic int
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainenmail_index_try_read_map(struct mail_index_map *map,
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen uoff_t file_size, bool *retry_r, bool try_retry)
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen{
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen struct mail_index *index = map->index;
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen const struct mail_index_header *hdr;
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen unsigned char read_buf[4096];
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen const void *buf;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen void *data = NULL;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen ssize_t ret;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen size_t pos, records_size, initial_buf_pos = 0;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen unsigned int records_count = 0, extra;
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen i_assert(map->rec_map->mmap_base == NULL);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
519e0a461271843833a2b42626ad93f6e7ddc497Timo Sirainen *retry_r = FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buf = read_buf; hdr = buf;
6307d76096764e66bddc63d4a3e5a1aa19cc528fJosef 'Jeff' Sipek
3ccab0bac68040f179a7de45c516cec258e28fdbTimo Sirainen if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
648d24583c1574441c4fa0331a90bd4d6e7996c5Timo Sirainen hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
0cea9b1f4fa0495a48f5f097e40492517d67e1baTimo Sirainen /* major version change - handle silently */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return 0;
0cea9b1f4fa0495a48f5f097e40492517d67e1baTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (ret > 0 || pos >= hdr->base_header_size)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (!mail_index_check_header_compat(index, hdr, file_size)) {
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen /* Can't use this file */
2af769daebd83719ac696a440e06f6020471cec0Timo Sirainen return 0;
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen }
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen initial_buf_pos = pos;
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen if (pos > hdr->header_size)
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen pos = hdr->header_size;
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen /* place the base header into memory. */
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen buffer_reset(map->hdr_copy_buf);
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen buffer_append(map->hdr_copy_buf, buf, pos);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (pos != hdr->header_size) {
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen /* @UNSAFE: read the rest of the header into memory */
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen data = buffer_append_space_unsafe(map->hdr_copy_buf,
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen hdr->header_size -
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen pos);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen ret = pread_full(index->fd, data,
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen hdr->header_size - pos, pos);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen if (ret > 0) {
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen /* header read, read the records now. */
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen records_size = (size_t)hdr->messages_count * hdr->record_size;
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen records_count = hdr->messages_count;
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen if (file_size - hdr->header_size < records_size ||
4aae8acbcfa9cac96b4af39bfabcbe569e804827Timo Sirainen (hdr->record_size != 0 &&
5bdad39213d28ab35e615a7f4ea1712ab25b6a80Timo Sirainen records_size / hdr->record_size != hdr->messages_count)) {
5bdad39213d28ab35e615a7f4ea1712ab25b6a80Timo Sirainen records_count = (file_size - hdr->header_size) /
5bdad39213d28ab35e615a7f4ea1712ab25b6a80Timo Sirainen hdr->record_size;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen records_size = (size_t)records_count * hdr->record_size;
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen "messages_count too large (%u > %u)",
4aae8acbcfa9cac96b4af39bfabcbe569e804827Timo Sirainen index->filepath, hdr->messages_count,
4aae8acbcfa9cac96b4af39bfabcbe569e804827Timo Sirainen records_count);
4aae8acbcfa9cac96b4af39bfabcbe569e804827Timo Sirainen }
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen if (map->rec_map->buffer == NULL) {
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen map->rec_map->buffer =
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen buffer_create_dynamic(default_pool,
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen records_size);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen /* @UNSAFE */
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen buffer_set_used_size(map->rec_map->buffer, 0);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (initial_buf_pos <= hdr->header_size)
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen extra = 0;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen else {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen extra = initial_buf_pos - hdr->header_size;
6cb2c6ecddcdbeac9e6c73a292244747e12a793eTimo Sirainen buffer_append(map->rec_map->buffer,
2af769daebd83719ac696a440e06f6020471cec0Timo Sirainen CONST_PTR_OFFSET(buf, hdr->header_size),
2af769daebd83719ac696a440e06f6020471cec0Timo Sirainen extra);
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen }
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen if (records_size > extra) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen data = buffer_append_space_unsafe(map->rec_map->buffer,
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen records_size - extra);
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen ret = pread_full(index->fd, data, records_size - extra,
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen hdr->header_size + extra);
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen }
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (ret < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (errno == ESTALE && try_retry) {
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen /* a new index file was renamed over this one. */
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen *retry_r = TRUE;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen return 0;
0d0451206a91e9f96e522075dce28a89adc2325dTimo Sirainen }
1c0590b2729567ad60dafde4d2c5f19635755a3dTimo Sirainen mail_index_set_syscall_error(index, "pread_full()");
1c0590b2729567ad60dafde4d2c5f19635755a3dTimo Sirainen return -1;
ba482d3624ca4f1b3d638e6e8470ba5134f21493Timo Sirainen }
ba482d3624ca4f1b3d638e6e8470ba5134f21493Timo Sirainen if (ret == 0) {
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen mail_index_set_error(index,
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen "Corrupted index file %s: File too small",
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen index->filepath);
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen return 0;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen }
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen map->rec_map->records =
84da9c6d6e162b064608cbfa9a47e0d60553c593Timo Sirainen buffer_get_modifiable_data(map->rec_map->buffer, NULL);
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen map->rec_map->records_count = records_count;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen mail_index_map_copy_hdr(map, hdr);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr_base = map->hdr_copy_buf->data;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen return 1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenstatic int mail_index_read_map(struct mail_index_map *map, uoff_t file_size)
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen{
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen struct mail_index *index = map->index;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen mail_index_sync_lost_handler_t *const *handlers;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen struct stat st;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen unsigned int i, count;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen int ret;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen bool try_retry, retry;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen /* notify all "sync lost" handlers */
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen handlers = array_get(&index->sync_lost_handlers, &count);
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen for (i = 0; i < count; i++)
425cbcea60cf689b0069698c83f8bdc474d70693Timo Sirainen (*handlers[i])(index);
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen for (i = 0;; i++) {
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen if (file_size == (uoff_t)-1) {
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen /* fstat() below failed */
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen ret = 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen retry = try_retry;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen } else {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen ret = mail_index_try_read_map(map, file_size,
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen &retry, try_retry);
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen }
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen if (ret != 0 || !retry)
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen break;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen /* ESTALE - reopen index file */
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen if (close(index->fd) < 0)
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen mail_index_set_syscall_error(index, "close()");
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen index->fd = -1;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen ret = mail_index_try_open_only(index);
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen if (ret <= 0) {
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen if (ret == 0) {
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen /* the file was lost */
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen errno = ENOENT;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen mail_index_set_syscall_error(index, "open()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen }
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen if (fstat(index->fd, &st) == 0)
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen file_size = st.st_size;
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen else {
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen if (errno != ESTALE) {
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen mail_index_set_syscall_error(index, "fstat()");
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen return -1;
ab70f55bb8d824ca1ed7c74196f2f502edd29cc7Timo Sirainen }
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen file_size = (uoff_t)-1;
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen }
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return ret;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen}
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenstatic void mail_index_header_init(struct mail_index *index,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen struct mail_index_header *hdr)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen{
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen memset(hdr, 0, sizeof(*hdr));
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen hdr->base_header_size = sizeof(*hdr);
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen hdr->header_size = sizeof(*hdr);
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen hdr->record_size = sizeof(struct mail_index_record);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen#ifndef WORDS_BIGENDIAN
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen#endif
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen
f501ad38c51cf1d8f4f84313922c785e6ae6e81fTimo Sirainen hdr->indexid = index->indexid;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen hdr->log_file_seq = 1;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen hdr->next_uid = 1;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen hdr->first_recent_uid = 1;
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen}
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
51b979b6414b940f04677a7e2d064be119345954Timo Sirainenstruct mail_index_map *mail_index_map_alloc(struct mail_index *index)
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen{
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen struct mail_index_map tmp_map;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen memset(&tmp_map, 0, sizeof(tmp_map));
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen mail_index_header_init(index, &tmp_map.hdr);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen tmp_map.index = index;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen tmp_map.hdr_base = &tmp_map.hdr;
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen /* a bit kludgy way to do this, but it initializes everything
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen nicely and correctly */
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen return mail_index_map_clone(&tmp_map);
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen}
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainenstatic int mail_index_map_latest_file(struct mail_index *index)
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen{
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen struct mail_index_map *old_map, *new_map;
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen struct stat st;
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen unsigned int lock_id;
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen uoff_t file_size;
bba52ecbb0cfb6585f1a4ff29695dd2d27af98d2Timo Sirainen bool use_mmap;
bba52ecbb0cfb6585f1a4ff29695dd2d27af98d2Timo Sirainen int ret, try;
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen ret = mail_index_reopen_if_changed(index);
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen if (ret <= 0) {
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen if (ret < 0)
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen return -1;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen /* the index file is lost/broken. let's hope that we can
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen build it from the transaction log. */
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen return 0;
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen }
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen /* the index file is still open, lock it */
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen if (mail_index_lock_shared(index, &lock_id) < 0)
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen return -1;
044b0557e92ae0bb3b25af49d5468bad3d17db43Timo Sirainen
044b0557e92ae0bb3b25af49d5468bad3d17db43Timo Sirainen if (index->nfs_flush)
044b0557e92ae0bb3b25af49d5468bad3d17db43Timo Sirainen nfs_flush_attr_cache_fd(index->filepath, index->fd);
044b0557e92ae0bb3b25af49d5468bad3d17db43Timo Sirainen
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen if (fstat(index->fd, &st) == 0)
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen file_size = st.st_size;
044b0557e92ae0bb3b25af49d5468bad3d17db43Timo Sirainen else {
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen if (errno != ESTALE) {
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen mail_index_set_syscall_error(index, "fstat()");
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen mail_index_unlock(index, &lock_id);
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen return -1;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen }
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen file_size = (uoff_t)-1;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen }
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen /* mmaping seems to be slower than just reading the file, so even if
131b073bdc3650083b00616dc778dd3017c2bbb5Timo Sirainen mmap isn't disabled don't use it unless the file is large enough */
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen use_mmap = !index->mmap_disable && file_size != (uoff_t)-1 &&
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen file_size > MAIL_INDEX_MMAP_MIN_SIZE;
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen new_map = mail_index_map_alloc(index);
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen if (use_mmap) {
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen new_map->rec_map->lock_id = lock_id;
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen ret = mail_index_mmap(new_map, file_size);
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen } else {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = mail_index_read_map(new_map, file_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_unlock(index, &lock_id);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (try = 0; ret > 0; try++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* make sure the header is ok before using this mapping */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = mail_index_map_check_header(new_map);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret > 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (mail_index_map_parse_extensions(new_map) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen ret = 0;
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen else if (mail_index_map_parse_keywords(new_map) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret != 0 || try == 2)
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen /* fsck and try again */
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen old_map = index->map;
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen index->map = new_map;
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen if (mail_index_fsck(index) < 0) {
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen ret = -1;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen break;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen }
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen /* fsck replaced the map */
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen new_map = index->map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen index->map = old_map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen if (ret <= 0) {
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen mail_index_unmap(&new_map);
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen return ret;
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen }
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen i_assert(new_map->rec_map->records != NULL);
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen index->last_read_log_file_seq = new_map->hdr.log_file_seq;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen index->last_read_log_file_head_offset =
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen new_map->hdr.log_file_head_offset;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen index->last_read_log_file_tail_offset =
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen new_map->hdr.log_file_tail_offset;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen index->last_read_stat = st;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen mail_index_unmap(&index->map);
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen index->map = new_map;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen return 1;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen}
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainenint mail_index_map(struct mail_index *index,
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen enum mail_index_sync_handler_type type)
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen{
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen int ret;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen i_assert(index->lock_type != F_WRLCK);
i_assert(!index->mapping);
index->mapping = TRUE;
if (index->map == NULL)
index->map = mail_index_map_alloc(index);
/* first try updating the existing mapping from transaction log. */
if (index->map->hdr.indexid != 0 && index->indexid != 0) {
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
ret = mail_index_sync_map(&index->map, type, FALSE);
} else {
ret = 0;
}
if (ret == 0) {
/* try to open and read the latest index. if it fails for
any reason, we'll fallback to updating the existing mapping
from transaction logs (which we'll also do even if the
reopening succeeds) */
(void)mail_index_map_latest_file(index);
/* if we're creating the index file, we don't have any
logs yet */
if (index->log->head != NULL && index->indexid != 0) {
/* and update the map with the latest changes from
transaction log */
ret = mail_index_sync_map(&index->map, type, TRUE);
}
}
index->mapping = FALSE;
return ret;
}
static void mail_index_record_map_free(struct mail_index_map *map,
struct mail_index_record_map *rec_map)
{
if (rec_map->lock_id != 0)
mail_index_unlock(map->index, &rec_map->lock_id);
if (rec_map->buffer != NULL) {
i_assert(rec_map->mmap_base == NULL);
buffer_free(rec_map->buffer);
rec_map->buffer = NULL;
} else if (rec_map->mmap_base != NULL) {
i_assert(rec_map->buffer == NULL);
if (munmap(rec_map->mmap_base, rec_map->mmap_size) < 0)
mail_index_set_syscall_error(map->index, "munmap()");
rec_map->mmap_base = NULL;
}
array_free(&rec_map->maps);
i_free(rec_map);
}
static void mail_index_record_map_unlink(struct mail_index_map *map)
{
struct mail_index_map *const *maps;
unsigned int i, count;
maps = array_get(&map->rec_map->maps, &count);
for (i = 0; i < count; i++) {
if (maps[i] == map) {
array_delete(&map->rec_map->maps, i, 1);
if (i == 0 && count == 1)
mail_index_record_map_free(map, map->rec_map);
return;
}
}
i_unreached();
}
void mail_index_unmap(struct mail_index_map **_map)
{
struct mail_index_map *map = *_map;
*_map = NULL;
if (--map->refcount > 0)
return;
i_assert(map->refcount == 0);
mail_index_record_map_unlink(map);
if (map->extension_pool != NULL)
pool_unref(map->extension_pool);
if (array_is_created(&map->keyword_idx_map))
array_free(&map->keyword_idx_map);
buffer_free(map->hdr_copy_buf);
i_free(map);
}
static void mail_index_map_copy_records(struct mail_index_record_map *dest,
const struct mail_index_record_map *src,
unsigned int record_size)
{
size_t size;
size = src->records_count * record_size;
dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024));
buffer_append(dest->buffer, src->records, size);
dest->records = buffer_get_modifiable_data(dest->buffer, NULL);
dest->records_count = src->records_count;
/* if the map is ever written back to disk, we need to keep track of
what has changed. */
dest->write_seq_first = src->write_seq_first;
dest->write_seq_last = src->write_seq_last;
}
static void mail_index_map_copy_header(struct mail_index_map *dest,
const struct mail_index_map *src)
{
/* use src->hdr copy directly, because if we got here
from syncing it has the latest changes. */
dest->hdr = src->hdr;
if (dest->hdr_copy_buf != NULL) {
if (src == dest)
return;
buffer_set_used_size(dest->hdr_copy_buf, 0);
} else {
dest->hdr_copy_buf =
buffer_create_dynamic(default_pool,
dest->hdr.header_size);
}
buffer_append(dest->hdr_copy_buf, &dest->hdr,
I_MIN(sizeof(dest->hdr), src->hdr.base_header_size));
if (src != dest) {
buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size,
CONST_PTR_OFFSET(src->hdr_base,
src->hdr.base_header_size),
src->hdr.header_size - src->hdr.base_header_size);
}
dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
i_assert(dest->hdr_copy_buf->used == dest->hdr.header_size);
}
static struct mail_index_record_map *
mail_index_record_map_alloc(struct mail_index_map *map)
{
struct mail_index_record_map *rec_map;
rec_map = i_new(struct mail_index_record_map, 1);
i_array_init(&rec_map->maps, 4);
array_append(&rec_map->maps, &map, 1);
return rec_map;
}
struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map)
{
struct mail_index_map *mem_map;
struct mail_index_ext *extensions;
unsigned int i, count;
mem_map = i_new(struct mail_index_map, 1);
mem_map->index = map->index;
mem_map->refcount = 1;
if (map->rec_map == NULL) {
mem_map->rec_map = mail_index_record_map_alloc(mem_map);
mem_map->rec_map->buffer =
buffer_create_dynamic(default_pool, 1024);
} else {
mem_map->rec_map = map->rec_map;
array_append(&mem_map->rec_map->maps, &mem_map, 1);
}
mail_index_map_copy_header(mem_map, map);
mem_map->write_atomic = map->write_atomic;
mem_map->write_base_header = map->write_base_header;
mem_map->write_ext_header = map->write_ext_header;
/* copy extensions */
if (array_is_created(&map->ext_id_map)) {
count = array_count(&map->ext_id_map);
mail_index_map_init_extbufs(mem_map, count + 2);
array_append_array(&mem_map->extensions, &map->extensions);
array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
/* fix the name pointers to use our own pool */
extensions = array_get_modifiable(&mem_map->extensions, &count);
for (i = 0; i < count; i++) {
i_assert(extensions[i].record_offset +
extensions[i].record_size <=
mem_map->hdr.record_size);
extensions[i].name = p_strdup(mem_map->extension_pool,
extensions[i].name);
}
}
/* copy keyword map */
if (array_is_created(&map->keyword_idx_map)) {
i_array_init(&mem_map->keyword_idx_map,
array_count(&map->keyword_idx_map) + 4);
array_append_array(&mem_map->keyword_idx_map,
&map->keyword_idx_map);
}
return mem_map;
}
void mail_index_record_map_move_to_private(struct mail_index_map *map)
{
struct mail_index_record_map *new_map;
if (array_count(&map->rec_map->maps) == 1)
return;
new_map = mail_index_record_map_alloc(map);
mail_index_map_copy_records(new_map, map->rec_map,
map->hdr.record_size);
mail_index_record_map_unlink(map);
map->rec_map = new_map;
}
void mail_index_map_move_to_memory(struct mail_index_map *map)
{
struct mail_index_record_map *new_map;
if (map->rec_map->mmap_base == NULL)
return;
i_assert(map->rec_map->lock_id != 0);
new_map = array_count(&map->rec_map->maps) == 1 ? map->rec_map :
mail_index_record_map_alloc(map);
mail_index_map_copy_records(new_map, map->rec_map,
map->hdr.record_size);
mail_index_map_copy_header(map, map);
if (new_map != map->rec_map) {
mail_index_record_map_unlink(map);
map->rec_map = new_map;
} else {
mail_index_unlock(map->index, &new_map->lock_id);
if (munmap(new_map->mmap_base, new_map->mmap_size) < 0)
mail_index_set_syscall_error(map->index, "munmap()");
new_map->mmap_base = NULL;
}
}
bool mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r)
{
const uint32_t *id;
if (!array_is_created(&map->ext_id_map) ||
ext_id >= array_count(&map->ext_id_map))
return FALSE;
id = array_idx(&map->ext_id_map, ext_id);
*idx_r = *id;
return *idx_r != (uint32_t)-1;
}