mail-index.c revision 0e17a51126ed6f376f9207217797103b610ff8eb
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "lib.h"
685a84a0ccb55cb9359dcd57d9eb3b6b836034a2Timo Sirainen#include "array.h"
5ab2ee0b9b7ad3867fcfd2a31fda0790370fbbbdTimo Sirainen#include "buffer.h"
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen#include "hash.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mmap-util.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "read-full.h"
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen#include "write-full.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-index-private.h"
06b0c3be9905099038964b068216bbed155701deTimo Sirainen#include "mail-index-sync-private.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-transaction-log.h"
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainen#include "mail-cache.h"
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainen
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainen#include <stdio.h>
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainen#include <stddef.h>
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainen#include <time.h>
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include <sys/stat.h>
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainen
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainenstatic void mail_index_create_in_memory(struct mail_index *index,
d13a8e21656e1b7005e094f7b9c9e3611105c648Timo Sirainen const struct mail_index_header *hdr);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_index *index;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index = i_new(struct mail_index, 1);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen index->dir = i_strdup(dir);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->prefix = i_strdup(prefix);
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen index->fd = -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen index->extension_pool = pool_alloconly_create("extension", 512);
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen ARRAY_CREATE(&index->extensions, index->extension_pool,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_index_registered_ext, 5);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ARRAY_CREATE(&index->sync_lost_handlers, default_pool,
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen mail_index_sync_lost_handler_t *, 4);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->mode = 0600;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->gid = (gid_t)-1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen index->keywords_ext_id =
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_ext_register(index, "keywords", 128, 2, 1);
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen index->keywords_pool = pool_alloconly_create("keywords", 512);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ARRAY_CREATE(&index->keywords, default_pool, const char *, 16);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->keywords_hash =
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen hash_create(default_pool, index->keywords_pool, 0,
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen strcase_hash, (hash_cmp_callback_t *)strcasecmp);
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen return index;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainenvoid mail_index_free(struct mail_index *index)
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen{
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen mail_index_close(index);
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen hash_destroy(index->keywords_hash);
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen pool_unref(index->extension_pool);
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen pool_unref(index->keywords_pool);
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen array_free(&index->sync_lost_handlers);
0db5b158a00c08955bdacc99b1e2cd1ec07f4311Timo Sirainen array_free(&index->keywords);
0db5b158a00c08955bdacc99b1e2cd1ec07f4311Timo Sirainen
0db5b158a00c08955bdacc99b1e2cd1ec07f4311Timo Sirainen i_free(index->error);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_free(index->dir);
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen i_free(index->prefix);
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen i_free(index);
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen}
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainenvoid mail_index_set_permissions(struct mail_index *index,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen mode_t mode, gid_t gid)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen index->mode = mode & 0666;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->gid = gid;
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainen}
f635aa2fd6c47ec297cf68981203a8dc7f8a23c3Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint32_t default_hdr_size,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen uint16_t default_record_size,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint16_t default_record_align)
b57cd9928909b51fa473c3eea81442e296006438Timo Sirainen{
b57cd9928909b51fa473c3eea81442e296006438Timo Sirainen const struct mail_index_registered_ext *extensions;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen struct mail_index_registered_ext rext;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen unsigned int i, ext_count;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen extensions = array_get(&index->extensions, &ext_count);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* see if it's already there */
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen for (i = 0; i < ext_count; i++) {
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen if (strcmp(extensions[i].name, name) == 0)
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen return i;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen memset(&rext, 0, sizeof(rext));
c87d1e148ae76cf20f3adc7fc84fd54219dc62d5Timo Sirainen rext.name = p_strdup(index->extension_pool, name);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen rext.index_idx = ext_count;
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen rext.hdr_size = default_hdr_size;
c87d1e148ae76cf20f3adc7fc84fd54219dc62d5Timo Sirainen rext.record_size = default_record_size;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen rext.record_align = default_record_align;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen array_append(&index->extensions, &rext, 1);
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen return ext_count;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen}
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainenvoid mail_index_register_expunge_handler(struct mail_index *index,
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen uint32_t ext_id,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen mail_index_expunge_handler_t *cb)
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen{
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen struct mail_index_registered_ext *rext;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen rext = array_idx_modifyable(&index->extensions, ext_id);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(rext->expunge_handler == NULL);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen rext->expunge_handler = cb;
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainen}
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainen
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainenvoid mail_index_unregister_expunge_handler(struct mail_index *index,
e8fd206cf9dca263278efba21864606126fc29b8Timo Sirainen uint32_t ext_id)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
06b0c3be9905099038964b068216bbed155701deTimo Sirainen struct mail_index_registered_ext *rext;
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainen
f635aa2fd6c47ec297cf68981203a8dc7f8a23c3Timo Sirainen rext = array_idx_modifyable(&index->extensions, ext_id);
06b0c3be9905099038964b068216bbed155701deTimo Sirainen i_assert(rext->expunge_handler != NULL);
685a84a0ccb55cb9359dcd57d9eb3b6b836034a2Timo Sirainen
685a84a0ccb55cb9359dcd57d9eb3b6b836034a2Timo Sirainen rext->expunge_handler = NULL;
685a84a0ccb55cb9359dcd57d9eb3b6b836034a2Timo Sirainen}
685a84a0ccb55cb9359dcd57d9eb3b6b836034a2Timo Sirainen
685a84a0ccb55cb9359dcd57d9eb3b6b836034a2Timo Sirainenvoid mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,
f635aa2fd6c47ec297cf68981203a8dc7f8a23c3Timo Sirainen mail_index_sync_handler_t *cb,
f635aa2fd6c47ec297cf68981203a8dc7f8a23c3Timo Sirainen enum mail_index_sync_handler_type type)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen struct mail_index_registered_ext *rext;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen rext = array_idx_modifyable(&index->extensions, ext_id);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen i_assert(rext->sync_handler.callback == NULL);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen rext->sync_handler.callback = cb;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen rext->sync_handler.type = type;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen}
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainenvoid mail_index_unregister_sync_handler(struct mail_index *index,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen uint32_t ext_id)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainen struct mail_index_registered_ext *rext;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen rext = array_idx_modifyable(&index->extensions, ext_id);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen i_assert(rext->sync_handler.callback != NULL);
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen rext->sync_handler.callback = NULL;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen rext->sync_handler.type = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenvoid mail_index_register_sync_lost_handler(struct mail_index *index,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_sync_lost_handler_t *cb)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_append(&index->sync_lost_handlers, &cb, 1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenvoid mail_index_unregister_sync_lost_handler(struct mail_index *index,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_sync_lost_handler_t *cb)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_sync_lost_handler_t *const *handlers;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen unsigned int i, count;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen handlers = array_get(&index->sync_lost_handlers, &count);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen for (i = 0; i < count; i++) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (handlers[i] == cb) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_delete(&index->sync_lost_handlers, i, 1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen break;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen }
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen unsigned int initial_count)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#define EXTENSION_NAME_APPROX_LEN 20
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen size_t size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (map->extension_pool == NULL) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen size = (sizeof(array_t) + BUFFER_APPROX_SIZE) * 2 +
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen initial_count * (EXTENSION_NAME_APPROX_LEN +
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen sizeof(struct mail_index_ext) +
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen sizeof(uint32_t));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen map->extension_pool =
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen pool_alloconly_create("extensions",
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen nearest_power(size));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen } else {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen p_clear(map->extension_pool);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ARRAY_CREATE(&map->extensions, map->extension_pool,
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen struct mail_index_ext, initial_count);
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen ARRAY_CREATE(&map->ext_id_map, map->extension_pool,
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen uint32_t, initial_count);
57ca8cc86193103127066c724815e7e7a24926dbTimo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenuint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const struct mail_index_ext *extensions;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen unsigned int i, size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen if (!array_is_created(&map->extensions))
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return (uint32_t)-1;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen extensions = array_get(&map->extensions, &size);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen for (i = 0; i < size; i++) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (strcmp(extensions[i].name, name) == 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return i;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return (uint32_t)-1;
a94936bafd127680184da114c6a177b37ff656e5Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenuint32_t
3e28b527dd6048a40684afd29cff0ee008fc0014Timo Sirainenmail_index_map_register_ext(struct mail_index *index,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_index_map *map, const char *name,
bc27fcd011f86208feaf73da9778a66ac7d7d3abTimo Sirainen uint32_t hdr_offset, uint32_t hdr_size,
3e28b527dd6048a40684afd29cff0ee008fc0014Timo Sirainen uint32_t record_offset, uint32_t record_size,
3e28b527dd6048a40684afd29cff0ee008fc0014Timo Sirainen uint32_t record_align, uint32_t reset_id)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_index_ext *ext;
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen uint32_t idx, empty_idx = (uint32_t)-1;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen if (!array_is_created(&map->extensions)) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_map_init_extbufs(map, 5);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen idx = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen } else {
a94936bafd127680184da114c6a177b37ff656e5Timo Sirainen idx = array_count(&map->extensions);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
4c9c55e15f35474f53f11659e796c63b1c34e884Timo Sirainen ext = array_append_space(&map->extensions);
9222c96ab9f42b25d5d5260f9e7a42c694715b0aTimo Sirainen ext->name = p_strdup(map->extension_pool, name);
9222c96ab9f42b25d5d5260f9e7a42c694715b0aTimo Sirainen ext->hdr_offset = hdr_offset;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ext->hdr_size = hdr_size;
3e28b527dd6048a40684afd29cff0ee008fc0014Timo Sirainen ext->record_offset = record_offset;
648d24583c1574441c4fa0331a90bd4d6e7996c5Timo Sirainen ext->record_size = record_size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ext->record_align = record_align;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ext->reset_id = reset_id;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ext->index_idx = mail_index_ext_register(index, name, hdr_size,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen record_size, record_align);
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen /* Update index ext_id -> map ext_id mapping. Fill non-used
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ext_ids with (uint32_t)-1 */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen while (array_count(&map->ext_id_map) < ext->index_idx)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen array_append(&map->ext_id_map, &empty_idx, 1);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return idx;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainenstatic int size_check(size_t *size_left, size_t size)
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen{
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen if (size > *size_left)
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen return FALSE;
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen *size_left -= size;
13a8c553f293349248b161ff851743498916e26eTimo Sirainen return TRUE;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen}
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainenstatic size_t get_align(size_t name_len)
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen{
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen size_t size = sizeof(struct mail_index_ext_header) + name_len;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstatic int mail_index_read_extensions(struct mail_index *index,
c95fc202215d2451372599db7092b16459f360a3Timo Sirainen struct mail_index_map *map)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen const struct mail_index_ext_header *ext_hdr;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen unsigned int i, old_count;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen const char *name;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen uint32_t ext_id, offset, name_offset;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen size_t size_left;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* extension headers always start from 64bit offsets, so if base header
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen doesn't happen to be 64bit aligned we'll skip some bytes */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* nothing to do, skip allocatations and all */
c95fc202215d2451372599db7092b16459f360a3Timo Sirainen return 1;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen }
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen old_count = array_count(&index->extensions);
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen mail_index_map_init_extbufs(map, old_count + 5);
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen ext_id = (uint32_t)-1;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen for (i = 0; i < old_count; i++)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_append(&map->ext_id_map, &ext_id, 1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen while (offset < map->hdr.header_size) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
3db8062598ce08e9320c84f77c267b9c70cb0809Timo Sirainen
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen /* Extension header contains:
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen - struct mail_index_ext_header
3db8062598ce08e9320c84f77c267b9c70cb0809Timo Sirainen - name (not 0-terminated)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen - 64bit alignment padding
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen - extension header contents
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen - 64bit alignment padding
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen */
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen size_left = map->hdr.header_size - offset;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (!size_check(&size_left, sizeof(*ext_hdr)) ||
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen !size_check(&size_left, ext_hdr->name_size) ||
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen !size_check(&size_left, get_align(ext_hdr->name_size)) ||
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen !size_check(&size_left, ext_hdr->hdr_size)) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
eeea0a402bcd9533e9e359f2a2518e3216162151Timo Sirainen "Header extension goes outside header",
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen index->filepath);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen return -1;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen offset += sizeof(*ext_hdr);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen name_offset = offset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen offset += ext_hdr->name_size + get_align(ext_hdr->name_size);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
e8fd206cf9dca263278efba21864606126fc29b8Timo Sirainen t_push();
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen ext_hdr->name_size);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen "Duplicate header extension %s",
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen index->filepath, name);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen t_pop();
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen return -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainen
e8fd206cf9dca263278efba21864606126fc29b8Timo Sirainen if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
e8fd206cf9dca263278efba21864606126fc29b8Timo Sirainen (map->hdr.record_size % ext_hdr->record_align) != 0) {
e8fd206cf9dca263278efba21864606126fc29b8Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen "Record field %s alignmentation %u not used",
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainen index->filepath, name, ext_hdr->record_align);
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainen t_pop();
4e9070b89501c0ce7ad6aaf4f5e49aba1ee56decTimo Sirainen return -1;
e8fd206cf9dca263278efba21864606126fc29b8Timo Sirainen }
e8fd206cf9dca263278efba21864606126fc29b8Timo Sirainen
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen mail_index_map_register_ext(index, map, name,
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen offset, ext_hdr->hdr_size,
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen ext_hdr->record_offset,
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen ext_hdr->record_size,
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen ext_hdr->record_align,
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen ext_hdr->reset_id);
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen t_pop();
a5be5a0d074ed0f3e4106612a2792e143a43efc6Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
a74e4a66db99a69cca71d7c5ac1feae46d92138fTimo Sirainen return 1;
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen}
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainenint mail_index_keyword_lookup(struct mail_index *index,
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen const char *keyword, int autocreate,
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen unsigned int *idx_r)
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen{
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen char *keyword_dup;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen void *value;
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen /* keywords_hash keeps a name => index mapping of keywords.
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen Keywords are never removed from it, so the index values are valid
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen for the lifetime of the mail_index. */
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen *idx_r = POINTER_CAST_TO(value, unsigned int);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen return TRUE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (!autocreate) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen *idx_r = (unsigned int)-1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return FALSE;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
fedb73c7e918653877286ede0fe18029b3cce7d3Timo Sirainen *idx_r = array_count(&index->keywords);
fedb73c7e918653877286ede0fe18029b3cce7d3Timo Sirainen
fedb73c7e918653877286ede0fe18029b3cce7d3Timo Sirainen hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r));
fedb73c7e918653877286ede0fe18029b3cce7d3Timo Sirainen array_append(&index->keywords, &keyword, 1);
fedb73c7e918653877286ede0fe18029b3cce7d3Timo Sirainen return TRUE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
a2550844936da8b78d7565b905a4dc5ffb3eef0eTimo Sirainen
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainenint mail_index_map_read_keywords(struct mail_index *index,
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen struct mail_index_map *map)
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen{
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen const struct mail_index_ext *ext;
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen const struct mail_index_keyword_header *kw_hdr;
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen const struct mail_index_keyword_header_rec *kw_rec;
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen const char *name;
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen unsigned int i, name_area_end_offset, old_count;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint32_t ext_id;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ext_id = mail_index_map_lookup_ext(map, "keywords");
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ext_id == (uint32_t)-1) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (array_is_created(&map->keyword_idx_map))
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_clear(&map->keyword_idx_map);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ext = array_idx(&map->extensions, ext_id);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* Extension header contains:
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen - struct mail_index_keyword_header
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen - struct mail_index_keyword_header_rec * keywords_count
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen - const char names[] * keywords_count
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen kw_rec = (const void *)(kw_hdr + 1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen name = (const char *)(kw_rec + kw_hdr->keywords_count);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_count(&map->keyword_idx_map);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* Keywords can only be added into same mapping. Removing requires a
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen new mapping (recreating the index file) */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (kw_hdr->keywords_count == old_count) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* nothing changed */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* make sure the header is valid */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (kw_hdr->keywords_count < old_count) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen "Keywords removed unexpectedly",
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->filepath);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen "keywords_count larger than header size",
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->filepath);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen for (i = 0; i < kw_hdr->keywords_count; i++) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (kw_rec[i].name_offset > name_area_end_offset) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen "name_offset points outside allocated header",
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->filepath);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen }
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen if (name[name_area_end_offset-1] != '\0') {
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen "Keyword header doesn't end with NUL",
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen index->filepath);
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen return -1;
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen }
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen /* create file -> index mapping */
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen if (!array_is_created(&map->keyword_idx_map)) {
b3f46fa8a890527996fd0288cf964b7f3567cd22Timo Sirainen ARRAY_CREATE(&map->keyword_idx_map, default_pool,
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen unsigned int, kw_hdr->keywords_count);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#ifdef DEBUG
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* Check that existing headers are still the same. It's behind DEBUG
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen since it's pretty useless waste of CPU normally. */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const unsigned int *old_idx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen unsigned int idx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen old_idx = array_idx(&map->keyword_idx_map, i);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) ||
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen idx != *old_idx) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen "Keywords changed unexpectedly",
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->filepath);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#endif
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* Register the newly seen keywords */
6bca3405636e3ec95724350c3a10d6fcb737782aTimo Sirainen i = array_count(&map->keyword_idx_map);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen for (; i < kw_hdr->keywords_count; i++) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen unsigned int idx;
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen
933de3f374e6971f50308cb7bea13a51a45287bfTimo Sirainen (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx);
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen array_append(&map->keyword_idx_map, &idx, 1);
933de3f374e6971f50308cb7bea13a51a45287bfTimo Sirainen }
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen return 0;
933de3f374e6971f50308cb7bea13a51a45287bfTimo Sirainen}
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainenconst array_t *mail_index_get_keywords(struct mail_index *index)
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen{
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen /* Make sure all the keywords are in index->keywords. It's quick to do
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen if nothing has changed. */
3ec16f23d3c94c225e5b74d84c6a69064922cd9dTimo Sirainen (void)mail_index_map_read_keywords(index, index->map);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return &index->keywords;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen}
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainenstatic int mail_index_check_header(struct mail_index *index,
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen struct mail_index_map *map)
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen{
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen const struct mail_index_header *hdr = &map->hdr;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen enum mail_index_header_compat_flags compat_flags = 0;
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen#ifndef WORDS_BIGENDIAN
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen#endif
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen /* major version change - handle silently(?) */
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen return -1;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (hdr->compat_flags != compat_flags) {
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen /* architecture change - handle silently(?) */
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen return -1;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen /* we've already complained about it */
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen return -1;
59811ccfeb7dccc51e3be50c299c2ed853a1e2f3Timo Sirainen }
59811ccfeb7dccc51e3be50c299c2ed853a1e2f3Timo Sirainen
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen /* following some extra checks that only take a bit of CPU */
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen "uid_validity = 0, next_uid = %u",
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen index->filepath, hdr->next_uid);
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen return -1;
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen }
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen if (hdr->record_size < sizeof(struct mail_index_record)) {
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen "record_size too small: %u < %"PRIuSIZE_T,
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen index->filepath, hdr->record_size,
59811ccfeb7dccc51e3be50c299c2ed853a1e2f3Timo Sirainen sizeof(struct mail_index_record));
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen return -1;
bfdf0fd7b6186f64cbdcbf1cb2bf9c42a9007b77Timo Sirainen }
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen return 0;
f1ea65eb88b4f67e572c4640882bdf00fe3a2186Timo Sirainen
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen if (hdr->next_uid == 0)
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen return 0;
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen if (hdr->recent_messages_count > hdr->messages_count ||
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen hdr->seen_messages_count > hdr->messages_count ||
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen hdr->deleted_messages_count > hdr->messages_count)
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen return 0;
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen if (hdr->first_recent_uid_lowwater > hdr->next_uid ||
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen hdr->first_unseen_uid_lowwater > hdr->next_uid ||
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen hdr->first_deleted_uid_lowwater > hdr->next_uid)
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen return 0;
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen return mail_index_read_extensions(index, map);
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen}
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainenstatic void mail_index_map_clear(struct mail_index *index,
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen struct mail_index_map *map)
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen{
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen if (map->buffer != NULL) {
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen i_assert(map->mmap_base == NULL);
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen buffer_free(map->buffer);
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen map->buffer = NULL;
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen } else if (map->mmap_base != NULL) {
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen i_assert(map->buffer == NULL);
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen if (munmap(map->mmap_base, map->mmap_size) < 0)
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen mail_index_set_syscall_error(index, "munmap()");
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen map->mmap_base = NULL;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (map->refcount > 0) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen map->mmap_size = 0;
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen map->mmap_used_size = 0;
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen map->records = NULL;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen map->records_count = 0;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainenvoid mail_index_unmap(struct mail_index *index, struct mail_index_map *map)
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen{
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen if (--map->refcount > 0)
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen return;
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen i_assert(map->refcount == 0);
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen mail_index_map_clear(index, map);
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen if (map->extension_pool != NULL)
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen pool_unref(map->extension_pool);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (array_is_created(&map->keyword_idx_map))
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen array_free(&map->keyword_idx_map);
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen buffer_free(map->hdr_copy_buf);
a343811459d1d2259e668f369646a20f02301c2cTimo Sirainen i_free(map);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen}
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainenstatic void mail_index_map_copy_hdr(struct mail_index_map *map,
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen const struct mail_index_header *hdr)
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen{
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (hdr->base_header_size < sizeof(map->hdr)) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen /* header smaller than ours, make a copy so our newer headers
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen won't have garbage in them */
648d24583c1574441c4fa0331a90bd4d6e7996c5Timo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen memcpy(&map->hdr, hdr, hdr->base_header_size);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen } else {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen map->hdr = *hdr;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
bfdf0fd7b6186f64cbdcbf1cb2bf9c42a9007b77Timo Sirainen}
bfdf0fd7b6186f64cbdcbf1cb2bf9c42a9007b77Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainenstatic int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen{
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen const struct mail_index_header *hdr;
7d075009a641d88a45940238676883a8eaf1507bTimo Sirainen unsigned int records_count;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen i_assert(!map->write_to_disk);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (map->buffer != NULL) {
0db5b158a00c08955bdacc99b1e2cd1ec07f4311Timo Sirainen /* we had temporarily used a buffer, eg. for updating index */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen buffer_free(map->buffer);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen map->buffer = NULL;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen map->mmap_base = index->readonly ?
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen mmap_ro_file(index->fd, &map->mmap_size) :
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen mmap_rw_file(index->fd, &map->mmap_size);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (map->mmap_base == MAP_FAILED) {
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen map->mmap_base = NULL;
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen mail_index_set_syscall_error(index, "mmap()");
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen return -1;
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen }
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen hdr = map->mmap_base;
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen if (map->mmap_size >
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen offsetof(struct mail_index_header, major_version) &&
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen /* major version change - handle silently */
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen return 0;
0ae010139a1bb3b29fbf117c5da1a6a6c6b7b5a0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
0db5b158a00c08955bdacc99b1e2cd1ec07f4311Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "File too small (%"PRIuSIZE_T")",
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen index->filepath, map->mmap_size);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen return 0;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen }
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen map->mmap_used_size = hdr->header_size +
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen hdr->messages_count * hdr->record_size;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen if (map->mmap_used_size > map->mmap_size) {
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen records_count = (map->mmap_size - hdr->header_size) /
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen hdr->record_size;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen "messages_count too large (%u > %u)",
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen index->filepath, hdr->messages_count,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen records_count);
43d32cbe60fdaef2699d99f1ca259053e9350411Timo Sirainen return 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_map_copy_hdr(map, hdr);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
0db5b158a00c08955bdacc99b1e2cd1ec07f4311Timo Sirainen map->hdr_base = map->mmap_base;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen map->records_count = map->hdr.messages_count;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen return 1;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen}
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstatic int mail_index_read_header(struct mail_index *index,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen struct mail_index_header *hdr, size_t *pos_r)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen size_t pos;
4371df92c07fb923aabca7e90d307eccac48b2d6Timo Sirainen int ret;
4371df92c07fb923aabca7e90d307eccac48b2d6Timo Sirainen
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen memset(hdr, 0, sizeof(*hdr));
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen ret = 1;
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen for (pos = 0; ret > 0 && pos < sizeof(*hdr); ) {
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen ret = pread(index->fd, PTR_OFFSET(hdr, pos),
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen sizeof(*hdr) - pos, pos);
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen if (ret > 0)
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen pos += ret;
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen }
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen *pos_r = pos;
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen return ret;
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen}
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic int mail_index_read_map(struct mail_index *index,
struct mail_index_map *map, int *retry_r)
{
struct mail_index_header hdr;
void *data = NULL;
ssize_t ret;
size_t pos, records_size;
i_assert(map->mmap_base == NULL);
*retry_r = FALSE;
ret = mail_index_read_header(index, &hdr, &pos);
if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
hdr.major_version != MAIL_INDEX_MAJOR_VERSION) {
/* major version change - handle silently */
return 0;
}
if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
(ret > 0 || pos >= hdr.base_header_size)) {
if (hdr.base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
hdr.header_size < hdr.base_header_size) {
mail_index_set_error(index, "Corrupted index file %s: "
"Corrupted header sizes (base %u, full %u)",
index->filepath, hdr.base_header_size,
hdr.header_size);
return 0;
}
buffer_reset(map->hdr_copy_buf);
buffer_append(map->hdr_copy_buf, &hdr, hdr.base_header_size);
/* @UNSAFE */
data = buffer_append_space_unsafe(map->hdr_copy_buf,
hdr.header_size -
hdr.base_header_size);
ret = pread_full(index->fd, data,
hdr.header_size - hdr.base_header_size,
hdr.base_header_size);
}
if (ret > 0) {
records_size = hdr.messages_count * hdr.record_size;
if (map->buffer == NULL) {
map->buffer = buffer_create_dynamic(default_pool,
records_size);
}
/* @UNSAFE */
buffer_set_used_size(map->buffer, 0);
data = buffer_append_space_unsafe(map->buffer, records_size);
ret = pread_full(index->fd, data, records_size,
hdr.header_size);
}
if (ret < 0) {
if (errno == ESTALE) {
*retry_r = TRUE;
return 0;
}
mail_index_set_syscall_error(index, "pread_full()");
return -1;
}
if (ret == 0) {
mail_index_set_error(index,
"Corrupted index file %s: File too small",
index->filepath);
return 0;
}
map->records = data;
map->records_count = hdr.messages_count;
mail_index_map_copy_hdr(map, &hdr);
map->hdr_base = map->hdr_copy_buf->data;
index->sync_log_file_seq = hdr.log_file_seq;
index->sync_log_file_offset = hdr.log_file_int_offset;
return 1;
}
static int mail_index_sync_from_transactions(struct mail_index *index,
struct mail_index_map **map,
int sync_to_index)
{
const struct mail_index_header *map_hdr = &(*map)->hdr;
struct mail_index_view *view;
struct mail_transaction_log_view *log_view;
struct mail_index_sync_map_ctx sync_map_ctx;
struct mail_index_header hdr;
const struct mail_transaction_header *thdr;
const void *tdata;
uint32_t prev_seq, max_seq;
uoff_t prev_offset, max_offset;
size_t pos;
int ret, skipped;
if (sync_to_index) {
/* read the real log position where we are supposed to be
synced */
ret = mail_index_read_header(index, &hdr, &pos);
if (ret < 0 && errno != ESTALE) {
mail_index_set_syscall_error(index, "pread()");
return -1;
}
if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
return 0;
if (map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset == hdr.log_file_int_offset) {
/* nothing to do */
return 1;
}
if (map_hdr->log_file_seq > hdr.log_file_seq ||
(map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset > hdr.log_file_int_offset)) {
/* we went too far, have to re-read the file */
return 0;
}
if (map_hdr->log_file_ext_offset !=
map_hdr->log_file_int_offset ||
hdr.log_file_ext_offset != hdr.log_file_int_offset) {
/* too much trouble to get this right. */
return 0;
}
max_seq = hdr.log_file_seq;
max_offset = hdr.log_file_int_offset;
} else {
/* sync everything there is */
max_seq = (uint32_t)-1;
max_offset = (uoff_t)-1;
}
log_view = mail_transaction_log_view_open(index->log);
if (mail_transaction_log_view_set(log_view,
map_hdr->log_file_seq,
map_hdr->log_file_int_offset,
max_seq, max_offset,
MAIL_TRANSACTION_TYPE_MASK) < 0) {
mail_transaction_log_view_close(log_view);
return 0;
}
index->map = *map;
view = mail_index_view_open(index);
mail_index_sync_map_init(&sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_VIEW);
while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,
&skipped)) > 0) {
if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {
ret = -1;
break;
}
}
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
i_assert(prev_seq <= max_seq &&
(prev_seq != max_seq || prev_offset <= max_offset));
index->map->hdr.log_file_seq = prev_seq;
index->map->hdr.log_file_int_offset =
index->map->hdr.log_file_ext_offset = prev_offset;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(view);
mail_transaction_log_view_close(log_view);
*map = index->map;
index->map = NULL;
if (sync_to_index) {
/* make sure we did everything right. note that although the
message counts should be equal, the flag counters may not */
i_assert(hdr.messages_count == (*map)->hdr.messages_count);
i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq);
i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset);
i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset);
}
return ret < 0 ? -1 : 1;
}
static int mail_index_read_map_with_retry(struct mail_index *index,
struct mail_index_map **map,
int sync_to_index)
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
int ret, retry;
if (index->log_locked) {
/* we're most likely syncing the index and we really don't
want to read more than what was synced last time. */
sync_to_index = TRUE;
}
if ((*map)->hdr.indexid != 0 && index->log != NULL) {
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
index->sync_update = TRUE;
ret = mail_index_sync_from_transactions(index, map,
sync_to_index);
index->sync_update = FALSE;
if (ret != 0)
return ret;
/* transaction log lost/broken, fallback to re-reading it */
}
/* notify all "sync lost" handlers */
handlers = array_get(&index->sync_lost_handlers, &count);
for (i = 0; i < count; i++)
(*handlers[i])(index);
for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) {
ret = mail_index_read_map(index, *map, &retry);
if (ret != 0 || !retry)
return ret;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
/* Too many ESTALE retries */
mail_index_set_syscall_error(index, "read_map()");
return -1;
}
static int mail_index_map_try_existing(struct mail_index_map *map)
{
const struct mail_index_header *hdr;
size_t used_size;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
return 0;
hdr = map->mmap_base;
/* always check corrupted-flag to avoid errors later */
if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)
return -1;
used_size = hdr->header_size + hdr->messages_count * hdr->record_size;
if (map->mmap_size >= used_size && map->hdr_base == hdr) {
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
return 1;
}
return 0;
}
int mail_index_map(struct mail_index *index, int force)
{
struct mail_index_map *map;
int ret;
i_assert(!index->mapping);
i_assert(index->map == NULL || index->map->refcount > 0);
i_assert(index->lock_type != F_UNLCK);
if (MAIL_INDEX_IS_IN_MEMORY(index)) {
if (index->map == NULL)
mail_index_create_in_memory(index, NULL);
return 1;
}
index->mapping = TRUE;
if (!force && index->map != NULL) {
i_assert(index->hdr != NULL);
ret = mail_index_map_try_existing(index->map);
if (ret != 0) {
index->mapping = FALSE;
return ret;
}
if (index->lock_type == F_WRLCK) {
/* we're syncing, don't break the mapping */
index->mapping = FALSE;
return 1;
}
}
if (index->map != NULL && index->map->refcount > 1) {
/* this map is already used by some views and they may have
pointers into it. leave them and create a new mapping. */
if (!index->mmap_disable) {
map = NULL;
} else {
/* create a copy of the mapping instead so we don't
have to re-read it */
map = mail_index_map_clone(index->map,
index->map->hdr.record_size);
}
index->map->refcount--;
index->map = NULL;
} else {
map = index->map;
}
if (map == NULL) {
map = i_new(struct mail_index_map, 1);
map->refcount = 1;
map->hdr_copy_buf =
buffer_create_dynamic(default_pool, sizeof(map->hdr));
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
i_assert(!map->write_to_disk);
} else if (map->mmap_base != NULL) {
i_assert(map->buffer == NULL);
if (munmap(map->mmap_base, map->mmap_size) < 0)
mail_index_set_syscall_error(index, "munmap()");
map->mmap_base = NULL;
}
index->hdr = NULL;
index->map = NULL;
if (!index->mmap_disable)
ret = mail_index_mmap(index, map);
else
ret = mail_index_read_map_with_retry(index, &map, force);
i_assert(index->map == NULL);
if (ret > 0) {
ret = mail_index_check_header(index, map);
if (ret < 0)
ret = 0;
else if (ret == 0) {
index->fsck = TRUE;
ret = 1;
}
}
if (ret <= 0) {
mail_index_map_clear(index, map);
mail_index_unmap(index, map);
index->mapping = FALSE;
return ret;
}
index->hdr = &map->hdr;
index->map = map;
i_assert(map->hdr.messages_count == map->records_count);
index->mapping = FALSE;
return 1;
}
int mail_index_get_latest_header(struct mail_index *index,
struct mail_index_header *hdr_r)
{
size_t pos;
int ret;
if (!index->mmap_disable) {
ret = mail_index_map(index, FALSE);
if (ret > 0)
*hdr_r = *index->hdr;
else
memset(hdr_r, 0, sizeof(*hdr_r));
} else {
ret = mail_index_read_header(index, hdr_r, &pos);
}
return ret;
}
struct mail_index_map *
mail_index_map_clone(struct mail_index_map *map, uint32_t new_record_size)
{
struct mail_index_map *mem_map;
struct mail_index_header *hdr;
struct mail_index_ext *extensions;
void *src, *dest;
size_t size, copy_size;
unsigned int i, count;
size = map->records_count * new_record_size;
mem_map = i_new(struct mail_index_map, 1);
mem_map->refcount = 1;
mem_map->buffer = buffer_create_dynamic(default_pool, size);
if (map->hdr.record_size == new_record_size)
buffer_append(mem_map->buffer, map->records, size);
else {
copy_size = I_MIN(map->hdr.record_size, new_record_size);
src = map->records;
for (i = 0; i < map->records_count; i++) {
dest = buffer_append_space_unsafe(mem_map->buffer,
new_record_size);
memcpy(dest, src, copy_size);
src = PTR_OFFSET(src, map->hdr.record_size);
}
}
mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL);
mem_map->records_count = map->records_count;
mem_map->hdr_copy_buf =
buffer_create_dynamic(default_pool, map->hdr.header_size);
if (map->hdr.base_header_size < sizeof(*hdr))
buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr));
buffer_write(mem_map->hdr_copy_buf, 0,
&map->hdr, map->hdr.base_header_size);
buffer_append(mem_map->hdr_copy_buf,
CONST_PTR_OFFSET(map->hdr_base,
map->hdr.base_header_size),
map->hdr.header_size - map->hdr.base_header_size);
hdr = buffer_get_modifyable_data(mem_map->hdr_copy_buf, NULL);
if (hdr->base_header_size < sizeof(*hdr))
hdr->base_header_size = sizeof(*hdr);
hdr->record_size = new_record_size;
mem_map->hdr = *hdr;
mem_map->hdr_base = hdr;
/* copy extensions */
if (array_is_created(&map->ext_id_map)) {
count = array_count(&map->ext_id_map);
mail_index_map_init_extbufs(mem_map, count + 2);
array_append_array(&mem_map->extensions, &map->extensions);
array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
/* fix the name pointers to use our own pool */
extensions = array_get_modifyable(&mem_map->extensions, &count);
for (i = 0; i < count; i++) {
extensions[i].name = p_strdup(mem_map->extension_pool,
extensions[i].name);
}
}
return mem_map;
}
int mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r)
{
const uint32_t *id;
if (!array_is_created(&map->ext_id_map) ||
ext_id >= array_count(&map->ext_id_map))
return 0;
id = array_idx(&map->ext_id_map, ext_id);
*idx_r = *id;
return *idx_r != (uint32_t)-1;
}
static int mail_index_try_open_only(struct mail_index *index)
{
int i;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
for (i = 0; i < 3; i++) {
index->fd = open(index->filepath, O_RDWR);
if (index->fd == -1 && errno == EACCES) {
index->fd = open(index->filepath, O_RDONLY);
index->readonly = TRUE;
}
if (index->fd != -1 || errno != ESTALE)
break;
/* May happen with some OSes with NFS. Try again, although
there's still a race condition with another computer
creating the index file again. However, we can't try forever
as ESTALE happens also if index directory has been deleted
from server.. */
}
if (index->fd == -1) {
if (errno != ENOENT)
return mail_index_set_syscall_error(index, "open()");
/* have to create it */
return 0;
}
return 1;
}
static int
mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)
{
unsigned int lock_id;
int ret;
if (lock_id_r != NULL)
*lock_id_r = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
ret = mail_index_try_open_only(index);
if (ret <= 0)
return ret;
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) {
(void)close(index->fd);
index->fd = -1;
return -1;
}
ret = mail_index_map(index, FALSE);
if (ret == 0) {
/* it's corrupted - recreate it */
mail_index_unlock(index, lock_id);
if (lock_id_r != NULL)
*lock_id_r = 0;
(void)close(index->fd);
index->fd = -1;
} else {
if (lock_id_r != NULL)
*lock_id_r = lock_id;
else
mail_index_unlock(index, lock_id);
}
return ret;
}
int mail_index_write_base_header(struct mail_index *index,
const struct mail_index_header *hdr)
{
size_t hdr_size;
hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
memcpy(index->map->mmap_base, hdr, hdr_size);
if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
return mail_index_set_syscall_error(index, "msync()");
index->map->hdr = *hdr;
} else {
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
mail_index_set_syscall_error(index,
"pwrite_full()");
return -1;
}
}
index->map->hdr = *hdr;
buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size);
}
return 0;
}
int mail_index_create_tmp_file(struct mail_index *index, const char **path_r)
{
mode_t old_mask;
const char *path;
int fd;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
path = *path_r = t_strconcat(index->filepath, ".tmp", NULL);
old_mask = umask(0);
fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode);
umask(old_mask);
if (fd == -1)
return mail_index_file_set_syscall_error(index, path, "open()");
if (index->gid != (gid_t)-1 && fchown(fd, (uid_t)-1, index->gid) < 0) {
mail_index_file_set_syscall_error(index, path, "fchown()");
return -1;
}
return fd;
}
static int mail_index_create(struct mail_index *index,
struct mail_index_header *hdr)
{
const char *path;
uint32_t seq;
uoff_t offset;
int ret;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->lock_type == F_UNLCK);
/* log file lock protects index creation */
if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
return -1;
ret = mail_index_try_open(index, NULL);
if (ret != 0) {
mail_transaction_log_sync_unlock(index->log);
return ret < 0 ? -1 : 0;
}
/* mark the existing log file as synced */
hdr->log_file_seq = seq;
hdr->log_file_int_offset = offset;
hdr->log_file_ext_offset = offset;
/* create it fully in index.tmp first */
index->fd = mail_index_create_tmp_file(index, &path);
if (index->fd == -1)
ret = -1;
else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
mail_index_file_set_syscall_error(index, path, "write_full()");
ret = -1;
} else {
index->lock_type = F_WRLCK;
ret = mail_index_map(index, FALSE);
index->lock_type = F_UNLCK;
}
if (ret == 0) {
/* it's corrupted even while we just created it,
should never happen unless someone pokes the file directly */
mail_index_set_error(index,
"Newly created index file is corrupted: %s", path);
ret = -1;
}
if (ret < 0) {
if (unlink(path) < 0 && errno != ENOENT) {
mail_index_file_set_syscall_error(index, path,
"unlink()");
}
} else {
/* make it visible to others */
if (rename(path, index->filepath) < 0) {
mail_index_set_error(index, "rename(%s, %s) failed: %m",
path, index->filepath);
ret = -1;
}
}
mail_transaction_log_sync_unlock(index->log);
return ret;
}
static void mail_index_header_init(struct mail_index_header *hdr)
{
time_t now = time(NULL);
i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
memset(hdr, 0, sizeof(*hdr));
hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
hdr->base_header_size = sizeof(*hdr);
hdr->header_size = sizeof(*hdr);
hdr->record_size = sizeof(struct mail_index_record);
#ifndef WORDS_BIGENDIAN
hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
hdr->indexid = now;
hdr->next_uid = 1;
}
static void mail_index_create_in_memory(struct mail_index *index,
const struct mail_index_header *hdr)
{
struct mail_index_header tmp_hdr;
struct mail_index_map tmp_map;
if (hdr == NULL) {
mail_index_header_init(&tmp_hdr);
hdr = &tmp_hdr;
}
memset(&tmp_map, 0, sizeof(tmp_map));
tmp_map.hdr = *hdr;
tmp_map.hdr_base = hdr;
/* a bit kludgy way to do this, but it initializes everything
nicely and correctly */
index->map = mail_index_map_clone(&tmp_map, hdr->record_size);
index->hdr = &index->map->hdr;
}
/* returns -1 = error, 0 = won't create, 1 = ok */
static int mail_index_open_files(struct mail_index *index,
enum mail_index_open_flags flags)
{
struct mail_index_header hdr;
unsigned int lock_id = 0;
int ret, created = FALSE;
ret = mail_index_try_open(index, &lock_id);
if (ret > 0)
hdr = *index->hdr;
else if (ret == 0) {
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
return 0;
mail_index_header_init(&hdr);
index->hdr = &hdr;
} else if (ret < 0)
return -1;
index->indexid = hdr.indexid;
index->log = mail_transaction_log_open_or_create(index);
if (index->log == NULL)
return -1;
if (index->fd == -1) {
if (index->indexid != hdr.indexid) {
/* looks like someone else created the transaction log
before we had the chance. use its indexid so we
don't try to create conflicting ones. */
hdr.indexid = index->indexid;
}
if (lock_id != 0) {
mail_index_unlock(index, lock_id);
lock_id = 0;
}
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (mail_index_create(index, &hdr) < 0)
return -1;
} else {
mail_index_create_in_memory(index, &hdr);
}
created = TRUE;
}
if (lock_id == 0) {
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0)
return -1;
}
index->cache = created ? mail_cache_create(index) :
mail_cache_open_or_create(index);
mail_index_unlock(index, lock_id);
return 1;
}
int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags,
enum mail_index_lock_method lock_method)
{
int i = 0, ret;
if (index->opened) {
if (index->hdr != NULL &&
(index->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
/* corrupted, reopen files */
mail_index_close(index);
} else {
return 0;
}
}
index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
i_strdup("(in-memory index)") :
i_strconcat(index->dir, "/", index->prefix, NULL);
for (;;) {
index->shared_lock_count = 0;
index->excl_lock_count = 0;
index->lock_type = F_UNLCK;
index->lock_id = 2;
index->readonly = FALSE;
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
index->log_locked = FALSE;
index->mmap_disable =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
index->mmap_no_write =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0;
index->lock_method = lock_method;
/* don't even bother to handle dotlocking without mmap being
disabled. that combination simply doesn't make any sense */
if (lock_method == MAIL_INDEX_LOCK_DOTLOCK &&
!index->mmap_disable) {
i_fatal("lock_method=dotlock and mmap_disable=no "
"combination isn't supported. "
"You don't _really_ want it anyway.");
}
ret = mail_index_open_files(index, flags);
if (ret <= 0)
break;
index->opened = TRUE;
if (index->fsck) {
index->fsck = FALSE;
ret = mail_index_fsck(index);
if (ret == 0) {
/* completely broken, reopen */
if (i++ < 3)
continue;
/* too many tries */
ret = -1;
}
}
break;
}
if (ret <= 0)
mail_index_close(index);
return ret;
}
void mail_index_close(struct mail_index *index)
{
if (index->log != NULL) {
mail_transaction_log_close(index->log);
index->log = NULL;
}
if (index->map != NULL) {
mail_index_unmap(index, index->map);
index->map = NULL;
}
if (index->cache != NULL) {
mail_cache_free(index->cache);
index->cache = NULL;
}
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
}
i_free(index->copy_lock_path);
index->copy_lock_path = NULL;
i_free(index->filepath);
index->filepath = NULL;
index->indexid = 0;
index->opened = FALSE;
}
int mail_index_reopen(struct mail_index *index, int fd)
{
struct mail_index_map *old_map;
unsigned int old_shared_locks, old_lock_id, lock_id = 0;
int ret, old_fd, old_lock_type;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
old_map = index->map;
old_fd = index->fd;
old_map->refcount++;
/* new file, new locks. the old fd can keep its locks, they don't
matter anymore as no-one's going to modify the file. */
old_lock_type = index->lock_type;
old_lock_id = index->lock_id;
old_shared_locks = index->shared_lock_count;
if (index->lock_type == F_RDLCK)
index->lock_type = F_UNLCK;
index->lock_id += 2;
index->shared_lock_count = 0;
if (fd != -1) {
index->fd = fd;
ret = 0;
} else {
i_assert(index->excl_lock_count == 0);
ret = mail_index_try_open_only(index);
if (ret > 0)
ret = mail_index_lock_shared(index, FALSE, &lock_id);
else if (ret == 0) {
/* index file is lost */
ret = -1;
}
}
if (ret == 0) {
/* read the new mapping. note that with mmap_disable we want
to keep the old mapping in index->map so we can update it
by reading transaction log. */
if (mail_index_map(index, TRUE) <= 0)
ret = -1;
}
if (lock_id != 0)
mail_index_unlock(index, lock_id);
if (ret == 0) {
mail_index_unmap(index, old_map);
if (close(old_fd) < 0)
mail_index_set_syscall_error(index, "close()");
} else {
if (index->map != NULL)
mail_index_unmap(index, index->map);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
}
index->map = old_map;
index->hdr = &index->map->hdr;
index->fd = old_fd;
index->lock_type = old_lock_type;
index->lock_id = old_lock_id;
index->shared_lock_count = old_shared_locks;
}
return ret;
}
int mail_index_reopen_if_needed(struct mail_index *index)
{
struct stat st1, st2;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (fstat(index->fd, &st1) < 0)
return mail_index_set_syscall_error(index, "fstat()");
if (stat(index->filepath, &st2) < 0) {
mail_index_set_syscall_error(index, "stat()");
if (errno != ENOENT)
return -1;
/* lost it? recreate */
(void)mail_index_mark_corrupted(index);
return -1;
}
if (st1.st_ino != st2.st_ino ||
!CMP_DEV_T(st1.st_dev, st2.st_dev)) {
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
} else {
return 0;
}
}
int mail_index_refresh(struct mail_index *index)
{
unsigned int lock_id;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (index->excl_lock_count > 0) {
/* we have index exclusively locked, nothing could
have changed. */
return 0;
}
if (!index->mmap_disable) {
/* reopening is all we need */
return mail_index_reopen_if_needed(index);
}
i_assert(!index->mapping);
/* mail_index_map() simply reads latest changes from transaction log,
which makes us fully refreshed. */
if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
return -1;
ret = mail_index_map(index, FALSE);
mail_index_unlock(index, lock_id);
return ret <= 0 ? -1 : 0;
}
struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
return index->cache;
}
int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
va_list va;
i_free(index->error);
if (fmt == NULL)
index->error = NULL;
else {
va_start(va, fmt);
index->error = i_strdup_vprintf(fmt, va);
va_end(va);
i_error("%s", index->error);
}
return -1;
}
void mail_index_set_inconsistent(struct mail_index *index)
{
index->indexid = 0;
}
void mail_index_mark_corrupted(struct mail_index *index)
{
struct mail_index_header hdr;
mail_index_set_inconsistent(index);
if (index->readonly)
return;
hdr = *index->hdr;
hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
if (mail_index_write_base_header(index, &hdr) == 0) {
if (!MAIL_INDEX_IS_IN_MEMORY(index) && fsync(index->fd) < 0)
mail_index_set_syscall_error(index, "fsync()");
}
}
int mail_index_set_syscall_error(struct mail_index *index,
const char *function)
{
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with index file %s: %m",
function, index->filepath);
}
int mail_index_file_set_syscall_error(struct mail_index *index,
const char *filepath,
const char *function)
{
i_assert(filepath != NULL);
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with file %s: %m",
function, filepath);
}
enum mail_index_error mail_index_get_last_error(struct mail_index *index)
{
if (index->nodiskspace)
return MAIL_INDEX_ERROR_DISKSPACE;
if (index->error != NULL)
return MAIL_INDEX_ERROR_INTERNAL;
return MAIL_INDEX_ERROR_NONE;
}
const char *mail_index_get_error_message(struct mail_index *index)
{
return index->error;
}
void mail_index_reset_error(struct mail_index *index)
{
if (index->error != NULL) {
i_free(index->error);
index->error = NULL;
}
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
}
uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
unsigned char buf[4];
i_assert(offset < 0x40000000);
i_assert((offset & 3) == 0);
offset >>= 2;
buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
buf[3] = 0x80 | (offset & 0x0000007f);
return *((uint32_t *) buf);
}
uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
const unsigned char *buf = (const unsigned char *) &offset;
if ((offset & 0x80808080) != 0x80808080)
return 0;
return (((uint32_t)buf[3] & 0x7f) << 2) |
(((uint32_t)buf[2] & 0x7f) << 9) |
(((uint32_t)buf[1] & 0x7f) << 16) |
(((uint32_t)buf[0] & 0x7f) << 23);
}