mail-index.c revision 3ec64baf598256c4a2b652899982ea235440a1d4
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen#include "lib.h"
46552a931924c2d743f045e95b08c3ce6beda91aTimo Sirainen#include "array.h"
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen#include "buffer.h"
c5f932968281763df360b9c97cef60f5f80d5e3dTimo Sirainen#include "hash.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "mmap-util.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "nfs-workarounds.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "read-full.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "write-full.h"
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen#include "mail-index-private.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "mail-index-sync-private.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "mail-transaction-log.h"
de754cb78f75e8b3b994cddafe41c9ed1467c33dTimo Sirainen#include "mail-cache.h"
f29756821a4c6b12b73e4a2a3e1c230117a43773Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include <stdio.h>
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include <stddef.h>
53dfcefa9440a49d703e49193819a79be99c9ba6Timo Sirainen#include <time.h>
f0d93763f210ecdb85a115fdd0210a16cfc5ff5cTimo Sirainen#include <sys/stat.h>
53dfcefa9440a49d703e49193819a79be99c9ba6Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenunsigned int mail_index_module_id = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainenstatic void mail_index_create_in_memory(struct mail_index *index,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct mail_index_header *hdr);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Boschstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch{
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch struct mail_index *index;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index = i_new(struct mail_index, 1);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->dir = i_strdup(dir);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->prefix = i_strdup(prefix);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->fd = -1;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->extension_pool =
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch pool_alloconly_create(MEMPOOL_GROWING"index extension", 1024);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch p_array_init(&index->extensions, index->extension_pool, 5);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch i_array_init(&index->sync_lost_handlers, 4);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch array_create(&index->mail_index_module_contexts, default_pool,
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch sizeof(void *), I_MIN(5, mail_index_module_id));
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->mode = 0600;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->gid = (gid_t)-1;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->keywords_ext_id =
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch mail_index_ext_register(index, "keywords", 128, 2, 1);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->keywords_pool = pool_alloconly_create("keywords", 512);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch i_array_init(&index->keywords, 16);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch index->keywords_hash =
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch hash_create(default_pool, index->keywords_pool, 0,
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch strcase_hash, (hash_cmp_callback_t *)strcasecmp);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch return index;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch}
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Boschvoid mail_index_free(struct mail_index **_index)
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index *index = *_index;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
2e78f05b11df23ec2731afaf8f19d5b5240cb29fTimo Sirainen *_index = NULL;
2e78f05b11df23ec2731afaf8f19d5b5240cb29fTimo Sirainen mail_index_close(index);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
d1e7425048c61d71f41f737ba947687198842dc2Timo Sirainen hash_destroy(index->keywords_hash);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen pool_unref(index->extension_pool);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen pool_unref(index->keywords_pool);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen array_free(&index->sync_lost_handlers);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen array_free(&index->keywords);
660b99a7059824676b2b8d6f79b8e15d47df25a2Timo Sirainen array_free(&index->mail_index_module_contexts);
660b99a7059824676b2b8d6f79b8e15d47df25a2Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_free(index->error);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_free(index->dir);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_free(index->prefix);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_free(index);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen}
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainenvoid mail_index_set_permissions(struct mail_index *index,
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen mode_t mode, gid_t gid)
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen{
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen index->mode = mode & 0666;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->gid = gid;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint32_t default_hdr_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint16_t default_record_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint16_t default_record_align)
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen{
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen const struct mail_index_registered_ext *extensions;
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen struct mail_index_registered_ext rext;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen unsigned int i, ext_count;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen extensions = array_get(&index->extensions, &ext_count);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen /* see if it's already there */
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen for (i = 0; i < ext_count; i++) {
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen if (strcmp(extensions[i].name, name) == 0)
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen return i;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen }
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen memset(&rext, 0, sizeof(rext));
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext.name = p_strdup(index->extension_pool, name);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext.index_idx = ext_count;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext.hdr_size = default_hdr_size;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext.record_size = default_record_size;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext.record_align = default_record_align;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen array_append(&index->extensions, &rext, 1);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen return ext_count;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen}
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainenvoid mail_index_register_expunge_handler(struct mail_index *index,
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen uint32_t ext_id, bool call_always,
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen mail_index_expunge_handler_t *cb,
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen void *context)
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen{
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen struct mail_index_registered_ext *rext;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen i_assert(rext->expunge_handler == NULL || rext->expunge_handler == cb);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext->expunge_handler = cb;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext->expunge_context = context;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext->expunge_handler_call_always = call_always;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenvoid mail_index_unregister_expunge_handler(struct mail_index *index,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint32_t ext_id)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen struct mail_index_registered_ext *rext;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(rext->expunge_handler != NULL);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen rext->expunge_handler = NULL;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenvoid mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_sync_handler_t *cb,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen enum mail_index_sync_handler_type type)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index_registered_ext *rext;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch rext = array_idx_modifiable(&index->extensions, ext_id);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(rext->sync_handler.callback == NULL);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen rext->sync_handler.callback = cb;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen rext->sync_handler.type = type;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenvoid mail_index_unregister_sync_handler(struct mail_index *index,
e7dd5065d21c569e5e6ddd713bd345dadd1cf51dTimo Sirainen uint32_t ext_id)
e7dd5065d21c569e5e6ddd713bd345dadd1cf51dTimo Sirainen{
d694a52bce62c52080c3f87a56dcf77030fd2712Timo Sirainen struct mail_index_registered_ext *rext;
d694a52bce62c52080c3f87a56dcf77030fd2712Timo Sirainen
009217abb57a24a4076092e8e4e165545747839eStephan Bosch rext = array_idx_modifiable(&index->extensions, ext_id);
009217abb57a24a4076092e8e4e165545747839eStephan Bosch i_assert(rext->sync_handler.callback != NULL);
009217abb57a24a4076092e8e4e165545747839eStephan Bosch
009217abb57a24a4076092e8e4e165545747839eStephan Bosch rext->sync_handler.callback = NULL;
009217abb57a24a4076092e8e4e165545747839eStephan Bosch rext->sync_handler.type = 0;
009217abb57a24a4076092e8e4e165545747839eStephan Bosch}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenvoid mail_index_register_sync_lost_handler(struct mail_index *index,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_sync_lost_handler_t *cb)
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch{
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch array_append(&index->sync_lost_handlers, &cb, 1);
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch}
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Boschvoid mail_index_unregister_sync_lost_handler(struct mail_index *index,
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch mail_index_sync_lost_handler_t *cb)
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch{
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch mail_index_sync_lost_handler_t *const *handlers;
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch unsigned int i, count;
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch handlers = array_get(&index->sync_lost_handlers, &count);
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch for (i = 0; i < count; i++) {
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen if (handlers[i] == cb) {
ab281fc992907b6cf6c730f672dc3aa4c6685015Timo Sirainen array_delete(&index->sync_lost_handlers, i, 1);
ab281fc992907b6cf6c730f672dc3aa4c6685015Timo Sirainen break;
6d24551e169c0808695db35d7a228f1970a84c75Timo Sirainen }
6d24551e169c0808695db35d7a228f1970a84c75Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
1bf5c6c20f3d51f13d3240cfb46e471074c86276Timo Sirainen unsigned int initial_count)
1bf5c6c20f3d51f13d3240cfb46e471074c86276Timo Sirainen{
1bf5c6c20f3d51f13d3240cfb46e471074c86276Timo Sirainen#define EXTENSION_NAME_APPROX_LEN 20
1bf5c6c20f3d51f13d3240cfb46e471074c86276Timo Sirainen#define EXT_GLOBAL_ALLOC_SIZE \
5a250816ffc4cc5db203f9410ea99b6601c7b91aTimo Sirainen ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#define EXT_PER_ALLOC_SIZE \
1bf5c6c20f3d51f13d3240cfb46e471074c86276Timo Sirainen (EXTENSION_NAME_APPROX_LEN + \
1bf5c6c20f3d51f13d3240cfb46e471074c86276Timo Sirainen sizeof(struct mail_index_ext) + sizeof(uint32_t))
1bf5c6c20f3d51f13d3240cfb46e471074c86276Timo Sirainen size_t size;
1bf5c6c20f3d51f13d3240cfb46e471074c86276Timo Sirainen
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch if (map->extension_pool == NULL) {
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch size = EXT_GLOBAL_ALLOC_SIZE +
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch initial_count * EXT_PER_ALLOC_SIZE;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch map->extension_pool =
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch pool_alloconly_create("map extensions",
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch nearest_power(size));
8ce3071e80b9973230048ecadfcb073fb82cc69fTimo Sirainen } else {
8ce3071e80b9973230048ecadfcb073fb82cc69fTimo Sirainen p_clear(map->extension_pool);
8ce3071e80b9973230048ecadfcb073fb82cc69fTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen /* try to use the existing pool's size for initial_count so
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen we don't grow it unneededly */
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen size = p_get_max_easy_alloc_size(map->extension_pool);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen EXT_PER_ALLOC_SIZE;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen p_array_init(&map->extensions, map->extension_pool, initial_count);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen}
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainenuint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen{
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen const struct mail_index_ext *extensions;
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen unsigned int i, size;
2e2a1d720ed53490e8e5c5031e773d395bd5683dTimo Sirainen
5a250816ffc4cc5db203f9410ea99b6601c7b91aTimo Sirainen if (!array_is_created(&map->extensions))
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return (uint32_t)-1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen extensions = array_get(&map->extensions, &size);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen for (i = 0; i < size; i++) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (strcmp(extensions[i].name, name) == 0)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return i;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return (uint32_t)-1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenuint32_t
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenmail_index_map_register_ext(struct mail_index *index,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index_map *map, const char *name,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint32_t hdr_offset, uint32_t hdr_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint32_t record_offset, uint32_t record_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint32_t record_align, uint32_t reset_id)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index_ext *ext;
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen uint32_t idx, empty_idx = (uint32_t)-1;
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen if (!array_is_created(&map->extensions)) {
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen mail_index_map_init_extbufs(map, 5);
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen idx = 0;
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen } else {
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen idx = array_count(&map->extensions);
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen }
b8afdaa1bffe2f27cd4b02bf3bfbd2d297c8e648Timo Sirainen i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext = array_append_space(&map->extensions);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext->name = p_strdup(map->extension_pool, name);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext->hdr_offset = hdr_offset;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext->hdr_size = hdr_size;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext->record_offset = record_offset;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext->record_size = record_size;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext->record_align = record_align;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext->reset_id = reset_id;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext->index_idx = mail_index_ext_register(index, name, hdr_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen record_size, record_align);
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* Update index ext_id -> map ext_id mapping. Fill non-used
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_ids with (uint32_t)-1 */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen while (array_count(&map->ext_id_map) < ext->index_idx)
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen array_append(&map->ext_id_map, &empty_idx, 1);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return idx;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen}
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic bool size_check(size_t *size_left, size_t size)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (size > *size_left)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return FALSE;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen *size_left -= size;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return TRUE;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic size_t get_align(size_t name_len)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen size_t size = sizeof(struct mail_index_ext_header) + name_len;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int mail_index_parse_extensions(struct mail_index *index,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index_map *map)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
cbaac1e9a69099a2c25e09b1db19bcbf9037e342Timo Sirainen const struct mail_index_ext_header *ext_hdr;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen unsigned int i, old_count;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const char *name;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint32_t ext_id, offset, name_offset;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen size_t size_left;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen /* extension headers always start from 64bit offsets, so if base header
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen doesn't happen to be 64bit aligned we'll skip some bytes */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
de754cb78f75e8b3b994cddafe41c9ed1467c33dTimo Sirainen /* nothing to do, skip allocatations and all */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen old_count = array_count(&index->extensions);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_map_init_extbufs(map, old_count + 5);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
660b99a7059824676b2b8d6f79b8e15d47df25a2Timo Sirainen ext_id = (uint32_t)-1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen for (i = 0; i < old_count; i++)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen array_append(&map->ext_id_map, &ext_id, 1);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen while (offset < map->hdr.header_size) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
e911b23f3e05308df9b98b1a3fdaf72e4302d8fdTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* Extension header contains:
77f1da4b5e2b800197d8db548235497d5e9d6a4fTimo Sirainen - struct mail_index_ext_header
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen - name (not 0-terminated)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen - 64bit alignment padding
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen - extension header contents
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen - 64bit alignment padding
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen size_left = map->hdr.header_size - offset;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (!size_check(&size_left, sizeof(*ext_hdr)) ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen !size_check(&size_left, ext_hdr->name_size) ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen !size_check(&size_left, get_align(ext_hdr->name_size)) ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen !size_check(&size_left, ext_hdr->hdr_size)) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "Header extension goes outside header",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen offset += sizeof(*ext_hdr);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen name_offset = offset;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen offset += ext_hdr->name_size + get_align(ext_hdr->name_size);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen t_push();
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr->name_size);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
18f1bbf05980d3c53ecae81b62574212f0891522Timo Sirainen "Duplicate header extension %s",
77f1da4b5e2b800197d8db548235497d5e9d6a4fTimo Sirainen index->filepath, name);
75e46142d8fbac811df8f2ca58d9a2f48a75d65fTimo Sirainen t_pop();
77f1da4b5e2b800197d8db548235497d5e9d6a4fTimo Sirainen return -1;
18f1bbf05980d3c53ecae81b62574212f0891522Timo Sirainen }
75e46142d8fbac811df8f2ca58d9a2f48a75d65fTimo Sirainen
75e46142d8fbac811df8f2ca58d9a2f48a75d65fTimo Sirainen if (map->hdr.record_size <
c5f932968281763df360b9c97cef60f5f80d5e3dTimo Sirainen ext_hdr->record_offset + ext_hdr->record_size) {
18f1bbf05980d3c53ecae81b62574212f0891522Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
18f1bbf05980d3c53ecae81b62574212f0891522Timo Sirainen "Record field %s points outside record size "
18f1bbf05980d3c53ecae81b62574212f0891522Timo Sirainen "(%u < %u+%u)", index->filepath, name,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen map->hdr.record_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr->record_offset, ext_hdr->record_size);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen t_pop();
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e911b23f3e05308df9b98b1a3fdaf72e4302d8fdTimo Sirainen }
e911b23f3e05308df9b98b1a3fdaf72e4302d8fdTimo Sirainen
e911b23f3e05308df9b98b1a3fdaf72e4302d8fdTimo Sirainen if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen (map->hdr.record_size % ext_hdr->record_align) != 0) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "Record field %s alignmentation %u not used",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath, name, ext_hdr->record_align);
e911b23f3e05308df9b98b1a3fdaf72e4302d8fdTimo Sirainen t_pop();
e911b23f3e05308df9b98b1a3fdaf72e4302d8fdTimo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_map_register_ext(index, map, name,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen offset, ext_hdr->hdr_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr->record_offset,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr->record_size,
2598b2f36365b52d9754b9348a5be29569293e46Timo Sirainen ext_hdr->record_align,
de754cb78f75e8b3b994cddafe41c9ed1467c33dTimo Sirainen ext_hdr->reset_id);
2598b2f36365b52d9754b9348a5be29569293e46Timo Sirainen t_pop();
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 1;
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenbool mail_index_keyword_lookup(struct mail_index *index,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const char *keyword, bool autocreate,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen unsigned int *idx_r)
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen char *keyword_dup;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen void *value;
04b7dc631f33bf61f273138c679da9bd0910fb6dTimo Sirainen
04b7dc631f33bf61f273138c679da9bd0910fb6dTimo Sirainen /* keywords_hash keeps a name => index mapping of keywords.
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen Keywords are never removed from it, so the index values are valid
04b7dc631f33bf61f273138c679da9bd0910fb6dTimo Sirainen for the lifetime of the mail_index. */
04b7dc631f33bf61f273138c679da9bd0910fb6dTimo Sirainen if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen *idx_r = POINTER_CAST_TO(value, unsigned int);
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen return TRUE;
de754cb78f75e8b3b994cddafe41c9ed1467c33dTimo Sirainen }
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen if (!autocreate) {
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen *idx_r = (unsigned int)-1;
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen return FALSE;
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen }
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen *idx_r = array_count(&index->keywords);
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen
de754cb78f75e8b3b994cddafe41c9ed1467c33dTimo Sirainen hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r));
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen array_append(&index->keywords, &keyword, 1);
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen return TRUE;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenint mail_index_map_parse_keywords(struct mail_index *index,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index_map *map)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct mail_index_ext *ext;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct mail_index_keyword_header *kw_hdr;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct mail_index_keyword_header_rec *kw_rec;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const char *name;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen unsigned int i, name_area_end_offset, old_count;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen uint32_t ext_id;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen map->keywords_read = TRUE;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
2598b2f36365b52d9754b9348a5be29569293e46Timo Sirainen ext_id = mail_index_map_lookup_ext(map, "keywords");
2598b2f36365b52d9754b9348a5be29569293e46Timo Sirainen if (ext_id == (uint32_t)-1) {
e7dd5065d21c569e5e6ddd713bd345dadd1cf51dTimo Sirainen if (array_is_created(&map->keyword_idx_map))
e7dd5065d21c569e5e6ddd713bd345dadd1cf51dTimo Sirainen array_clear(&map->keyword_idx_map);
e7dd5065d21c569e5e6ddd713bd345dadd1cf51dTimo Sirainen return 0;
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen }
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen ext = array_idx(&map->extensions, ext_id);
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen /* Extension header contains:
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen - struct mail_index_keyword_header
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen - struct mail_index_keyword_header_rec * keywords_count
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen - const char names[] * keywords_count
c51644e9e04effbbc9c415cadcfbcb4d9465855cTimo Sirainen */
2598b2f36365b52d9754b9348a5be29569293e46Timo Sirainen i_assert(ext->hdr_offset < map->hdr.header_size);
2598b2f36365b52d9754b9348a5be29569293e46Timo Sirainen kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
f0d93763f210ecdb85a115fdd0210a16cfc5ff5cTimo Sirainen kw_rec = (const void *)(kw_hdr + 1);
f0d93763f210ecdb85a115fdd0210a16cfc5ff5cTimo Sirainen name = (const char *)(kw_rec + kw_hdr->keywords_count);
6303191abcb37164f435ccdc56e9dbddf1288851Timo Sirainen
2598b2f36365b52d9754b9348a5be29569293e46Timo Sirainen old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
2598b2f36365b52d9754b9348a5be29569293e46Timo Sirainen array_count(&map->keyword_idx_map);
2ef0e8ee48c9683f7bd6698798efa3328e4322d1Timo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen /* Keywords can only be added into same mapping. Removing requires a
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen new mapping (recreating the index file) */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (kw_hdr->keywords_count == old_count) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* nothing changed */
1093de32efb2a231949566d4bd8aa55a8f43fb70Timo Sirainen return 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* make sure the header is valid */
ab281fc992907b6cf6c730f672dc3aa4c6685015Timo Sirainen if (kw_hdr->keywords_count < old_count) {
6d24551e169c0808695db35d7a228f1970a84c75Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "Keywords removed unexpectedly",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen "keywords_count larger than header size",
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen index->filepath);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return -1;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
280503e88a6b2f72a32a8fbe363794abaaa845d6Timo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
280503e88a6b2f72a32a8fbe363794abaaa845d6Timo Sirainen for (i = 0; i < kw_hdr->keywords_count; i++) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if (kw_rec[i].name_offset > name_area_end_offset) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "name_offset points outside allocated header",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (name[name_area_end_offset-1] != '\0') {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "Keyword header doesn't end with NUL",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* create file -> index mapping */
660b99a7059824676b2b8d6f79b8e15d47df25a2Timo Sirainen if (!array_is_created(&map->keyword_idx_map))
42fcc708268a89aa9640693e71d13a2bb76e19c8Timo Sirainen i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
42fcc708268a89aa9640693e71d13a2bb76e19c8Timo Sirainen
42fcc708268a89aa9640693e71d13a2bb76e19c8Timo Sirainen#ifdef DEBUG
42fcc708268a89aa9640693e71d13a2bb76e19c8Timo Sirainen /* Check that existing headers are still the same. It's behind DEBUG
42fcc708268a89aa9640693e71d13a2bb76e19c8Timo Sirainen since it's pretty useless waste of CPU normally. */
42fcc708268a89aa9640693e71d13a2bb76e19c8Timo Sirainen for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const unsigned int *old_idx;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen unsigned int idx;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen old_idx = array_idx(&map->keyword_idx_map, i);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) ||
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen idx != *old_idx) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen "Keywords changed unexpectedly",
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen index->filepath);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return -1;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen#endif
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen /* Register the newly seen keywords */
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen i = array_count(&map->keyword_idx_map);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen for (; i < kw_hdr->keywords_count; i++) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen const char *keyword = name + kw_rec[i].name_offset;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen unsigned int idx;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen array_append(&map->keyword_idx_map, &idx, 1);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return 0;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch}
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Boschconst ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index)
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch{
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch /* Make sure all the keywords are in index->keywords. It's quick to do
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch if nothing has changed. */
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch (void)mail_index_map_parse_keywords(index, index->map);
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch return &index->keywords;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch}
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Boschstatic int mail_index_check_header(struct mail_index *index,
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen struct mail_index_map *map)
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch{
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen const struct mail_index_header *hdr = &map->hdr;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch enum mail_index_header_compat_flags compat_flags = 0;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch#ifndef WORDS_BIGENDIAN
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch#endif
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch /* major version change - handle silently(?) */
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return -1;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch }
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch if (hdr->compat_flags != compat_flags) {
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch /* architecture change - handle silently(?) */
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch return -1;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen /* we've already complained about it */
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return -1;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen /* following some extra checks that only take a bit of CPU */
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen "uid_validity = 0, next_uid = %u",
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen index->filepath, hdr->next_uid);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return -1;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if (hdr->record_size < sizeof(struct mail_index_record)) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen "record_size too small: %u < %"PRIuSIZE_T,
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen index->filepath, hdr->record_size,
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen sizeof(struct mail_index_record));
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return -1;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return 0;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if (hdr->next_uid == 0)
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (hdr->recent_messages_count > hdr->messages_count ||
decb23442f9e6cd5c4845a9cb162029b8c6d5f0fTimo Sirainen hdr->seen_messages_count > hdr->messages_count ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen hdr->deleted_messages_count > hdr->messages_count)
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen return 0;
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen if (hdr->first_recent_uid_lowwater > hdr->next_uid ||
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen hdr->first_unseen_uid_lowwater > hdr->next_uid ||
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen hdr->first_deleted_uid_lowwater > hdr->next_uid)
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody return 0;
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen if (map->records_count > 0) {
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen /* last message's UID must be smaller than next_uid.
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen also make sure it's not zero. */
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen const struct mail_index_record *rec;
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen rec = MAIL_INDEX_MAP_IDX(map, map->records_count-1);
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen if (rec->uid == 0 || rec->uid >= hdr->next_uid)
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen return 0;
decb23442f9e6cd5c4845a9cb162029b8c6d5f0fTimo Sirainen }
decb23442f9e6cd5c4845a9cb162029b8c6d5f0fTimo Sirainen
7fa573e6ea36024f618492e7d3649a69c1b41028Timo Sirainen return mail_index_parse_extensions(index, map);
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen}
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainenstatic void mail_index_map_clear(struct mail_index *index,
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen struct mail_index_map *map)
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (map->buffer != NULL) {
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen i_assert(map->mmap_base == NULL);
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen buffer_free(map->buffer);
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen map->buffer = NULL;
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen } else if (map->mmap_base != NULL) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(map->buffer == NULL);
f0339f522dc9c8e2e8a29ef9a3f937c431c6bd1bTimo Sirainen if (munmap(map->mmap_base, map->mmap_size) < 0)
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen mail_index_set_syscall_error(index, "munmap()");
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen map->mmap_base = NULL;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen }
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if (map->refcount > 0) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen map->mmap_size = 0;
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen map->mmap_used_size = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen map->records = NULL;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen map->records_count = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen}
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainenvoid mail_index_unmap(struct mail_index *index, struct mail_index_map **_map)
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen{
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen struct mail_index_map *map = *_map;
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen *_map = NULL;
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen if (--map->refcount > 0)
ab90f702ceedb7ba445a9a592be0b213b27cbafaStephan Bosch return;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(map->refcount == 0);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_map_clear(index, map);
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen if (map->extension_pool != NULL)
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen pool_unref(map->extension_pool);
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen if (array_is_created(&map->keyword_idx_map))
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen array_free(&map->keyword_idx_map);
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen buffer_free(map->hdr_copy_buf);
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen i_free(map);
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen}
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainenstatic void mail_index_map_copy_hdr(struct mail_index_map *map,
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen const struct mail_index_header *hdr)
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen{
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (hdr->base_header_size < sizeof(map->hdr)) {
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen /* header smaller than ours, make a copy so our newer headers
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen won't have garbage in them */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen memcpy(&map->hdr, hdr, hdr->base_header_size);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen } else {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen map->hdr = *hdr;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct mail_index_header *hdr;
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch unsigned int records_count;
6b107c647b2120d3906ff7d368f8a16900f6833fTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(!map->write_to_disk);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (map->buffer != NULL) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* we had temporarily used a buffer, eg. for updating index */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen buffer_free(map->buffer);
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen map->buffer = NULL;
fd3d068169c6ec587c9c446f2ee45560a444334aTimo Sirainen }
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen map->mmap_base = index->readonly ?
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mmap_ro_file(index->fd, &map->mmap_size) :
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mmap_rw_file(index->fd, &map->mmap_size);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if (map->mmap_base == MAP_FAILED) {
6b107c647b2120d3906ff7d368f8a16900f6833fTimo Sirainen map->mmap_base = NULL;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_syscall_error(index, "mmap()");
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen hdr = map->mmap_base;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (map->mmap_size >
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen offsetof(struct mail_index_header, major_version) &&
ce1a6c9b82117d253df9acd77e54ac84dd8a247eTimo Sirainen hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
36b072d84a9076c3c483bf710444a716e987ccc3Stephan Bosch /* major version change - handle silently */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 0;
fd3d068169c6ec587c9c446f2ee45560a444334aTimo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "File too small (%"PRIuSIZE_T")",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath, map->mmap_size);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen return 0;
6b107c647b2120d3906ff7d368f8a16900f6833fTimo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen map->mmap_used_size = hdr->header_size +
6b107c647b2120d3906ff7d368f8a16900f6833fTimo Sirainen hdr->messages_count * hdr->record_size;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (map->mmap_used_size > map->mmap_size) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen records_count = (map->mmap_size - hdr->header_size) /
6b107c647b2120d3906ff7d368f8a16900f6833fTimo Sirainen hdr->record_size;
63e376747537cb2dfaa0e36d1bafd19df1443a4eTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "messages_count too large (%u > %u)",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath, hdr->messages_count,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen records_count);
660b99a7059824676b2b8d6f79b8e15d47df25a2Timo Sirainen return 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
7a23d586f07ec376e28e8f6f3f3392a4ac8b83bbTimo Sirainen mail_index_map_copy_hdr(map, hdr);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen map->hdr_base = map->mmap_base;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
6b107c647b2120d3906ff7d368f8a16900f6833fTimo Sirainen map->records_count = map->hdr.messages_count;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 1;
7a23d586f07ec376e28e8f6f3f3392a4ac8b83bbTimo Sirainen}
0348172a5278d1f5aa2440f30346c390ddc17318Timo Sirainen
0348172a5278d1f5aa2440f30346c390ddc17318Timo Sirainenstatic int mail_index_read_header(struct mail_index *index,
cb2c44f33d9d48f58e4c5e42ba2526a0c100218aTimo Sirainen void *buf, size_t buf_size, size_t *pos_r)
0348172a5278d1f5aa2440f30346c390ddc17318Timo Sirainen{
0348172a5278d1f5aa2440f30346c390ddc17318Timo Sirainen size_t pos;
7a23d586f07ec376e28e8f6f3f3392a4ac8b83bbTimo Sirainen int ret;
7a23d586f07ec376e28e8f6f3f3392a4ac8b83bbTimo Sirainen
7a23d586f07ec376e28e8f6f3f3392a4ac8b83bbTimo Sirainen memset(buf, 0, sizeof(struct mail_index_header));
7a23d586f07ec376e28e8f6f3f3392a4ac8b83bbTimo Sirainen
3b22894b8805b186c73d8b754001e8d7e944be85Timo Sirainen /* try to read the whole header, but it's not necessarily an error to
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen read less since the older versions of the index format could be
ab90f702ceedb7ba445a9a592be0b213b27cbafaStephan Bosch smaller. Request reading up to buf_size, but accept if we only got
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen the header. */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen pos = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen do {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ret = pread(index->fd, PTR_OFFSET(buf, pos),
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen buf_size - pos, pos);
decb23442f9e6cd5c4845a9cb162029b8c6d5f0fTimo Sirainen if (ret > 0)
decb23442f9e6cd5c4845a9cb162029b8c6d5f0fTimo Sirainen pos += ret;
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen } while (ret > 0 && pos < sizeof(struct mail_index_header));
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
6998ca95b4947c90647ac5d4794ebd6311acada2Timo Sirainen *pos_r = pos;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return ret;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenmail_index_read_map(struct mail_index *index, struct mail_index_map *map,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen bool *retry_r, bool try_retry)
b66d803de86bfb411165b3465b0d9ef64ecfe2a1Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct mail_index_header *hdr;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct stat st;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen unsigned char buf[512];
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen void *data = NULL;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ssize_t ret;
f29756821a4c6b12b73e4a2a3e1c230117a43773Timo Sirainen size_t pos, records_size;
b11269887905780bc8cb7762bbb157aa59961cacTimo Sirainen unsigned int records_count;
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen i_assert(map->mmap_base == NULL);
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen *retry_r = FALSE;
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen ret = mail_index_read_header(index, buf, sizeof(buf), &pos);
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen hdr = (const struct mail_index_header *)buf;
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
d3a7d023b47d2a137f01109e7b38702dca3f11d3Timo Sirainen /* major version change - handle silently */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
5a250816ffc4cc5db203f9410ea99b6601c7b91aTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (fstat(index->fd, &st) < 0) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_syscall_error(index, "fstat()");
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen (ret > 0 || pos >= hdr->base_header_size)) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen hdr->header_size < hdr->base_header_size) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "Corrupted header sizes (base %u, full %u)",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath, hdr->base_header_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen hdr->header_size);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 0;
e2ce85866a669d1881d2d31791619d2e7c63c253Timo Sirainen }
e2ce85866a669d1881d2d31791619d2e7c63c253Timo Sirainen if (hdr->header_size > (uoff_t)st.st_size) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "Corrupted header size (%u > %"PRIuUOFF_T")",
b11269887905780bc8cb7762bbb157aa59961cacTimo Sirainen index->filepath, hdr->header_size,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen st.st_size);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if (pos > hdr->header_size)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen pos = hdr->header_size;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* place the base header into memory. */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen buffer_reset(map->hdr_copy_buf);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen buffer_append(map->hdr_copy_buf, buf, pos);
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (pos != hdr->header_size) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* @UNSAFE: read the rest of the header into memory */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen data = buffer_append_space_unsafe(map->hdr_copy_buf,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen hdr->header_size -
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen pos);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ret = pread_full(index->fd, data,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen hdr->header_size - pos, pos);
80e461e945dd4d5b0d08535413176cf1756e8523Timo Sirainen }
80e461e945dd4d5b0d08535413176cf1756e8523Timo Sirainen }
80e461e945dd4d5b0d08535413176cf1756e8523Timo Sirainen
80e461e945dd4d5b0d08535413176cf1756e8523Timo Sirainen if (ret > 0) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* header read, read the records now. */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen records_size = (size_t)hdr->messages_count * hdr->record_size;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen if ((uoff_t)st.st_size - hdr->header_size < records_size ||
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen records_size / hdr->messages_count != hdr->record_size) {
9f627b360ed38fdc54cb02ec5e67246c3f0d5b0fTimo Sirainen records_count = (st.st_size - hdr->header_size) /
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen hdr->record_size;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "messages_count too large (%u > %u)",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen index->filepath, hdr->messages_count,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen records_count);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 0;
}
if (map->buffer == NULL) {
map->buffer = buffer_create_dynamic(default_pool,
records_size);
}
/* @UNSAFE */
buffer_set_used_size(map->buffer, 0);
data = buffer_append_space_unsafe(map->buffer, records_size);
ret = pread_full(index->fd, data, records_size,
hdr->header_size);
}
if (ret < 0) {
if (errno == ESTALE && try_retry) {
/* a new index file was renamed over this one. */
*retry_r = TRUE;
return 0;
}
mail_index_set_syscall_error(index, "pread_full()");
return -1;
}
if (ret == 0) {
mail_index_set_error(index,
"Corrupted index file %s: File too small",
index->filepath);
return 0;
}
map->records = data;
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
map->hdr_base = map->hdr_copy_buf->data;
index->sync_log_file_seq = hdr->log_file_seq;
index->sync_log_file_offset = hdr->log_file_int_offset;
return 1;
}
bool mail_index_is_ext_synced(struct mail_transaction_log_view *log_view,
struct mail_index_map *map)
{
uint32_t prev_seq;
uoff_t prev_offset;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
return prev_seq < map->hdr.log_file_seq ||
(prev_seq == map->hdr.log_file_seq &&
prev_offset < map->hdr.log_file_ext_offset);
}
static int mail_index_sync_from_transactions(struct mail_index *index,
struct mail_index_map **map,
bool sync_to_index)
{
const struct mail_index_header *map_hdr = &(*map)->hdr;
struct mail_index_view *view;
struct mail_transaction_log_view *log_view;
struct mail_index_sync_map_ctx sync_map_ctx;
struct mail_index_header hdr;
const struct mail_transaction_header *thdr;
const void *tdata;
uint32_t prev_seq, max_seq;
uoff_t prev_offset, max_offset;
size_t pos;
int ret;
bool skipped, check_ext_offsets, broken;
if (sync_to_index) {
/* read the real log position where we are supposed to be
synced */
ret = mail_index_read_header(index, &hdr, sizeof(hdr), &pos);
if (ret < 0 && errno != ESTALE) {
mail_index_set_syscall_error(index, "pread()");
return -1;
}
if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
return 0;
if (map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset == hdr.log_file_int_offset) {
/* nothing to do */
return 1;
}
if (map_hdr->log_file_seq > hdr.log_file_seq ||
(map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset > hdr.log_file_int_offset)) {
/* we went too far, have to re-read the file */
return 0;
}
if (map_hdr->log_file_ext_offset !=
map_hdr->log_file_int_offset ||
hdr.log_file_ext_offset != hdr.log_file_int_offset) {
/* too much trouble to get this right. */
return 0;
}
max_seq = hdr.log_file_seq;
max_offset = hdr.log_file_int_offset;
} else {
/* sync everything there is */
max_seq = (uint32_t)-1;
max_offset = (uoff_t)-1;
}
log_view = mail_transaction_log_view_open(index->log);
if (mail_transaction_log_view_set(log_view,
map_hdr->log_file_seq,
map_hdr->log_file_int_offset,
max_seq, max_offset,
MAIL_TRANSACTION_TYPE_MASK) <= 0) {
/* can't use it. sync by re-reading index. */
mail_transaction_log_view_close(&log_view);
return 0;
}
index->map = *map;
view = mail_index_view_open(index);
mail_index_sync_map_init(&sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_HEAD);
check_ext_offsets = TRUE; broken = FALSE;
while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,
&skipped)) > 0) {
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0 &&
check_ext_offsets) {
if (mail_index_is_ext_synced(log_view, index->map))
continue;
check_ext_offsets = FALSE;
}
if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {
ret = 0;
broken = TRUE;
break;
}
}
if (ret == 0 && !broken)
ret = 1;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
i_assert(prev_seq <= max_seq &&
(prev_seq != max_seq || prev_offset <= max_offset));
index->map->hdr.log_file_seq = prev_seq;
index->map->hdr.log_file_int_offset =
index->map->hdr.log_file_ext_offset = prev_offset;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(&view);
mail_transaction_log_view_close(&log_view);
*map = index->map;
index->map = NULL;
if (sync_to_index && ret > 0) {
/* make sure we did everything right. note that although the
message counts should be equal, the flag counters may not */
i_assert(hdr.messages_count == (*map)->hdr.messages_count);
i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq);
i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset);
i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset);
}
return ret;
}
static int mail_index_read_map_with_retry(struct mail_index *index,
struct mail_index_map **map,
bool sync_to_index)
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
int ret;
bool retry;
if (index->log_locked) {
/* we're most likely syncing the index and we really don't
want to read more than what was synced last time. */
sync_to_index = TRUE;
}
if ((*map)->hdr.indexid != 0 && index->log != NULL) {
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
index->sync_update = TRUE;
ret = mail_index_sync_from_transactions(index, map,
sync_to_index);
index->sync_update = FALSE;
if (ret != 0)
return ret;
/* transaction log lost/broken, fallback to re-reading it */
}
/* notify all "sync lost" handlers */
handlers = array_get(&index->sync_lost_handlers, &count);
for (i = 0; i < count; i++)
(*handlers[i])(index);
for (i = 0;; i++) {
ret = mail_index_read_map(index, *map, &retry,
i < MAIL_INDEX_ESTALE_RETRY_COUNT);
if (ret != 0 || !retry)
return ret;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
}
static int mail_index_map_try_existing(struct mail_index *index)
{
struct mail_index_map *map = index->map;
const struct mail_index_header *hdr;
size_t used_size;
int ret;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
return 0;
hdr = map->mmap_base;
/* always check corrupted-flag to avoid errors later */
if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)
return -1;
used_size = hdr->header_size + hdr->messages_count * hdr->record_size;
if (map->mmap_size >= used_size && map->hdr_base == hdr) {
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
/* make sure the header is still valid. it also re-parses
extensions although they shouldn't change without the whole
index being recreated */
ret = mail_index_check_header(index, map);
if (ret > 0)
return 1;
/* broken. fallback to re-mmaping which will catch it */
}
return 0;
}
int mail_index_map(struct mail_index *index, bool force)
{
struct mail_index_map *map;
int ret;
i_assert(!index->mapping);
i_assert(index->map == NULL || index->map->refcount > 0);
i_assert(index->lock_type != F_UNLCK);
if (MAIL_INDEX_IS_IN_MEMORY(index)) {
if (index->map == NULL)
mail_index_create_in_memory(index, NULL);
return 1;
}
index->mapping = TRUE;
if (!force && index->map != NULL) {
i_assert(index->hdr != NULL);
ret = mail_index_map_try_existing(index);
if (ret != 0) {
index->mapping = FALSE;
return ret;
}
if (index->lock_type == F_WRLCK) {
/* we're syncing, don't break the mapping */
index->mapping = FALSE;
return 1;
}
}
if (index->map != NULL && index->map->refcount > 1) {
/* this map is already used by some views and they may have
pointers into it. leave them and create a new mapping. */
if (!index->mmap_disable) {
map = NULL;
} else {
/* create a copy of the mapping instead so we don't
have to re-read it */
map = mail_index_map_clone(index->map,
index->map->hdr.record_size);
}
index->map->refcount--;
index->map = NULL;
} else {
map = index->map;
}
if (map == NULL) {
map = i_new(struct mail_index_map, 1);
map->refcount = 1;
map->hdr_copy_buf =
buffer_create_dynamic(default_pool, sizeof(map->hdr));
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
i_assert(!map->write_to_disk);
} else if (map->mmap_base != NULL) {
i_assert(map->buffer == NULL);
if (munmap(map->mmap_base, map->mmap_size) < 0)
mail_index_set_syscall_error(index, "munmap()");
map->mmap_base = NULL;
}
index->hdr = NULL;
index->map = NULL;
if (!index->mmap_disable)
ret = mail_index_mmap(index, map);
else
ret = mail_index_read_map_with_retry(index, &map, force);
i_assert(index->map == NULL);
if (ret > 0) {
ret = mail_index_check_header(index, map);
if (ret < 0)
ret = 0;
else if (ret == 0) {
index->fsck = TRUE;
ret = 1;
}
}
if (ret <= 0) {
mail_index_map_clear(index, map);
mail_index_unmap(index, &map);
index->mapping = FALSE;
return ret;
}
index->hdr = &map->hdr;
index->map = map;
i_assert(map->hdr.messages_count == map->records_count);
index->mapping = FALSE;
return 1;
}
int mail_index_get_latest_header(struct mail_index *index,
struct mail_index_header *hdr_r)
{
size_t pos;
unsigned int i;
int ret;
if (!index->mmap_disable) {
ret = mail_index_map(index, FALSE);
if (ret > 0)
*hdr_r = *index->hdr;
else
memset(hdr_r, 0, sizeof(*hdr_r));
return ret;
}
for (i = 0;; i++) {
ret = mail_index_read_header(index, hdr_r, sizeof(*hdr_r),
&pos);
if (ret <= 0 || errno != ESTALE ||
i == MAIL_INDEX_ESTALE_RETRY_COUNT)
break;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
if (ret < 0)
mail_index_set_syscall_error(index, "pread_full()");
return ret;
}
struct mail_index_map *
mail_index_map_clone(const struct mail_index_map *map, uint32_t new_record_size)
{
struct mail_index_map *mem_map;
struct mail_index_header *hdr;
struct mail_index_ext *extensions;
void *src, *dest;
size_t size, copy_size;
unsigned int i, count;
size = map->records_count * new_record_size;
mem_map = i_new(struct mail_index_map, 1);
mem_map->refcount = 1;
mem_map->buffer = buffer_create_dynamic(default_pool, size);
if (map->hdr.record_size == new_record_size)
buffer_append(mem_map->buffer, map->records, size);
else {
copy_size = I_MIN(map->hdr.record_size, new_record_size);
src = map->records;
for (i = 0; i < map->records_count; i++) {
dest = buffer_append_space_unsafe(mem_map->buffer,
new_record_size);
memcpy(dest, src, copy_size);
src = PTR_OFFSET(src, map->hdr.record_size);
}
}
mem_map->records = buffer_get_modifiable_data(mem_map->buffer, NULL);
mem_map->records_count = map->records_count;
mem_map->hdr_copy_buf =
buffer_create_dynamic(default_pool, map->hdr.header_size);
if (map->hdr.base_header_size < sizeof(*hdr))
buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr));
buffer_write(mem_map->hdr_copy_buf, 0,
&map->hdr, map->hdr.base_header_size);
buffer_append(mem_map->hdr_copy_buf,
CONST_PTR_OFFSET(map->hdr_base,
map->hdr.base_header_size),
map->hdr.header_size - map->hdr.base_header_size);
hdr = buffer_get_modifiable_data(mem_map->hdr_copy_buf, NULL);
if (hdr->base_header_size < sizeof(*hdr))
hdr->base_header_size = sizeof(*hdr);
hdr->record_size = new_record_size;
mem_map->hdr = *hdr;
mem_map->hdr_base = hdr;
/* if we're syncing transaction log into memory and later use the
mapping for updating the index, we need to remember what has
changed */
mem_map->write_atomic = map->write_atomic;
if (map->write_to_disk) {
mem_map->write_seq_first = map->write_seq_first;
mem_map->write_seq_last = map->write_seq_last;
}
/* copy extensions */
if (array_is_created(&map->ext_id_map)) {
count = array_count(&map->ext_id_map);
mail_index_map_init_extbufs(mem_map, count + 2);
array_append_array(&mem_map->extensions, &map->extensions);
array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
/* fix the name pointers to use our own pool */
extensions = array_get_modifiable(&mem_map->extensions, &count);
for (i = 0; i < count; i++) {
i_assert(extensions[i].record_offset +
extensions[i].record_size <= hdr->record_size);
extensions[i].name = p_strdup(mem_map->extension_pool,
extensions[i].name);
}
}
return mem_map;
}
int mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r)
{
const uint32_t *id;
if (!array_is_created(&map->ext_id_map) ||
ext_id >= array_count(&map->ext_id_map))
return 0;
id = array_idx(&map->ext_id_map, ext_id);
*idx_r = *id;
return *idx_r != (uint32_t)-1;
}
static int mail_index_try_open_only(struct mail_index *index)
{
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
/* Note that our caller must close index->fd by itself.
mail_index_reopen() for example wants to revert back to old
index file if opening the new one fails. */
index->fd = nfs_safe_open(index->filepath, O_RDWR);
index->readonly = FALSE;
if (index->fd == -1 && errno == EACCES) {
index->fd = open(index->filepath, O_RDONLY);
index->readonly = TRUE;
}
if (index->fd == -1) {
if (errno != ENOENT)
return mail_index_set_syscall_error(index, "open()");
/* have to create it */
return 0;
}
return 1;
}
static int
mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)
{
unsigned int lock_id;
int ret;
i_assert(index->fd == -1);
i_assert(index->lock_type == F_UNLCK);
if (lock_id_r != NULL)
*lock_id_r = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
ret = mail_index_try_open_only(index);
if (ret <= 0)
return ret;
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) {
(void)close(index->fd);
index->fd = -1;
return -1;
}
ret = mail_index_map(index, FALSE);
if (ret == 0) {
/* it's corrupted - recreate it */
mail_index_unlock(index, lock_id);
if (lock_id_r != NULL)
*lock_id_r = 0;
i_assert(index->file_lock == NULL);
(void)close(index->fd);
index->fd = -1;
} else {
if (lock_id_r != NULL)
*lock_id_r = lock_id;
else
mail_index_unlock(index, lock_id);
}
return ret;
}
int mail_index_write_base_header(struct mail_index *index,
const struct mail_index_header *hdr)
{
size_t hdr_size;
hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
memcpy(index->map->mmap_base, hdr, hdr_size);
if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
return mail_index_set_syscall_error(index, "msync()");
index->map->hdr = *hdr;
} else {
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
mail_index_set_syscall_error(index,
"pwrite_full()");
return -1;
}
}
index->map->hdr = *hdr;
buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size);
}
return 0;
}
int mail_index_create_tmp_file(struct mail_index *index, const char **path_r)
{
mode_t old_mask;
const char *path;
int fd;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
path = *path_r = t_strconcat(index->filepath, ".tmp", NULL);
old_mask = umask(0);
fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode);
umask(old_mask);
if (fd == -1)
return mail_index_file_set_syscall_error(index, path, "open()");
if (index->gid != (gid_t)-1 && fchown(fd, (uid_t)-1, index->gid) < 0) {
mail_index_file_set_syscall_error(index, path, "fchown()");
return -1;
}
return fd;
}
static int mail_index_create(struct mail_index *index,
struct mail_index_header *hdr)
{
const char *path;
uint32_t seq;
uoff_t offset;
int ret;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->lock_type == F_UNLCK);
/* log file lock protects index creation */
if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
return -1;
ret = mail_index_try_open(index, NULL);
if (ret != 0) {
mail_transaction_log_sync_unlock(index->log);
return ret < 0 ? -1 : 0;
}
/* mark the existing log file as synced */
hdr->log_file_seq = seq;
hdr->log_file_int_offset = offset;
hdr->log_file_ext_offset = offset;
/* create it fully in index.tmp first */
index->fd = mail_index_create_tmp_file(index, &path);
if (index->fd == -1)
ret = -1;
else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
mail_index_file_set_syscall_error(index, path, "write_full()");
ret = -1;
} else {
index->lock_type = F_WRLCK;
ret = mail_index_map(index, FALSE);
index->lock_type = F_UNLCK;
}
if (ret == 0) {
/* it's corrupted even while we just created it,
should never happen unless someone pokes the file directly */
mail_index_set_error(index,
"Newly created index file is corrupted: %s", path);
ret = -1;
}
if (ret < 0) {
if (unlink(path) < 0 && errno != ENOENT) {
mail_index_file_set_syscall_error(index, path,
"unlink()");
}
} else {
/* make it visible to others */
if (rename(path, index->filepath) < 0) {
mail_index_set_error(index, "rename(%s, %s) failed: %m",
path, index->filepath);
ret = -1;
}
}
mail_transaction_log_sync_unlock(index->log);
return ret;
}
static void mail_index_header_init(struct mail_index_header *hdr)
{
time_t now = time(NULL);
i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
memset(hdr, 0, sizeof(*hdr));
hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
hdr->base_header_size = sizeof(*hdr);
hdr->header_size = sizeof(*hdr);
hdr->record_size = sizeof(struct mail_index_record);
#ifndef WORDS_BIGENDIAN
hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
hdr->indexid = now;
hdr->next_uid = 1;
}
static void mail_index_create_in_memory(struct mail_index *index,
const struct mail_index_header *hdr)
{
struct mail_index_header tmp_hdr;
struct mail_index_map tmp_map;
if (hdr == NULL) {
mail_index_header_init(&tmp_hdr);
hdr = &tmp_hdr;
}
memset(&tmp_map, 0, sizeof(tmp_map));
tmp_map.hdr = *hdr;
tmp_map.hdr_base = hdr;
/* a bit kludgy way to do this, but it initializes everything
nicely and correctly */
index->map = mail_index_map_clone(&tmp_map, hdr->record_size);
index->hdr = &index->map->hdr;
}
/* returns -1 = error, 0 = won't create, 1 = ok */
static int mail_index_open_files(struct mail_index *index,
enum mail_index_open_flags flags)
{
struct mail_index_header hdr;
unsigned int lock_id = 0;
int ret;
bool created = FALSE;
ret = mail_index_try_open(index, &lock_id);
if (ret > 0)
hdr = *index->hdr;
else if (ret == 0) {
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0 &&
!MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
mail_index_header_init(&hdr);
index->hdr = &hdr;
} else if (ret < 0)
return -1;
index->indexid = hdr.indexid;
index->log = mail_transaction_log_open_or_create(index);
if (index->log == NULL) {
if (ret == 0)
index->hdr = NULL;
return -1;
}
if (index->fd == -1) {
mail_index_header_init(&hdr);
index->hdr = &hdr;
/* index->indexid may be updated by transaction log opening,
in case someone else had already created a new log file */
hdr.indexid = index->indexid;
if (lock_id != 0) {
mail_index_unlock(index, lock_id);
lock_id = 0;
}
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (mail_index_create(index, &hdr) < 0) {
/* fallback to in-memory index */
mail_index_move_to_memory(index);
mail_index_create_in_memory(index, &hdr);
}
} else {
mail_index_create_in_memory(index, &hdr);
}
created = TRUE;
}
i_assert(index->hdr != &hdr);
if (lock_id == 0) {
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0)
return -1;
}
index->cache = created ? mail_cache_create(index) :
mail_cache_open_or_create(index);
mail_index_unlock(index, lock_id);
return 1;
}
int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags,
enum file_lock_method lock_method)
{
int i = 0, ret;
if (index->opened) {
if (index->hdr != NULL &&
(index->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
/* corrupted, reopen files */
mail_index_close(index);
} else {
return 1;
}
}
index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
i_strdup("(in-memory index)") :
i_strconcat(index->dir, "/", index->prefix, NULL);
for (;;) {
index->shared_lock_count = 0;
index->excl_lock_count = 0;
index->lock_type = F_UNLCK;
index->lock_id = 2;
index->readonly = FALSE;
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
index->log_locked = FALSE;
index->mmap_disable =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
index->mmap_no_write =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0;
index->use_excl_dotlocks =
(flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
index->fsync_disable =
(flags & MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE) != 0;
index->lock_method = lock_method;
/* don't even bother to handle dotlocking without mmap being
disabled. that combination simply doesn't make any sense */
if (lock_method == FILE_LOCK_METHOD_DOTLOCK &&
!index->mmap_disable) {
i_fatal("lock_method=dotlock and mmap_disable=no "
"combination isn't supported. "
"You don't _really_ want it anyway.");
}
ret = mail_index_open_files(index, flags);
if (ret <= 0)
break;
index->opened = TRUE;
if (index->fsck) {
index->fsck = FALSE;
ret = mail_index_fsck(index);
if (ret == 0) {
/* completely broken, reopen */
if (i++ < 3)
continue;
/* too many tries */
ret = -1;
}
}
break;
}
if (ret <= 0)
mail_index_close(index);
return ret;
}
void mail_index_close(struct mail_index *index)
{
if (index->log != NULL)
mail_transaction_log_close(&index->log);
if (index->map != NULL)
mail_index_unmap(index, &index->map);
if (index->cache != NULL)
mail_cache_free(&index->cache);
if (index->file_lock != NULL)
file_lock_free(&index->file_lock);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
}
i_free_and_null(index->copy_lock_path);
i_free_and_null(index->filepath);
index->indexid = 0;
index->opened = FALSE;
}
int mail_index_reopen(struct mail_index *index, int fd)
{
struct mail_index_map *old_map;
struct file_lock *old_file_lock;
unsigned int old_shared_locks, old_lock_id, lock_id = 0;
int ret, old_fd, old_lock_type;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->excl_lock_count == 0);
old_map = index->map;
old_fd = index->fd;
old_map->refcount++;
/* new file, new locks. the old fd can keep its locks, they don't
matter anymore as no-one's going to modify the file. */
old_lock_type = index->lock_type;
old_lock_id = index->lock_id;
old_shared_locks = index->shared_lock_count;
old_file_lock = index->file_lock;
if (index->lock_type == F_RDLCK)
index->lock_type = F_UNLCK;
index->lock_id += 2;
index->shared_lock_count = 0;
index->file_lock = NULL;
if (fd != -1) {
index->fd = fd;
ret = 0;
} else {
ret = mail_index_try_open_only(index);
if (ret > 0)
ret = mail_index_lock_shared(index, FALSE, &lock_id);
else if (ret == 0) {
/* index file is lost */
ret = -1;
}
}
if (ret == 0) {
/* read the new mapping. note that with mmap_disable we want
to keep the old mapping in index->map so we can update it
by reading transaction log. */
if (mail_index_map(index, TRUE) <= 0)
ret = -1;
}
if (lock_id != 0)
mail_index_unlock(index, lock_id);
if (ret == 0) {
mail_index_unmap(index, &old_map);
if (old_file_lock != NULL)
file_lock_free(&old_file_lock);
if (close(old_fd) < 0)
mail_index_set_syscall_error(index, "close()");
} else {
if (index->map != NULL)
mail_index_unmap(index, &index->map);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
}
index->map = old_map;
index->hdr = &index->map->hdr;
index->fd = old_fd;
index->file_lock = old_file_lock;
index->lock_type = old_lock_type;
index->lock_id = old_lock_id;
index->shared_lock_count = old_shared_locks;
}
return ret;
}
int mail_index_reopen_if_needed(struct mail_index *index)
{
struct stat st1, st2;
i_assert(index->copy_lock_path == NULL);
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (fstat(index->fd, &st1) < 0) {
if (errno == ESTALE) {
/* deleted, reopen */
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
}
return mail_index_set_syscall_error(index, "fstat()");
}
if (nfs_safe_stat(index->filepath, &st2) < 0) {
mail_index_set_syscall_error(index, "stat()");
if (errno != ENOENT)
return -1;
/* lost it? recreate later */
mail_index_mark_corrupted(index);
return -1;
}
if (st1.st_ino != st2.st_ino ||
!CMP_DEV_T(st1.st_dev, st2.st_dev)) {
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
} else {
return 0;
}
}
int mail_index_refresh(struct mail_index *index)
{
unsigned int lock_id;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (index->excl_lock_count > 0) {
/* we have index exclusively locked, nothing could
have changed. */
return 0;
}
if (!index->mmap_disable) {
/* reopening is all we need */
return mail_index_reopen_if_needed(index);
}
i_assert(!index->mapping);
/* mail_index_map() simply reads latest changes from transaction log,
which makes us fully refreshed. */
if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
return -1;
ret = mail_index_map(index, FALSE);
mail_index_unlock(index, lock_id);
return ret <= 0 ? -1 : 0;
}
struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
return index->cache;
}
int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
va_list va;
i_free(index->error);
if (fmt == NULL)
index->error = NULL;
else {
va_start(va, fmt);
index->error = i_strdup_vprintf(fmt, va);
va_end(va);
i_error("%s", index->error);
}
return -1;
}
void mail_index_set_inconsistent(struct mail_index *index)
{
index->indexid = 0;
}
int mail_index_move_to_memory(struct mail_index *index)
{
struct mail_index_map *map;
int ret = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
/* set the index as being into memory */
i_free_and_null(index->dir);
if (index->map == NULL) {
/* mbox file was never even opened. just mark it as being in
memory and let the caller re-open the index. */
i_assert(index->fd == -1);
return -1;
}
/* move index map to memory */
map = mail_index_map_clone(index->map, index->map->hdr.record_size);
mail_index_unmap(index, &index->map);
index->map = map;
index->hdr = &map->hdr;
if (index->log != NULL) {
/* move transaction log to memory */
if (mail_transaction_log_move_to_memory(index->log) < 0)
ret = -1;
}
if (index->file_lock != NULL)
file_lock_free(&index->file_lock);
/* close the index file. */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
return ret;
}
void mail_index_mark_corrupted(struct mail_index *index)
{
struct mail_index_header hdr;
mail_index_set_inconsistent(index);
if (index->readonly || index->map == NULL)
return;
hdr = *index->hdr;
hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
if (mail_index_write_base_header(index, &hdr) == 0) {
if (!MAIL_INDEX_IS_IN_MEMORY(index) && fsync(index->fd) < 0)
mail_index_set_syscall_error(index, "fsync()");
}
}
int mail_index_set_syscall_error(struct mail_index *index,
const char *function)
{
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with index file %s: %m",
function, index->filepath);
}
int mail_index_file_set_syscall_error(struct mail_index *index,
const char *filepath,
const char *function)
{
i_assert(filepath != NULL);
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with file %s: %m",
function, filepath);
}
enum mail_index_error mail_index_get_last_error(struct mail_index *index)
{
if (index->nodiskspace)
return MAIL_INDEX_ERROR_DISKSPACE;
if (index->error != NULL)
return MAIL_INDEX_ERROR_INTERNAL;
return MAIL_INDEX_ERROR_NONE;
}
const char *mail_index_get_error_message(struct mail_index *index)
{
return index->error;
}
void mail_index_reset_error(struct mail_index *index)
{
if (index->error != NULL) {
i_free(index->error);
index->error = NULL;
}
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
}
uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
unsigned char buf[4];
i_assert(offset < 0x40000000);
i_assert((offset & 3) == 0);
offset >>= 2;
buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
buf[3] = 0x80 | (offset & 0x0000007f);
return *((uint32_t *) buf);
}
uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
const unsigned char *buf = (const unsigned char *) &offset;
if ((offset & 0x80808080) != 0x80808080)
return 0;
return (((uint32_t)buf[3] & 0x7f) << 2) |
(((uint32_t)buf[2] & 0x7f) << 9) |
(((uint32_t)buf[1] & 0x7f) << 16) |
(((uint32_t)buf[0] & 0x7f) << 23);
}