mail-index.c revision da985034a708db2f61394b30d117050ae6829ee5
183bea41fa640dc8117f3eb45ff935cd81377a84Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "lib.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "array.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "buffer.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "hash.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mmap-util.h"
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen#include "nfs-workarounds.h"
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen#include "read-full.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "write-full.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-index-private.h"
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen#include "mail-index-sync-private.h"
fb08a91e3f2949ecefb647fa38206ca9aad5307fTimo Sirainen#include "mail-transaction-log.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-cache.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include <stdio.h>
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen#include <stddef.h>
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include <time.h>
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include <sys/stat.h>
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d0720f3037064af4b92eccfc20a8814adcacf827Timo Sirainenunsigned int mail_index_module_id = 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainenstatic void mail_index_create_in_memory(struct mail_index *index,
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen const struct mail_index_header *hdr);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen struct mail_index *index;
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen index = i_new(struct mail_index, 1);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen index->dir = i_strdup(dir);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen index->prefix = i_strdup(prefix);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen index->fd = -1;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen index->extension_pool = pool_alloconly_create("extension", 512);
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen ARRAY_CREATE(&index->extensions, index->extension_pool,
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen struct mail_index_registered_ext, 5);
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen ARRAY_CREATE(&index->sync_lost_handlers, default_pool,
c0e5c6a86e1de5d4f5591d39b4aa921a23c807d7Timo Sirainen mail_index_sync_lost_handler_t *, 4);
c0e5c6a86e1de5d4f5591d39b4aa921a23c807d7Timo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen ARRAY_CREATE(&index->mail_index_module_contexts, default_pool,
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen void *, I_MIN(5, mail_index_module_id));
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen index->mode = 0600;
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen index->gid = (gid_t)-1;
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen index->keywords_ext_id =
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen mail_index_ext_register(index, "keywords", 128, 2, 1);
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen index->keywords_pool = pool_alloconly_create("keywords", 512);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen ARRAY_CREATE(&index->keywords, default_pool, const char *, 16);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen index->keywords_hash =
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen hash_create(default_pool, index->keywords_pool, 0,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen strcase_hash, (hash_cmp_callback_t *)strcasecmp);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen return index;
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen}
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainenvoid mail_index_free(struct mail_index **_index)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen{
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen struct mail_index *index = *_index;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen *_index = NULL;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen mail_index_close(index);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen hash_destroy(index->keywords_hash);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen pool_unref(index->extension_pool);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen pool_unref(index->keywords_pool);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen array_free(&index->sync_lost_handlers);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen array_free(&index->keywords);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen i_free(index->error);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen i_free(index->dir);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen i_free(index->prefix);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen i_free(index);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen}
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainenvoid mail_index_set_permissions(struct mail_index *index,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen mode_t mode, gid_t gid)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen{
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen index->mode = mode & 0666;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen index->gid = gid;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen}
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen uint32_t default_hdr_size,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen uint16_t default_record_size,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen uint16_t default_record_align)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen{
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen const struct mail_index_registered_ext *extensions;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen struct mail_index_registered_ext rext;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen unsigned int i, ext_count;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen extensions = array_get(&index->extensions, &ext_count);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen /* see if it's already there */
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen for (i = 0; i < ext_count; i++) {
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen if (strcmp(extensions[i].name, name) == 0)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen return i;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen }
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen memset(&rext, 0, sizeof(rext));
c0e5c6a86e1de5d4f5591d39b4aa921a23c807d7Timo Sirainen rext.name = p_strdup(index->extension_pool, name);
93a7d1ee4b518b5c85f9721dc6539e4dab6aae00Timo Sirainen rext.index_idx = ext_count;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen rext.hdr_size = default_hdr_size;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen rext.record_size = default_record_size;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen rext.record_align = default_record_align;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
c05d0937e228c2817fa2295fc53e8cb81ae5cb8aTimo Sirainen array_append(&index->extensions, &rext, 1);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen return ext_count;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen}
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainenvoid mail_index_register_expunge_handler(struct mail_index *index,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen uint32_t ext_id,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen mail_index_expunge_handler_t *cb,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen void *context, bool call_always)
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen{
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen struct mail_index_registered_ext *rext;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen i_assert(rext->expunge_handler == NULL);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rext->expunge_handler = cb;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rext->expunge_context = context;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen rext->expunge_handler_call_always = call_always;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenvoid mail_index_unregister_expunge_handler(struct mail_index *index,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t ext_id)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen struct mail_index_registered_ext *rext;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(rext->expunge_handler != NULL);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rext->expunge_handler = NULL;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenvoid mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_sync_handler_t *cb,
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen enum mail_index_sync_handler_type type)
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_index_registered_ext *rext;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
f6edc54aa72596af8da681c07223108c322712d5Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(rext->sync_handler.callback == NULL);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rext->sync_handler.callback = cb;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rext->sync_handler.type = type;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainenvoid mail_index_unregister_sync_handler(struct mail_index *index,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t ext_id)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen struct mail_index_registered_ext *rext;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(rext->sync_handler.callback != NULL);
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen rext->sync_handler.callback = NULL;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rext->sync_handler.type = 0;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen}
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainenvoid mail_index_register_sync_lost_handler(struct mail_index *index,
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen mail_index_sync_lost_handler_t *cb)
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen{
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen array_append(&index->sync_lost_handlers, &cb, 1);
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen}
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainenvoid mail_index_unregister_sync_lost_handler(struct mail_index *index,
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen mail_index_sync_lost_handler_t *cb)
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen{
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen mail_index_sync_lost_handler_t *const *handlers;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen unsigned int i, count;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen handlers = array_get(&index->sync_lost_handlers, &count);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen for (i = 0; i < count; i++) {
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if (handlers[i] == cb) {
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen array_delete(&index->sync_lost_handlers, i, 1);
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen break;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen }
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen }
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen}
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen unsigned int initial_count)
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen{
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen#define EXTENSION_NAME_APPROX_LEN 20
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen size_t size;
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen
9cd232cda7563ad81c01776e5ebc5ed2b3cef898Timo Sirainen if (map->extension_pool == NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen size = (sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2 +
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen initial_count * (EXTENSION_NAME_APPROX_LEN +
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen sizeof(struct mail_index_ext) +
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen sizeof(uint32_t));
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map->extension_pool =
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen pool_alloconly_create("extensions",
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen nearest_power(size));
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen } else {
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen p_clear(map->extension_pool);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen ARRAY_CREATE(&map->extensions, map->extension_pool,
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen struct mail_index_ext, initial_count);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen ARRAY_CREATE(&map->ext_id_map, map->extension_pool,
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen uint32_t, initial_count);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen}
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenuint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen{
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen const struct mail_index_ext *extensions;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen unsigned int i, size;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen if (!array_is_created(&map->extensions))
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen return (uint32_t)-1;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen extensions = array_get(&map->extensions, &size);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen for (i = 0; i < size; i++) {
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen if (strcmp(extensions[i].name, name) == 0)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen return i;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen }
2f6a8ef44beaee4ef74adfcda455ce426c6e7c45Timo Sirainen return (uint32_t)-1;
2f6a8ef44beaee4ef74adfcda455ce426c6e7c45Timo Sirainen}
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenuint32_t
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenmail_index_map_register_ext(struct mail_index *index,
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen struct mail_index_map *map, const char *name,
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen uint32_t hdr_offset, uint32_t hdr_size,
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen uint32_t record_offset, uint32_t record_size,
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen uint32_t record_align, uint32_t reset_id)
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen{
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen struct mail_index_ext *ext;
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen uint32_t idx, empty_idx = (uint32_t)-1;
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen if (!array_is_created(&map->extensions)) {
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen mail_index_map_init_extbufs(map, 5);
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen idx = 0;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen } else {
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen idx = array_count(&map->extensions);
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen }
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen ext = array_append_space(&map->extensions);
699b27536eeb446e05ed9c9c1ab35cc6174ac0aaTimo Sirainen ext->name = p_strdup(map->extension_pool, name);
dadd250347d90d90bd65f529d45b50f79d6a0674Timo Sirainen ext->hdr_offset = hdr_offset;
3a79fdaf3253dae045dfa14d2a88b94086327da4Timo Sirainen ext->hdr_size = hdr_size;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen ext->record_offset = record_offset;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen ext->record_size = record_size;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen ext->record_align = record_align;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen ext->reset_id = reset_id;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen ext->index_idx = mail_index_ext_register(index, name, hdr_size,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen record_size, record_align);
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen /* Update index ext_id -> map ext_id mapping. Fill non-used
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen ext_ids with (uint32_t)-1 */
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen while (array_count(&map->ext_id_map) < ext->index_idx)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen array_append(&map->ext_id_map, &empty_idx, 1);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen return idx;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen}
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainenstatic bool size_check(size_t *size_left, size_t size)
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen{
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen if (size > *size_left)
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen return FALSE;
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen *size_left -= size;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen return TRUE;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen}
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainenstatic size_t get_align(size_t name_len)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen size_t size = sizeof(struct mail_index_ext_header) + name_len;
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int mail_index_parse_extensions(struct mail_index *index,
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen struct mail_index_map *map)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen const struct mail_index_ext_header *ext_hdr;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int i, old_count;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const char *name;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t ext_id, offset, name_offset;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen size_t size_left;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* extension headers always start from 64bit offsets, so if base header
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen doesn't happen to be 64bit aligned we'll skip some bytes */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* nothing to do, skip allocatations and all */
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen return 1;
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen }
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen old_count = array_count(&index->extensions);
2806f15ceb68023baf65a9daad9dfdf54c622708Timo Sirainen mail_index_map_init_extbufs(map, old_count + 5);
2806f15ceb68023baf65a9daad9dfdf54c622708Timo Sirainen
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen ext_id = (uint32_t)-1;
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen for (i = 0; i < old_count; i++)
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen array_append(&map->ext_id_map, &ext_id, 1);
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen while (offset < map->hdr.header_size) {
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen /* Extension header contains:
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen - struct mail_index_ext_header
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen - name (not 0-terminated)
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen - 64bit alignment padding
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen - extension header contents
2f6a8ef44beaee4ef74adfcda455ce426c6e7c45Timo Sirainen - 64bit alignment padding
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen */
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen size_left = map->hdr.header_size - offset;
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen if (!size_check(&size_left, sizeof(*ext_hdr)) ||
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen !size_check(&size_left, ext_hdr->name_size) ||
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen !size_check(&size_left, get_align(ext_hdr->name_size)) ||
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen !size_check(&size_left, ext_hdr->hdr_size)) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen "Header extension goes outside header",
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen index->filepath);
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen offset += sizeof(*ext_hdr);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen name_offset = offset;
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen offset += ext_hdr->name_size + get_align(ext_hdr->name_size);
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen t_push();
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
f7f13e206c9839f6e868088034b0b59d1d9be13aTimo Sirainen ext_hdr->name_size);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen "Duplicate header extension %s",
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen index->filepath, name);
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen t_pop();
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen return -1;
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen }
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen (map->hdr.record_size % ext_hdr->record_align) != 0) {
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen "Record field %s alignmentation %u not used",
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen index->filepath, name, ext_hdr->record_align);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen t_pop();
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen mail_index_map_register_ext(index, map, name,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen offset, ext_hdr->hdr_size,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ext_hdr->record_offset,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ext_hdr->record_size,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ext_hdr->record_align,
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen ext_hdr->reset_id);
605cce7e45c80fa8a07a10155c353ca541a3041fTimo Sirainen t_pop();
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen
acef354e742a39416b0697e1554f5d49b0369850Timo Sirainen offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 1;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen}
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenbool mail_index_keyword_lookup(struct mail_index *index,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen const char *keyword, bool autocreate,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen unsigned int *idx_r)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen{
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen char *keyword_dup;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen void *value;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen /* keywords_hash keeps a name => index mapping of keywords.
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen Keywords are never removed from it, so the index values are valid
3005627bf2ed223194c2d08a8c1630769d048f69Timo Sirainen for the lifetime of the mail_index. */
3005627bf2ed223194c2d08a8c1630769d048f69Timo Sirainen if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen *idx_r = POINTER_CAST_TO(value, unsigned int);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen return TRUE;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if (!autocreate) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen *idx_r = (unsigned int)-1;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen return FALSE;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen *idx_r = array_count(&index->keywords);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r));
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen array_append(&index->keywords, &keyword, 1);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen return TRUE;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen}
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenint mail_index_map_parse_keywords(struct mail_index *index,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen struct mail_index_map *map)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_index_ext *ext;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_index_keyword_header *kw_hdr;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_index_keyword_header_rec *kw_rec;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const char *name;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int i, name_area_end_offset, old_count;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen uint32_t ext_id;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen ext_id = mail_index_map_lookup_ext(map, "keywords");
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen if (ext_id == (uint32_t)-1) {
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen if (array_is_created(&map->keyword_idx_map))
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen array_clear(&map->keyword_idx_map);
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen return 0;
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen }
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen ext = array_idx(&map->extensions, ext_id);
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen /* Extension header contains:
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen - struct mail_index_keyword_header
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen - struct mail_index_keyword_header_rec * keywords_count
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen - const char names[] * keywords_count
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen */
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen i_assert(ext->hdr_offset < map->hdr.header_size);
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen kw_rec = (const void *)(kw_hdr + 1);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen name = (const char *)(kw_rec + kw_hdr->keywords_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen array_count(&map->keyword_idx_map);
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen /* Keywords can only be added into same mapping. Removing requires a
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen new mapping (recreating the index file) */
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen if (kw_hdr->keywords_count == old_count) {
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen /* nothing changed */
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen return 0;
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen /* make sure the header is valid */
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen if (kw_hdr->keywords_count < old_count) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen "Keywords removed unexpectedly",
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen index->filepath);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen }
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen "keywords_count larger than header size",
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen index->filepath);
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen return -1;
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen }
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen for (i = 0; i < kw_hdr->keywords_count; i++) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (kw_rec[i].name_offset > name_area_end_offset) {
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen "name_offset points outside allocated header",
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen index->filepath);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (name[name_area_end_offset-1] != '\0') {
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen "Keyword header doesn't end with NUL",
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen index->filepath);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen }
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* create file -> index mapping */
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen if (!array_is_created(&map->keyword_idx_map)) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ARRAY_CREATE(&map->keyword_idx_map, default_pool,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int, kw_hdr->keywords_count);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen#ifdef DEBUG
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen /* Check that existing headers are still the same. It's behind DEBUG
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen since it's pretty useless waste of CPU normally. */
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen const unsigned int *old_idx;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen unsigned int idx;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen old_idx = array_idx(&map->keyword_idx_map, i);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) ||
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen idx != *old_idx) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen "Keywords changed unexpectedly",
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen index->filepath);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen return -1;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen }
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen }
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen#endif
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen /* Register the newly seen keywords */
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen i = array_count(&map->keyword_idx_map);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen for (; i < kw_hdr->keywords_count; i++) {
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen unsigned int idx;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen array_append(&map->keyword_idx_map, &idx, 1);
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen }
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen return 0;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen}
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainenconst ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index)
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen{
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen /* Make sure all the keywords are in index->keywords. It's quick to do
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if nothing has changed. */
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen (void)mail_index_map_parse_keywords(index, index->map);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return &index->keywords;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainenstatic int mail_index_check_header(struct mail_index *index,
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen struct mail_index_map *map)
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen{
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen const struct mail_index_header *hdr = &map->hdr;
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen enum mail_index_header_compat_flags compat_flags = 0;
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen#ifndef WORDS_BIGENDIAN
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen#endif
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen /* major version change - handle silently(?) */
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen return -1;
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen }
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen if (hdr->compat_flags != compat_flags) {
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen /* architecture change - handle silently(?) */
7bd72e4deca3cbf757dd1ea298486d9f3bc24226Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* we've already complained about it */
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen return -1;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen }
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen /* following some extra checks that only take a bit of CPU */
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen "uid_validity = 0, next_uid = %u",
904324b95569604138d24f2dc951f9fb3cc246dcTimo Sirainen index->filepath, hdr->next_uid);
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen return -1;
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen }
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen if (hdr->record_size < sizeof(struct mail_index_record)) {
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen "record_size too small: %u < %"PRIuSIZE_T,
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen index->filepath, hdr->record_size,
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen sizeof(struct mail_index_record));
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen return -1;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen }
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (hdr->next_uid == 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (hdr->recent_messages_count > hdr->messages_count ||
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen hdr->seen_messages_count > hdr->messages_count ||
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen hdr->deleted_messages_count > hdr->messages_count)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (hdr->first_recent_uid_lowwater > hdr->next_uid ||
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen hdr->first_unseen_uid_lowwater > hdr->next_uid ||
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen hdr->first_deleted_uid_lowwater > hdr->next_uid)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return mail_index_parse_extensions(index, map);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainenstatic void mail_index_map_clear(struct mail_index *index,
1c7b0cbdb08cccbd25c19ae0fb69abe8ed9ee9b4Timo Sirainen struct mail_index_map *map)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen if (map->buffer != NULL) {
870bcf0d0c07f7d915f1f571f38968426ba575a1Timo Sirainen i_assert(map->mmap_base == NULL);
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen buffer_free(map->buffer);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map->buffer = NULL;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen } else if (map->mmap_base != NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(map->buffer == NULL);
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (munmap(map->mmap_base, map->mmap_size) < 0)
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen mail_index_set_syscall_error(index, "munmap()");
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen map->mmap_base = NULL;
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen }
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen if (map->refcount > 0) {
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map->mmap_size = 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map->mmap_used_size = 0;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen map->records = NULL;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen map->records_count = 0;
2b2e5f7a24c24d971351877ad4c5150662856bfbTimo Sirainen }
601b455f4d5e780044b9e4fac5f687c1b07ae145Timo Sirainen}
2806f15ceb68023baf65a9daad9dfdf54c622708Timo Sirainen
3b959c98e05e780de2a063a4a9d8d393dc61ed58Timo Sirainenvoid mail_index_unmap(struct mail_index *index, struct mail_index_map **_map)
3b959c98e05e780de2a063a4a9d8d393dc61ed58Timo Sirainen{
3b959c98e05e780de2a063a4a9d8d393dc61ed58Timo Sirainen struct mail_index_map *map = *_map;
57dc3cb5d5e315272353abf55f702eefc084db26Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen *_map = NULL;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (--map->refcount > 0)
return;
i_assert(map->refcount == 0);
mail_index_map_clear(index, map);
if (map->extension_pool != NULL)
pool_unref(map->extension_pool);
if (array_is_created(&map->keyword_idx_map))
array_free(&map->keyword_idx_map);
buffer_free(map->hdr_copy_buf);
i_free(map);
}
static void mail_index_map_copy_hdr(struct mail_index_map *map,
const struct mail_index_header *hdr)
{
if (hdr->base_header_size < sizeof(map->hdr)) {
/* header smaller than ours, make a copy so our newer headers
won't have garbage in them */
memset(&map->hdr, 0, sizeof(map->hdr));
memcpy(&map->hdr, hdr, hdr->base_header_size);
} else {
map->hdr = *hdr;
}
}
static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
{
const struct mail_index_header *hdr;
unsigned int records_count;
i_assert(!map->write_to_disk);
if (map->buffer != NULL) {
/* we had temporarily used a buffer, eg. for updating index */
buffer_free(map->buffer);
map->buffer = NULL;
}
map->mmap_base = index->readonly ?
mmap_ro_file(index->fd, &map->mmap_size) :
mmap_rw_file(index->fd, &map->mmap_size);
if (map->mmap_base == MAP_FAILED) {
map->mmap_base = NULL;
mail_index_set_syscall_error(index, "mmap()");
return -1;
}
hdr = map->mmap_base;
if (map->mmap_size >
offsetof(struct mail_index_header, major_version) &&
hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
/* major version change - handle silently */
return 0;
}
if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
mail_index_set_error(index, "Corrupted index file %s: "
"File too small (%"PRIuSIZE_T")",
index->filepath, map->mmap_size);
return 0;
}
map->mmap_used_size = hdr->header_size +
hdr->messages_count * hdr->record_size;
if (map->mmap_used_size > map->mmap_size) {
records_count = (map->mmap_size - hdr->header_size) /
hdr->record_size;
mail_index_set_error(index, "Corrupted index file %s: "
"messages_count too large (%u > %u)",
index->filepath, hdr->messages_count,
records_count);
return 0;
}
mail_index_map_copy_hdr(map, hdr);
map->hdr_base = map->mmap_base;
map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
map->records_count = map->hdr.messages_count;
return 1;
}
static int mail_index_read_header(struct mail_index *index,
void *buf, size_t buf_size, size_t *pos_r)
{
size_t pos;
int ret;
memset(buf, 0, sizeof(struct mail_index_header));
/* try to read the whole header, but it's not necessarily an error to
read less since the older versions of the index format could be
smaller. Request reading up to buf_size, but accept if we only got
the header. */
pos = 0;
do {
ret = pread(index->fd, PTR_OFFSET(buf, pos),
buf_size - pos, pos);
if (ret > 0)
pos += ret;
} while (ret > 0 && pos < sizeof(struct mail_index_header));
*pos_r = pos;
return ret;
}
static int
mail_index_read_map(struct mail_index *index, struct mail_index_map *map,
bool *retry_r, bool try_retry)
{
const struct mail_index_header *hdr;
unsigned char buf[512];
void *data = NULL;
ssize_t ret;
size_t pos, records_size;
i_assert(map->mmap_base == NULL);
*retry_r = FALSE;
ret = mail_index_read_header(index, buf, sizeof(buf), &pos);
hdr = (const struct mail_index_header *)buf;
if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
/* major version change - handle silently */
return 0;
}
if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
(ret > 0 || pos >= hdr->base_header_size)) {
if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
hdr->header_size < hdr->base_header_size) {
mail_index_set_error(index, "Corrupted index file %s: "
"Corrupted header sizes (base %u, full %u)",
index->filepath, hdr->base_header_size,
hdr->header_size);
return 0;
}
if (pos > hdr->header_size)
pos = hdr->header_size;
/* place the base header into memory. */
buffer_reset(map->hdr_copy_buf);
buffer_append(map->hdr_copy_buf, buf, pos);
if (pos != hdr->header_size) {
/* @UNSAFE: read the rest of the header into memory */
data = buffer_append_space_unsafe(map->hdr_copy_buf,
hdr->header_size -
pos);
ret = pread_full(index->fd, data,
hdr->header_size - pos, pos);
}
}
if (ret > 0) {
/* header read, read the records now. */
records_size = hdr->messages_count * hdr->record_size;
if (map->buffer == NULL) {
map->buffer = buffer_create_dynamic(default_pool,
records_size);
}
/* @UNSAFE */
buffer_set_used_size(map->buffer, 0);
data = buffer_append_space_unsafe(map->buffer, records_size);
ret = pread_full(index->fd, data, records_size,
hdr->header_size);
}
if (ret < 0) {
if (errno == ESTALE && try_retry) {
/* a new index file was renamed over this one. */
*retry_r = TRUE;
return 0;
}
mail_index_set_syscall_error(index, "pread_full()");
return -1;
}
if (ret == 0) {
mail_index_set_error(index,
"Corrupted index file %s: File too small",
index->filepath);
return 0;
}
map->records = data;
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
map->hdr_base = map->hdr_copy_buf->data;
index->sync_log_file_seq = hdr->log_file_seq;
index->sync_log_file_offset = hdr->log_file_int_offset;
return 1;
}
bool mail_index_is_ext_synced(struct mail_transaction_log_view *log_view,
struct mail_index_map *map)
{
uint32_t prev_seq;
uoff_t prev_offset;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
return prev_seq < map->hdr.log_file_seq ||
(prev_seq == map->hdr.log_file_seq &&
prev_offset < map->hdr.log_file_ext_offset);
}
static int mail_index_sync_from_transactions(struct mail_index *index,
struct mail_index_map **map,
bool sync_to_index)
{
const struct mail_index_header *map_hdr = &(*map)->hdr;
struct mail_index_view *view;
struct mail_transaction_log_view *log_view;
struct mail_index_sync_map_ctx sync_map_ctx;
struct mail_index_header hdr;
const struct mail_transaction_header *thdr;
const void *tdata;
uint32_t prev_seq, max_seq;
uoff_t prev_offset, max_offset;
size_t pos;
int ret;
bool skipped, check_ext_offsets, broken;
if (sync_to_index) {
/* read the real log position where we are supposed to be
synced */
ret = mail_index_read_header(index, &hdr, sizeof(hdr), &pos);
if (ret < 0 && errno != ESTALE) {
mail_index_set_syscall_error(index, "pread()");
return -1;
}
if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
return 0;
if (map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset == hdr.log_file_int_offset) {
/* nothing to do */
return 1;
}
if (map_hdr->log_file_seq > hdr.log_file_seq ||
(map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset > hdr.log_file_int_offset)) {
/* we went too far, have to re-read the file */
return 0;
}
if (map_hdr->log_file_ext_offset !=
map_hdr->log_file_int_offset ||
hdr.log_file_ext_offset != hdr.log_file_int_offset) {
/* too much trouble to get this right. */
return 0;
}
max_seq = hdr.log_file_seq;
max_offset = hdr.log_file_int_offset;
} else {
/* sync everything there is */
max_seq = (uint32_t)-1;
max_offset = (uoff_t)-1;
}
log_view = mail_transaction_log_view_open(index->log);
if (mail_transaction_log_view_set(log_view,
map_hdr->log_file_seq,
map_hdr->log_file_int_offset,
max_seq, max_offset,
MAIL_TRANSACTION_TYPE_MASK) <= 0) {
/* can't use it. sync by re-reading index. */
mail_transaction_log_view_close(&log_view);
return 0;
}
index->map = *map;
view = mail_index_view_open(index);
mail_index_sync_map_init(&sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_HEAD);
check_ext_offsets = TRUE; broken = FALSE;
while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,
&skipped)) > 0) {
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0 &&
check_ext_offsets) {
if (mail_index_is_ext_synced(log_view, index->map))
continue;
check_ext_offsets = FALSE;
}
if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {
ret = 0;
broken = TRUE;
break;
}
}
if (ret == 0 && !broken)
ret = 1;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
i_assert(prev_seq <= max_seq &&
(prev_seq != max_seq || prev_offset <= max_offset));
index->map->hdr.log_file_seq = prev_seq;
index->map->hdr.log_file_int_offset =
index->map->hdr.log_file_ext_offset = prev_offset;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(&view);
mail_transaction_log_view_close(&log_view);
*map = index->map;
index->map = NULL;
if (sync_to_index && ret > 0) {
/* make sure we did everything right. note that although the
message counts should be equal, the flag counters may not */
i_assert(hdr.messages_count == (*map)->hdr.messages_count);
i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq);
i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset);
i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset);
}
return ret;
}
static int mail_index_read_map_with_retry(struct mail_index *index,
struct mail_index_map **map,
bool sync_to_index)
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
int ret;
bool retry;
if (index->log_locked) {
/* we're most likely syncing the index and we really don't
want to read more than what was synced last time. */
sync_to_index = TRUE;
}
if ((*map)->hdr.indexid != 0 && index->log != NULL) {
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
index->sync_update = TRUE;
ret = mail_index_sync_from_transactions(index, map,
sync_to_index);
index->sync_update = FALSE;
if (ret != 0)
return ret;
/* transaction log lost/broken, fallback to re-reading it */
}
/* notify all "sync lost" handlers */
handlers = array_get(&index->sync_lost_handlers, &count);
for (i = 0; i < count; i++)
(*handlers[i])(index);
for (i = 0;; i++) {
ret = mail_index_read_map(index, *map, &retry,
i < MAIL_INDEX_ESTALE_RETRY_COUNT);
if (ret != 0 || !retry)
return ret;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
/* Too many ESTALE retries */
mail_index_set_syscall_error(index, "read_map()");
return -1;
}
static int mail_index_map_try_existing(struct mail_index_map *map)
{
const struct mail_index_header *hdr;
size_t used_size;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
return 0;
hdr = map->mmap_base;
/* always check corrupted-flag to avoid errors later */
if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)
return -1;
used_size = hdr->header_size + hdr->messages_count * hdr->record_size;
if (map->mmap_size >= used_size && map->hdr_base == hdr) {
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
return 1;
}
return 0;
}
int mail_index_map(struct mail_index *index, bool force)
{
struct mail_index_map *map;
int ret;
i_assert(!index->mapping);
i_assert(index->map == NULL || index->map->refcount > 0);
i_assert(index->lock_type != F_UNLCK);
if (MAIL_INDEX_IS_IN_MEMORY(index)) {
if (index->map == NULL)
mail_index_create_in_memory(index, NULL);
return 1;
}
index->mapping = TRUE;
if (!force && index->map != NULL) {
i_assert(index->hdr != NULL);
ret = mail_index_map_try_existing(index->map);
if (ret != 0) {
index->mapping = FALSE;
return ret;
}
if (index->lock_type == F_WRLCK) {
/* we're syncing, don't break the mapping */
index->mapping = FALSE;
return 1;
}
}
if (index->map != NULL && index->map->refcount > 1) {
/* this map is already used by some views and they may have
pointers into it. leave them and create a new mapping. */
if (!index->mmap_disable) {
map = NULL;
} else {
/* create a copy of the mapping instead so we don't
have to re-read it */
map = mail_index_map_clone(index->map,
index->map->hdr.record_size);
}
index->map->refcount--;
index->map = NULL;
} else {
map = index->map;
}
if (map == NULL) {
map = i_new(struct mail_index_map, 1);
map->refcount = 1;
map->hdr_copy_buf =
buffer_create_dynamic(default_pool, sizeof(map->hdr));
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
i_assert(!map->write_to_disk);
} else if (map->mmap_base != NULL) {
i_assert(map->buffer == NULL);
if (munmap(map->mmap_base, map->mmap_size) < 0)
mail_index_set_syscall_error(index, "munmap()");
map->mmap_base = NULL;
}
index->hdr = NULL;
index->map = NULL;
if (!index->mmap_disable)
ret = mail_index_mmap(index, map);
else
ret = mail_index_read_map_with_retry(index, &map, force);
i_assert(index->map == NULL);
if (ret > 0) {
ret = mail_index_check_header(index, map);
if (ret < 0)
ret = 0;
else if (ret == 0) {
index->fsck = TRUE;
ret = 1;
}
}
if (ret <= 0) {
mail_index_map_clear(index, map);
mail_index_unmap(index, &map);
index->mapping = FALSE;
return ret;
}
index->hdr = &map->hdr;
index->map = map;
i_assert(map->hdr.messages_count == map->records_count);
index->mapping = FALSE;
return 1;
}
int mail_index_get_latest_header(struct mail_index *index,
struct mail_index_header *hdr_r)
{
size_t pos;
unsigned int i;
int ret;
if (!index->mmap_disable) {
ret = mail_index_map(index, FALSE);
if (ret > 0)
*hdr_r = *index->hdr;
else
memset(hdr_r, 0, sizeof(*hdr_r));
return ret;
}
for (i = 0;; i++) {
ret = mail_index_read_header(index, hdr_r, sizeof(*hdr_r),
&pos);
if (ret <= 0 || errno != ESTALE ||
i == MAIL_INDEX_ESTALE_RETRY_COUNT)
break;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
if (ret < 0)
mail_index_set_syscall_error(index, "pread_full()");
return ret;
}
struct mail_index_map *
mail_index_map_clone(struct mail_index_map *map, uint32_t new_record_size)
{
struct mail_index_map *mem_map;
struct mail_index_header *hdr;
struct mail_index_ext *extensions;
void *src, *dest;
size_t size, copy_size;
unsigned int i, count;
size = map->records_count * new_record_size;
mem_map = i_new(struct mail_index_map, 1);
mem_map->refcount = 1;
mem_map->buffer = buffer_create_dynamic(default_pool, size);
if (map->hdr.record_size == new_record_size)
buffer_append(mem_map->buffer, map->records, size);
else {
copy_size = I_MIN(map->hdr.record_size, new_record_size);
src = map->records;
for (i = 0; i < map->records_count; i++) {
dest = buffer_append_space_unsafe(mem_map->buffer,
new_record_size);
memcpy(dest, src, copy_size);
src = PTR_OFFSET(src, map->hdr.record_size);
}
}
mem_map->records = buffer_get_modifiable_data(mem_map->buffer, NULL);
mem_map->records_count = map->records_count;
mem_map->hdr_copy_buf =
buffer_create_dynamic(default_pool, map->hdr.header_size);
if (map->hdr.base_header_size < sizeof(*hdr))
buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr));
buffer_write(mem_map->hdr_copy_buf, 0,
&map->hdr, map->hdr.base_header_size);
buffer_append(mem_map->hdr_copy_buf,
CONST_PTR_OFFSET(map->hdr_base,
map->hdr.base_header_size),
map->hdr.header_size - map->hdr.base_header_size);
hdr = buffer_get_modifiable_data(mem_map->hdr_copy_buf, NULL);
if (hdr->base_header_size < sizeof(*hdr))
hdr->base_header_size = sizeof(*hdr);
hdr->record_size = new_record_size;
mem_map->hdr = *hdr;
mem_map->hdr_base = hdr;
/* if we're syncing transaction log into memory and later use the
mapping for updating the index, we need to remember what has
changed */
mem_map->write_atomic = map->write_atomic;
if (map->write_to_disk) {
mem_map->write_seq_first = map->write_seq_first;
mem_map->write_seq_last = map->write_seq_last;
}
/* copy extensions */
if (array_is_created(&map->ext_id_map)) {
count = array_count(&map->ext_id_map);
mail_index_map_init_extbufs(mem_map, count + 2);
array_append_array(&mem_map->extensions, &map->extensions);
array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
/* fix the name pointers to use our own pool */
extensions = array_get_modifiable(&mem_map->extensions, &count);
for (i = 0; i < count; i++) {
extensions[i].name = p_strdup(mem_map->extension_pool,
extensions[i].name);
}
}
return mem_map;
}
int mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r)
{
const uint32_t *id;
if (!array_is_created(&map->ext_id_map) ||
ext_id >= array_count(&map->ext_id_map))
return 0;
id = array_idx(&map->ext_id_map, ext_id);
*idx_r = *id;
return *idx_r != (uint32_t)-1;
}
static int mail_index_try_open_only(struct mail_index *index)
{
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
/* Note that our caller must close index->fd by itself.
mail_index_reopen() for example wants to revert back to old
index file if opening the new one fails. */
index->fd = nfs_safe_open(index->filepath, O_RDWR);
index->readonly = FALSE;
if (index->fd == -1 && errno == EACCES) {
index->fd = open(index->filepath, O_RDONLY);
index->readonly = TRUE;
}
if (index->fd == -1) {
if (errno != ENOENT)
return mail_index_set_syscall_error(index, "open()");
/* have to create it */
return 0;
}
return 1;
}
static int
mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)
{
unsigned int lock_id;
int ret;
i_assert(index->fd == -1);
if (lock_id_r != NULL)
*lock_id_r = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
ret = mail_index_try_open_only(index);
if (ret <= 0)
return ret;
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) {
(void)close(index->fd);
index->fd = -1;
return -1;
}
ret = mail_index_map(index, FALSE);
if (ret == 0) {
/* it's corrupted - recreate it */
mail_index_unlock(index, lock_id);
if (lock_id_r != NULL)
*lock_id_r = 0;
(void)close(index->fd);
index->fd = -1;
} else {
if (lock_id_r != NULL)
*lock_id_r = lock_id;
else
mail_index_unlock(index, lock_id);
}
return ret;
}
int mail_index_write_base_header(struct mail_index *index,
const struct mail_index_header *hdr)
{
size_t hdr_size;
hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
memcpy(index->map->mmap_base, hdr, hdr_size);
if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
return mail_index_set_syscall_error(index, "msync()");
index->map->hdr = *hdr;
} else {
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
mail_index_set_syscall_error(index,
"pwrite_full()");
return -1;
}
}
index->map->hdr = *hdr;
buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size);
}
return 0;
}
int mail_index_create_tmp_file(struct mail_index *index, const char **path_r)
{
mode_t old_mask;
const char *path;
int fd;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
path = *path_r = t_strconcat(index->filepath, ".tmp", NULL);
old_mask = umask(0);
fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode);
umask(old_mask);
if (fd == -1)
return mail_index_file_set_syscall_error(index, path, "open()");
if (index->gid != (gid_t)-1 && fchown(fd, (uid_t)-1, index->gid) < 0) {
mail_index_file_set_syscall_error(index, path, "fchown()");
return -1;
}
return fd;
}
static int mail_index_create(struct mail_index *index,
struct mail_index_header *hdr)
{
const char *path;
uint32_t seq;
uoff_t offset;
int ret;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->lock_type == F_UNLCK);
/* log file lock protects index creation */
if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
return -1;
ret = mail_index_try_open(index, NULL);
if (ret != 0) {
mail_transaction_log_sync_unlock(index->log);
return ret < 0 ? -1 : 0;
}
/* mark the existing log file as synced */
hdr->log_file_seq = seq;
hdr->log_file_int_offset = offset;
hdr->log_file_ext_offset = offset;
/* create it fully in index.tmp first */
index->fd = mail_index_create_tmp_file(index, &path);
if (index->fd == -1)
ret = -1;
else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
mail_index_file_set_syscall_error(index, path, "write_full()");
ret = -1;
} else {
index->lock_type = F_WRLCK;
ret = mail_index_map(index, FALSE);
index->lock_type = F_UNLCK;
}
if (ret == 0) {
/* it's corrupted even while we just created it,
should never happen unless someone pokes the file directly */
mail_index_set_error(index,
"Newly created index file is corrupted: %s", path);
ret = -1;
}
if (ret < 0) {
if (unlink(path) < 0 && errno != ENOENT) {
mail_index_file_set_syscall_error(index, path,
"unlink()");
}
} else {
/* make it visible to others */
if (rename(path, index->filepath) < 0) {
mail_index_set_error(index, "rename(%s, %s) failed: %m",
path, index->filepath);
ret = -1;
}
}
mail_transaction_log_sync_unlock(index->log);
return ret;
}
static void mail_index_header_init(struct mail_index_header *hdr)
{
time_t now = time(NULL);
i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
memset(hdr, 0, sizeof(*hdr));
hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
hdr->base_header_size = sizeof(*hdr);
hdr->header_size = sizeof(*hdr);
hdr->record_size = sizeof(struct mail_index_record);
#ifndef WORDS_BIGENDIAN
hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
hdr->indexid = now;
hdr->next_uid = 1;
}
static void mail_index_create_in_memory(struct mail_index *index,
const struct mail_index_header *hdr)
{
struct mail_index_header tmp_hdr;
struct mail_index_map tmp_map;
if (hdr == NULL) {
mail_index_header_init(&tmp_hdr);
hdr = &tmp_hdr;
}
memset(&tmp_map, 0, sizeof(tmp_map));
tmp_map.hdr = *hdr;
tmp_map.hdr_base = hdr;
/* a bit kludgy way to do this, but it initializes everything
nicely and correctly */
index->map = mail_index_map_clone(&tmp_map, hdr->record_size);
index->hdr = &index->map->hdr;
}
/* returns -1 = error, 0 = won't create, 1 = ok */
static int mail_index_open_files(struct mail_index *index,
enum mail_index_open_flags flags)
{
struct mail_index_header hdr;
unsigned int lock_id = 0;
int ret;
bool created = FALSE;
ret = mail_index_try_open(index, &lock_id);
if (ret > 0)
hdr = *index->hdr;
else if (ret == 0) {
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0 &&
!MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
mail_index_header_init(&hdr);
index->hdr = &hdr;
} else if (ret < 0)
return -1;
index->indexid = hdr.indexid;
index->log = mail_transaction_log_open_or_create(index);
if (index->log == NULL)
return -1;
if (index->fd == -1) {
if (index->indexid != hdr.indexid) {
/* looks like someone else created the transaction log
before we had the chance. use its indexid so we
don't try to create conflicting ones. */
hdr.indexid = index->indexid;
}
if (lock_id != 0) {
mail_index_unlock(index, lock_id);
lock_id = 0;
}
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (mail_index_create(index, &hdr) < 0) {
/* fallback to in-memory index */
mail_index_move_to_memory(index);
mail_index_create_in_memory(index, &hdr);
}
} else {
mail_index_create_in_memory(index, &hdr);
}
created = TRUE;
}
if (lock_id == 0) {
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0)
return -1;
}
index->cache = created ? mail_cache_create(index) :
mail_cache_open_or_create(index);
mail_index_unlock(index, lock_id);
return 1;
}
int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags,
enum mail_index_lock_method lock_method)
{
int i = 0, ret;
if (index->opened) {
if (index->hdr != NULL &&
(index->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
/* corrupted, reopen files */
mail_index_close(index);
} else {
return 1;
}
}
index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
i_strdup("(in-memory index)") :
i_strconcat(index->dir, "/", index->prefix, NULL);
for (;;) {
index->shared_lock_count = 0;
index->excl_lock_count = 0;
index->lock_type = F_UNLCK;
index->lock_id = 2;
index->readonly = FALSE;
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
index->log_locked = FALSE;
index->mmap_disable =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
index->mmap_no_write =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0;
index->lock_method = lock_method;
/* don't even bother to handle dotlocking without mmap being
disabled. that combination simply doesn't make any sense */
if (lock_method == MAIL_INDEX_LOCK_DOTLOCK &&
!index->mmap_disable) {
i_fatal("lock_method=dotlock and mmap_disable=no "
"combination isn't supported. "
"You don't _really_ want it anyway.");
}
ret = mail_index_open_files(index, flags);
if (ret <= 0)
break;
index->opened = TRUE;
if (index->fsck) {
index->fsck = FALSE;
ret = mail_index_fsck(index);
if (ret == 0) {
/* completely broken, reopen */
if (i++ < 3)
continue;
/* too many tries */
ret = -1;
}
}
break;
}
if (ret <= 0)
mail_index_close(index);
return ret;
}
void mail_index_close(struct mail_index *index)
{
if (index->log != NULL)
mail_transaction_log_close(&index->log);
if (index->map != NULL)
mail_index_unmap(index, &index->map);
if (index->cache != NULL)
mail_cache_free(&index->cache);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
}
i_free_and_null(index->copy_lock_path);
i_free_and_null(index->filepath);
index->indexid = 0;
index->opened = FALSE;
}
int mail_index_reopen(struct mail_index *index, int fd)
{
struct mail_index_map *old_map;
unsigned int old_shared_locks, old_lock_id, lock_id = 0;
int ret, old_fd, old_lock_type;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->copy_lock_path == NULL || index->excl_lock_count == 0);
old_map = index->map;
old_fd = index->fd;
old_map->refcount++;
/* new file, new locks. the old fd can keep its locks, they don't
matter anymore as no-one's going to modify the file. */
old_lock_type = index->lock_type;
old_lock_id = index->lock_id;
old_shared_locks = index->shared_lock_count;
if (index->lock_type == F_RDLCK)
index->lock_type = F_UNLCK;
index->lock_id += 2;
index->shared_lock_count = 0;
if (fd != -1) {
index->fd = fd;
ret = 0;
} else {
i_assert(index->excl_lock_count == 0);
ret = mail_index_try_open_only(index);
if (ret > 0)
ret = mail_index_lock_shared(index, FALSE, &lock_id);
else if (ret == 0) {
/* index file is lost */
ret = -1;
}
}
if (ret == 0) {
/* read the new mapping. note that with mmap_disable we want
to keep the old mapping in index->map so we can update it
by reading transaction log. */
if (mail_index_map(index, TRUE) <= 0)
ret = -1;
}
if (lock_id != 0)
mail_index_unlock(index, lock_id);
if (ret == 0) {
mail_index_unmap(index, &old_map);
if (close(old_fd) < 0)
mail_index_set_syscall_error(index, "close()");
} else {
if (index->map != NULL)
mail_index_unmap(index, &index->map);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
}
index->map = old_map;
index->hdr = &index->map->hdr;
index->fd = old_fd;
index->lock_type = old_lock_type;
index->lock_id = old_lock_id;
index->shared_lock_count = old_shared_locks;
}
return ret;
}
int mail_index_reopen_if_needed(struct mail_index *index)
{
struct stat st1, st2;
i_assert(index->copy_lock_path == NULL);
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (fstat(index->fd, &st1) < 0) {
if (errno == ESTALE) {
/* deleted, reopen */
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
}
return mail_index_set_syscall_error(index, "fstat()");
}
if (nfs_safe_stat(index->filepath, &st2) < 0) {
mail_index_set_syscall_error(index, "stat()");
if (errno != ENOENT)
return -1;
/* lost it? recreate later */
mail_index_mark_corrupted(index);
return -1;
}
if (st1.st_ino != st2.st_ino ||
!CMP_DEV_T(st1.st_dev, st2.st_dev)) {
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
} else {
return 0;
}
}
int mail_index_refresh(struct mail_index *index)
{
unsigned int lock_id;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (index->excl_lock_count > 0) {
/* we have index exclusively locked, nothing could
have changed. */
return 0;
}
if (!index->mmap_disable) {
/* reopening is all we need */
return mail_index_reopen_if_needed(index);
}
i_assert(!index->mapping);
/* mail_index_map() simply reads latest changes from transaction log,
which makes us fully refreshed. */
if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
return -1;
ret = mail_index_map(index, FALSE);
mail_index_unlock(index, lock_id);
return ret <= 0 ? -1 : 0;
}
struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
return index->cache;
}
int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
va_list va;
i_free(index->error);
if (fmt == NULL)
index->error = NULL;
else {
va_start(va, fmt);
index->error = i_strdup_vprintf(fmt, va);
va_end(va);
i_error("%s", index->error);
}
return -1;
}
void mail_index_set_inconsistent(struct mail_index *index)
{
index->indexid = 0;
}
int mail_index_move_to_memory(struct mail_index *index)
{
struct mail_index_map *map;
int ret = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
/* set the index as being into memory */
i_free_and_null(index->dir);
if (index->map == NULL) {
/* mbox file was never even opened. just mark it as being in
memory and let the caller re-open the index. */
i_assert(index->fd == -1);
return -1;
}
/* move index map to memory */
map = mail_index_map_clone(index->map, index->map->hdr.record_size);
mail_index_unmap(index, &index->map);
index->map = map;
index->hdr = &map->hdr;
if (index->log != NULL) {
/* move transaction log to memory */
if (mail_transaction_log_move_to_memory(index->log) < 0)
ret = -1;
}
/* close the index file. */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
return ret;
}
void mail_index_mark_corrupted(struct mail_index *index)
{
struct mail_index_header hdr;
mail_index_set_inconsistent(index);
if (index->readonly)
return;
hdr = *index->hdr;
hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
if (mail_index_write_base_header(index, &hdr) == 0) {
if (!MAIL_INDEX_IS_IN_MEMORY(index) && fsync(index->fd) < 0)
mail_index_set_syscall_error(index, "fsync()");
}
}
int mail_index_set_syscall_error(struct mail_index *index,
const char *function)
{
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with index file %s: %m",
function, index->filepath);
}
int mail_index_file_set_syscall_error(struct mail_index *index,
const char *filepath,
const char *function)
{
i_assert(filepath != NULL);
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with file %s: %m",
function, filepath);
}
enum mail_index_error mail_index_get_last_error(struct mail_index *index)
{
if (index->nodiskspace)
return MAIL_INDEX_ERROR_DISKSPACE;
if (index->error != NULL)
return MAIL_INDEX_ERROR_INTERNAL;
return MAIL_INDEX_ERROR_NONE;
}
const char *mail_index_get_error_message(struct mail_index *index)
{
return index->error;
}
void mail_index_reset_error(struct mail_index *index)
{
if (index->error != NULL) {
i_free(index->error);
index->error = NULL;
}
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
}
uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
unsigned char buf[4];
i_assert(offset < 0x40000000);
i_assert((offset & 3) == 0);
offset >>= 2;
buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
buf[3] = 0x80 | (offset & 0x0000007f);
return *((uint32_t *) buf);
}
uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
const unsigned char *buf = (const unsigned char *) &offset;
if ((offset & 0x80808080) != 0x80808080)
return 0;
return (((uint32_t)buf[3] & 0x7f) << 2) |
(((uint32_t)buf[2] & 0x7f) << 9) |
(((uint32_t)buf[1] & 0x7f) << 16) |
(((uint32_t)buf[0] & 0x7f) << 23);
}