mail-index.c revision 369a1084c500a9df7448ffa9409ce32e42060bc2
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi/* Copyright (C) 2003-2004 Timo Sirainen */
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#include "lib.h"
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#include "array.h"
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#include "buffer.h"
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#include "hash.h"
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi#include "mmap-util.h"
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#include "nfs-workarounds.h"
d29c67900d3d104e24ed213ed4a10d204decdf18Aki Tuomi#include "read-full.h"
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#include "write-full.h"
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#include "mail-index-private.h"
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#include "mail-index-sync-private.h"
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#include "mail-transaction-log.h"
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#include "mail-cache.h"
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#include <stdio.h>
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#include <stddef.h>
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#include <time.h>
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#include <sys/stat.h>
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomiunsigned int mail_index_module_id = 0;
53309922d2ec3321e011966e8f2b210821ecd5e3Timo Sirainen
eba7f36feec8d02c4c394e55ff4effd47e33d311Timo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);
d78f1ac9dc0f3e6c64cebe9ee331ec6b3c160e89Timo Sirainenstatic void mail_index_create_in_memory(struct mail_index *index,
68cc278710182485b6c09e9a9ff8db90a727f343Aki Tuomi const struct mail_index_header *hdr);
676d5601a8e087b25eb4dd5509079ab832e66831Aki Tuomi
285f3c4cf828ebe9ff345080aa1df3755613af29Timo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)
6ef83bcdc4e40d6b387857e5f7d58cd86c71ec50Timo Sirainen{
6ef83bcdc4e40d6b387857e5f7d58cd86c71ec50Timo Sirainen struct mail_index *index;
99819adc9638200bc2ec1b68da9d2c68e8435740Aki Tuomi
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi index = i_new(struct mail_index, 1);
99819adc9638200bc2ec1b68da9d2c68e8435740Aki Tuomi index->dir = i_strdup(dir);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->prefix = i_strdup(prefix);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->fd = -1;
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi index->extension_pool = pool_alloconly_create("extension", 512);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi p_array_init(&index->extensions, index->extension_pool, 5);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi i_array_init(&index->sync_lost_handlers, 4);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi array_create(&index->mail_index_module_contexts, default_pool,
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi sizeof(void *), I_MIN(5, mail_index_module_id));
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi index->mode = 0600;
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi index->gid = (gid_t)-1;
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi index->keywords_ext_id =
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi mail_index_ext_register(index, "keywords", 128, 2, 1);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi index->keywords_pool = pool_alloconly_create("keywords", 512);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi i_array_init(&index->keywords, 16);
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen index->keywords_hash =
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi hash_create(default_pool, index->keywords_pool, 0,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi strcase_hash, (hash_cmp_callback_t *)strcasecmp);
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen return index;
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi}
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomivoid mail_index_free(struct mail_index **_index)
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi{
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi struct mail_index *index = *_index;
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi *_index = NULL;
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi mail_index_close(index);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi hash_destroy(index->keywords_hash);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi pool_unref(index->extension_pool);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi pool_unref(index->keywords_pool);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi array_free(&index->sync_lost_handlers);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi array_free(&index->keywords);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi array_free(&index->mail_index_module_contexts);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi i_free(index->error);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi i_free(index->dir);
c466bbccb4b8b0026aea13540ebdef3bddbd67ddAki Tuomi i_free(index->prefix);
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen i_free(index);
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen}
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainenvoid mail_index_set_permissions(struct mail_index *index,
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen mode_t mode, gid_t gid)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->mode = mode & 0666;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->gid = gid;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi}
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomiuint32_t mail_index_ext_register(struct mail_index *index, const char *name,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi uint32_t default_hdr_size,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi uint16_t default_record_size,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi uint16_t default_record_align)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi const struct mail_index_registered_ext *extensions;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi struct mail_index_registered_ext rext;
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi unsigned int i, ext_count;
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi extensions = array_get(&index->extensions, &ext_count);
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi /* see if it's already there */
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi for (i = 0; i < ext_count; i++) {
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi if (strcmp(extensions[i].name, name) == 0)
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi return i;
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi }
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi
7a8ef11587fd50d5888223fc3d91911775a21ba5Aki Tuomi memset(&rext, 0, sizeof(rext));
7d500ecf27acc5b65615ee9e72d6da6bacf799d2Timo Sirainen rext.name = p_strdup(index->extension_pool, name);
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen rext.index_idx = ext_count;
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen rext.hdr_size = default_hdr_size;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi rext.record_size = default_record_size;
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen rext.record_align = default_record_align;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi array_append(&index->extensions, &rext, 1);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return ext_count;
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen}
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomivoid mail_index_register_expunge_handler(struct mail_index *index,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi uint32_t ext_id, bool call_always,
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen mail_index_expunge_handler_t *cb,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi void *context)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi struct mail_index_registered_ext *rext;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi rext = array_idx_modifiable(&index->extensions, ext_id);
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen i_assert(rext->expunge_handler == NULL || rext->expunge_handler == cb);
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi rext->expunge_handler = cb;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi rext->expunge_context = context;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi rext->expunge_handler_call_always = call_always;
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen}
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomivoid mail_index_unregister_expunge_handler(struct mail_index *index,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi uint32_t ext_id)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi struct mail_index_registered_ext *rext;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen i_assert(rext->expunge_handler != NULL);
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen rext->expunge_handler = NULL;
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen}
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainenvoid mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi mail_index_sync_handler_t *cb,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi enum mail_index_sync_handler_type type)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi struct mail_index_registered_ext *rext;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
36a052b7bd94ccb47abbb6b15c1380f03780ba20Timo Sirainen i_assert(rext->sync_handler.callback == NULL);
36a052b7bd94ccb47abbb6b15c1380f03780ba20Timo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen rext->sync_handler.callback = cb;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen rext->sync_handler.type = type;
36a052b7bd94ccb47abbb6b15c1380f03780ba20Timo Sirainen}
2ac5f36aa7c2e7a07ba8815d43a6d7483f62e74cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainenvoid mail_index_unregister_sync_handler(struct mail_index *index,
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen uint32_t ext_id)
d9a7e950a9cd21f2b4a90ec7759fca9e8fcc7995Timo Sirainen{
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen struct mail_index_registered_ext *rext;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
d9a7e950a9cd21f2b4a90ec7759fca9e8fcc7995Timo Sirainen i_assert(rext->sync_handler.callback != NULL);
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen rext->sync_handler.callback = NULL;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen rext->sync_handler.type = 0;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen}
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainenvoid mail_index_register_sync_lost_handler(struct mail_index *index,
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen mail_index_sync_lost_handler_t *cb)
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen{
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen array_append(&index->sync_lost_handlers, &cb, 1);
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen}
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainenvoid mail_index_unregister_sync_lost_handler(struct mail_index *index,
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen mail_index_sync_lost_handler_t *cb)
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen{
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen mail_index_sync_lost_handler_t *const *handlers;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen unsigned int i, count;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen handlers = array_get(&index->sync_lost_handlers, &count);
36a052b7bd94ccb47abbb6b15c1380f03780ba20Timo Sirainen for (i = 0; i < count; i++) {
36a052b7bd94ccb47abbb6b15c1380f03780ba20Timo Sirainen if (handlers[i] == cb) {
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen array_delete(&index->sync_lost_handlers, i, 1);
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen break;
2ac5f36aa7c2e7a07ba8815d43a6d7483f62e74cTimo Sirainen }
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen }
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen}
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen unsigned int initial_count)
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen{
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#define EXTENSION_NAME_APPROX_LEN 20
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#define EXT_GLOBAL_ALLOC_SIZE \
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen#define EXT_PER_ALLOC_SIZE \
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen (EXTENSION_NAME_APPROX_LEN + \
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen sizeof(struct mail_index_ext) + sizeof(uint32_t))
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen size_t size;
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen if (map->extension_pool == NULL) {
3b759465ad5088a8d491b49c9ebb0dc9d3f66f7eTimo Sirainen size = EXT_GLOBAL_ALLOC_SIZE +
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen initial_count * EXT_PER_ALLOC_SIZE;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen map->extension_pool =
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen pool_alloconly_create("extensions",
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen nearest_power(size));
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen } else {
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen p_clear(map->extension_pool);
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen /* try to use the existing pool's size for initial_count so
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen we don't grow it unneededly */
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen size = p_get_max_easy_alloc_size(map->extension_pool);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen EXT_PER_ALLOC_SIZE;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen }
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi p_array_init(&map->extensions, map->extension_pool, initial_count);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen}
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainenuint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen{
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen const struct mail_index_ext *extensions;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen unsigned int i, size;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen if (!array_is_created(&map->extensions))
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen return (uint32_t)-1;
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen extensions = array_get(&map->extensions, &size);
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen for (i = 0; i < size; i++) {
368ec06f20b95d29d0bbd5655030048d060a2582Timo Sirainen if (strcmp(extensions[i].name, name) == 0)
55accf49e32ff93bfdd92961cb54ccc8c329147cTimo Sirainen return i;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen return (uint32_t)-1;
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi}
db8b229230860a6c12daa0017a49396986368897Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomiuint32_t
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomimail_index_map_register_ext(struct mail_index *index,
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi struct mail_index_map *map, const char *name,
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi uint32_t hdr_offset, uint32_t hdr_size,
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi uint32_t record_offset, uint32_t record_size,
c5e46dba179864f6f1adf196d46e7a0371b11914Josef 'Jeff' Sipek uint32_t record_align, uint32_t reset_id)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi struct mail_index_ext *ext;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi uint32_t idx, empty_idx = (uint32_t)-1;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi if (!array_is_created(&map->extensions)) {
db8b229230860a6c12daa0017a49396986368897Aki Tuomi mail_index_map_init_extbufs(map, 5);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi idx = 0;
db8b229230860a6c12daa0017a49396986368897Aki Tuomi } else {
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi idx = array_count(&map->extensions);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
05128fda80748e107bccdece0a3d23551e99e8f3Timo Sirainen i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);
db8b229230860a6c12daa0017a49396986368897Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext = array_append_space(&map->extensions);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext->name = p_strdup(map->extension_pool, name);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext->hdr_offset = hdr_offset;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext->hdr_size = hdr_size;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext->record_offset = record_offset;
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi ext->record_size = record_size;
db8b229230860a6c12daa0017a49396986368897Aki Tuomi ext->record_align = record_align;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext->reset_id = reset_id;
db8b229230860a6c12daa0017a49396986368897Aki Tuomi
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi ext->index_idx = mail_index_ext_register(index, name, hdr_size,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi record_size, record_align);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
db8b229230860a6c12daa0017a49396986368897Aki Tuomi /* Update index ext_id -> map ext_id mapping. Fill non-used
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext_ids with (uint32_t)-1 */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi while (array_count(&map->ext_id_map) < ext->index_idx)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi array_append(&map->ext_id_map, &empty_idx, 1);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return idx;
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi}
db8b229230860a6c12daa0017a49396986368897Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomistatic bool size_check(size_t *size_left, size_t size)
db8b229230860a6c12daa0017a49396986368897Aki Tuomi{
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi if (size > *size_left)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return FALSE;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi *size_left -= size;
db8b229230860a6c12daa0017a49396986368897Aki Tuomi return TRUE;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi}
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomistatic size_t get_align(size_t name_len)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi size_t size = sizeof(struct mail_index_ext_header) + name_len;
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
db8b229230860a6c12daa0017a49396986368897Aki Tuomi}
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi
db8b229230860a6c12daa0017a49396986368897Aki Tuomistatic int mail_index_parse_extensions(struct mail_index *index,
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi struct mail_index_map *map)
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi{
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi const struct mail_index_ext_header *ext_hdr;
db8b229230860a6c12daa0017a49396986368897Aki Tuomi unsigned int i, old_count;
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi const char *name;
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi uint32_t ext_id, offset, name_offset;
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi size_t size_left;
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi /* extension headers always start from 64bit offsets, so if base header
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi doesn't happen to be 64bit aligned we'll skip some bytes */
4afc67eb96f8d6b7dc94d63d3c7fe4f556c4fceeTimo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen /* nothing to do, skip allocatations and all */
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen return 1;
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen }
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi
f169d423d8db89e36f764263aa4cfe46c62d5a2bAki Tuomi old_count = array_count(&index->extensions);
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen mail_index_map_init_extbufs(map, old_count + 5);
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen ext_id = (uint32_t)-1;
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen for (i = 0; i < old_count; i++)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi array_append(&map->ext_id_map, &ext_id, 1);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi while (offset < map->hdr.header_size) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi /* Extension header contains:
db8b229230860a6c12daa0017a49396986368897Aki Tuomi - struct mail_index_ext_header
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi - name (not 0-terminated)
db8b229230860a6c12daa0017a49396986368897Aki Tuomi - 64bit alignment padding
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi - extension header contents
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi - 64bit alignment padding
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi */
db8b229230860a6c12daa0017a49396986368897Aki Tuomi size_left = map->hdr.header_size - offset;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (!size_check(&size_left, sizeof(*ext_hdr)) ||
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi !size_check(&size_left, ext_hdr->name_size) ||
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi !size_check(&size_left, get_align(ext_hdr->name_size)) ||
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi !size_check(&size_left, ext_hdr->hdr_size)) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi mail_index_set_error(index, "Corrupted index file %s: "
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen "Header extension goes outside header",
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen index->filepath);
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen return -1;
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen }
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen offset += sizeof(*ext_hdr);
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen name_offset = offset;
c5e46dba179864f6f1adf196d46e7a0371b11914Josef 'Jeff' Sipek offset += ext_hdr->name_size + get_align(ext_hdr->name_size);
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen t_push();
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen ext_hdr->name_size);
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen "Duplicate header extension %s",
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen index->filepath, name);
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen t_pop();
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen return -1;
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen }
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen (map->hdr.record_size % ext_hdr->record_align) != 0) {
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen "Record field %s alignmentation %u not used",
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->filepath, name, ext_hdr->record_align);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi t_pop();
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return -1;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi mail_index_map_register_ext(index, map, name,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi offset, ext_hdr->hdr_size,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext_hdr->record_offset,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext_hdr->record_size,
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi ext_hdr->record_align,
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi ext_hdr->reset_id);
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi t_pop();
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi }
98800cb9f53b7cc7b5dbfc78f2803707b9b0d324Aki Tuomi return 1;
98800cb9f53b7cc7b5dbfc78f2803707b9b0d324Aki Tuomi}
98800cb9f53b7cc7b5dbfc78f2803707b9b0d324Aki Tuomi
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomibool mail_index_keyword_lookup(struct mail_index *index,
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi const char *keyword, bool autocreate,
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi unsigned int *idx_r)
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi{
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi char *keyword_dup;
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi void *value;
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi /* keywords_hash keeps a name => index mapping of keywords.
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi Keywords are never removed from it, so the index values are valid
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi for the lifetime of the mail_index. */
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) {
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi *idx_r = POINTER_CAST_TO(value, unsigned int);
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi return TRUE;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
98800cb9f53b7cc7b5dbfc78f2803707b9b0d324Aki Tuomi
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi if (!autocreate) {
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi *idx_r = (unsigned int)-1;
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi return FALSE;
98800cb9f53b7cc7b5dbfc78f2803707b9b0d324Aki Tuomi }
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
98800cb9f53b7cc7b5dbfc78f2803707b9b0d324Aki Tuomi *idx_r = array_count(&index->keywords);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r));
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi array_append(&index->keywords, &keyword, 1);
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi return TRUE;
98800cb9f53b7cc7b5dbfc78f2803707b9b0d324Aki Tuomi}
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomiint mail_index_map_parse_keywords(struct mail_index *index,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi struct mail_index_map *map)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi const struct mail_index_ext *ext;
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi const struct mail_index_keyword_header *kw_hdr;
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi const struct mail_index_keyword_header_rec *kw_rec;
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi const char *name;
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi unsigned int i, name_area_end_offset, old_count;
235ef8cd3865984cc27f88a18d14ea98adb53f09Aki Tuomi uint32_t ext_id;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext_id = mail_index_map_lookup_ext(map, "keywords");
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (ext_id == (uint32_t)-1) {
cd75585e26420cef09cc699c6e02ab46e21937d1Aki Tuomi if (array_is_created(&map->keyword_idx_map))
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi array_clear(&map->keyword_idx_map);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return 0;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi ext = array_idx(&map->extensions, ext_id);
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi /* Extension header contains:
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi - struct mail_index_keyword_header
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi - struct mail_index_keyword_header_rec * keywords_count
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi - const char names[] * keywords_count
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi i_assert(ext->hdr_offset < map->hdr.header_size);
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi kw_rec = (const void *)(kw_hdr + 1);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi name = (const char *)(kw_rec + kw_hdr->keywords_count);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi array_count(&map->keyword_idx_map);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi /* Keywords can only be added into same mapping. Removing requires a
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi new mapping (recreating the index file) */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (kw_hdr->keywords_count == old_count) {
a18ba96c4ecebc9386b92385c546c868f1eae82eTimo Sirainen /* nothing changed */
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch return 0;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
14102a0c5db8828ca8c7751ec96587fadc97a0bcTimo Sirainen
14102a0c5db8828ca8c7751ec96587fadc97a0bcTimo Sirainen /* make sure the header is valid */
14102a0c5db8828ca8c7751ec96587fadc97a0bcTimo Sirainen if (kw_hdr->keywords_count < old_count) {
14102a0c5db8828ca8c7751ec96587fadc97a0bcTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
14102a0c5db8828ca8c7751ec96587fadc97a0bcTimo Sirainen "Keywords removed unexpectedly",
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->filepath);
a18ba96c4ecebc9386b92385c546c868f1eae82eTimo Sirainen return -1;
a18ba96c4ecebc9386b92385c546c868f1eae82eTimo Sirainen }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi mail_index_set_error(index, "Corrupted index file %s: "
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi "keywords_count larger than header size",
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->filepath);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return -1;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi for (i = 0; i < kw_hdr->keywords_count; i++) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (kw_rec[i].name_offset > name_area_end_offset) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi mail_index_set_error(index, "Corrupted index file %s: "
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi "name_offset points outside allocated header",
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->filepath);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return -1;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi if (name[name_area_end_offset-1] != '\0') {
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi mail_index_set_error(index, "Corrupted index file %s: "
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi "Keyword header doesn't end with NUL",
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi index->filepath);
79ec87ea6861e2dd447f69ab44a7cc4e4fce3fddAki Tuomi return -1;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
3d27e1102558215203b73c58a2cba84dccf0dd1aTimo Sirainen
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi /* create file -> index mapping */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (!array_is_created(&map->keyword_idx_map))
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#ifdef DEBUG
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi /* Check that existing headers are still the same. It's behind DEBUG
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi since it's pretty useless waste of CPU normally. */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi const char *keyword = name + kw_rec[i].name_offset;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi const unsigned int *old_idx;
e93184a9055c2530366dfe617e07199603c399ddMartti Rannanjärvi unsigned int idx;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi old_idx = array_idx(&map->keyword_idx_map, i);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) ||
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi idx != *old_idx) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi mail_index_set_error(index, "Corrupted index file %s: "
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi "Keywords changed unexpectedly",
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->filepath);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return -1;
79bbb900ffba886779474dfb04c41408f0ba0602Timo Sirainen }
36a052b7bd94ccb47abbb6b15c1380f03780ba20Timo Sirainen }
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi#endif
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi /* Register the newly seen keywords */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi i = array_count(&map->keyword_idx_map);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi for (; i < kw_hdr->keywords_count; i++) {
6a8c95b0693c93601e948e06bfe1f89abdd43307Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi unsigned int idx;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx);
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi array_append(&map->keyword_idx_map, &idx, 1);
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return 0;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi}
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomiconst ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index)
36a052b7bd94ccb47abbb6b15c1380f03780ba20Timo Sirainen{
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi /* Make sure all the keywords are in index->keywords. It's quick to do
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if nothing has changed. */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi (void)mail_index_map_parse_keywords(index, index->map);
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return &index->keywords;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi}
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomistatic int mail_index_check_header(struct mail_index *index,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi struct mail_index_map *map)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi const struct mail_index_header *hdr = &map->hdr;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi enum mail_index_header_compat_flags compat_flags = 0;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi#ifndef WORDS_BIGENDIAN
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi#endif
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi /* major version change - handle silently(?) */
c5e46dba179864f6f1adf196d46e7a0371b11914Josef 'Jeff' Sipek return -1;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (hdr->compat_flags != compat_flags) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi /* architecture change - handle silently(?) */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return -1;
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi /* we've already complained about it */
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi return -1;
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi }
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi /* following some extra checks that only take a bit of CPU */
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi mail_index_set_error(index, "Corrupted index file %s: "
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi "uid_validity = 0, next_uid = %u",
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->filepath, hdr->next_uid);
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen return -1;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (hdr->record_size < sizeof(struct mail_index_record)) {
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi mail_index_set_error(index, "Corrupted index file %s: "
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi "record_size too small: %u < %"PRIuSIZE_T,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi index->filepath, hdr->record_size,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi sizeof(struct mail_index_record));
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return -1;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi }
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return 0;
0746d8c498e2bb336399c37715d4fef9d64560c7Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (hdr->next_uid == 0)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return 0;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (hdr->recent_messages_count > hdr->messages_count ||
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi hdr->seen_messages_count > hdr->messages_count ||
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi hdr->deleted_messages_count > hdr->messages_count)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return 0;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (hdr->first_recent_uid_lowwater > hdr->next_uid ||
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi hdr->first_unseen_uid_lowwater > hdr->next_uid ||
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi hdr->first_deleted_uid_lowwater > hdr->next_uid)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return 0;
b0d00847d0e1ab20662efce5b24f8fee6897c6feTimo Sirainen
b0d00847d0e1ab20662efce5b24f8fee6897c6feTimo Sirainen if (map->records_count > 0) {
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainen /* last message's UID must be smaller than next_uid */
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi const struct mail_index_record *rec;
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi rec = MAIL_INDEX_MAP_IDX(map, map->records_count-1);
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi if (rec->uid >= hdr->next_uid)
2029c2cb37d7308e74329c9a15b6cf07a3468314Timo Sirainen return 0;
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi }
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi return mail_index_parse_extensions(index, map);
aaa1b6bb4cd2d7f8f4e7977d61176ea1c8f7e32bAki Tuomi}
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi
1e8a6a8708b612eee65f83ef6874aab94b15eb50Timo Sirainenstatic void mail_index_map_clear(struct mail_index *index,
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi struct mail_index_map *map)
14af7be4aa26d55c341cd6efe32bb2add2c39830Aki Tuomi{
if (map->buffer != NULL) {
i_assert(map->mmap_base == NULL);
buffer_free(map->buffer);
map->buffer = NULL;
} else if (map->mmap_base != NULL) {
i_assert(map->buffer == NULL);
if (munmap(map->mmap_base, map->mmap_size) < 0)
mail_index_set_syscall_error(index, "munmap()");
map->mmap_base = NULL;
}
if (map->refcount > 0) {
memset(&map->hdr, 0, sizeof(map->hdr));
map->mmap_size = 0;
map->mmap_used_size = 0;
map->records = NULL;
map->records_count = 0;
}
}
void mail_index_unmap(struct mail_index *index, struct mail_index_map **_map)
{
struct mail_index_map *map = *_map;
*_map = NULL;
if (--map->refcount > 0)
return;
i_assert(map->refcount == 0);
mail_index_map_clear(index, map);
if (map->extension_pool != NULL)
pool_unref(map->extension_pool);
if (array_is_created(&map->keyword_idx_map))
array_free(&map->keyword_idx_map);
buffer_free(map->hdr_copy_buf);
i_free(map);
}
static void mail_index_map_copy_hdr(struct mail_index_map *map,
const struct mail_index_header *hdr)
{
if (hdr->base_header_size < sizeof(map->hdr)) {
/* header smaller than ours, make a copy so our newer headers
won't have garbage in them */
memset(&map->hdr, 0, sizeof(map->hdr));
memcpy(&map->hdr, hdr, hdr->base_header_size);
} else {
map->hdr = *hdr;
}
}
static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
{
const struct mail_index_header *hdr;
unsigned int records_count;
i_assert(!map->write_to_disk);
if (map->buffer != NULL) {
/* we had temporarily used a buffer, eg. for updating index */
buffer_free(map->buffer);
map->buffer = NULL;
}
map->mmap_base = index->readonly ?
mmap_ro_file(index->fd, &map->mmap_size) :
mmap_rw_file(index->fd, &map->mmap_size);
if (map->mmap_base == MAP_FAILED) {
map->mmap_base = NULL;
mail_index_set_syscall_error(index, "mmap()");
return -1;
}
hdr = map->mmap_base;
if (map->mmap_size >
offsetof(struct mail_index_header, major_version) &&
hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
/* major version change - handle silently */
return 0;
}
if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
mail_index_set_error(index, "Corrupted index file %s: "
"File too small (%"PRIuSIZE_T")",
index->filepath, map->mmap_size);
return 0;
}
map->mmap_used_size = hdr->header_size +
hdr->messages_count * hdr->record_size;
if (map->mmap_used_size > map->mmap_size) {
records_count = (map->mmap_size - hdr->header_size) /
hdr->record_size;
mail_index_set_error(index, "Corrupted index file %s: "
"messages_count too large (%u > %u)",
index->filepath, hdr->messages_count,
records_count);
return 0;
}
mail_index_map_copy_hdr(map, hdr);
map->hdr_base = map->mmap_base;
map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
map->records_count = map->hdr.messages_count;
return 1;
}
static int mail_index_read_header(struct mail_index *index,
void *buf, size_t buf_size, size_t *pos_r)
{
size_t pos;
int ret;
memset(buf, 0, sizeof(struct mail_index_header));
/* try to read the whole header, but it's not necessarily an error to
read less since the older versions of the index format could be
smaller. Request reading up to buf_size, but accept if we only got
the header. */
pos = 0;
do {
ret = pread(index->fd, PTR_OFFSET(buf, pos),
buf_size - pos, pos);
if (ret > 0)
pos += ret;
} while (ret > 0 && pos < sizeof(struct mail_index_header));
*pos_r = pos;
return ret;
}
static int
mail_index_read_map(struct mail_index *index, struct mail_index_map *map,
bool *retry_r, bool try_retry)
{
const struct mail_index_header *hdr;
unsigned char buf[512];
void *data = NULL;
ssize_t ret;
size_t pos, records_size;
i_assert(map->mmap_base == NULL);
*retry_r = FALSE;
ret = mail_index_read_header(index, buf, sizeof(buf), &pos);
hdr = (const struct mail_index_header *)buf;
if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
/* major version change - handle silently */
return 0;
}
if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
(ret > 0 || pos >= hdr->base_header_size)) {
if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
hdr->header_size < hdr->base_header_size) {
mail_index_set_error(index, "Corrupted index file %s: "
"Corrupted header sizes (base %u, full %u)",
index->filepath, hdr->base_header_size,
hdr->header_size);
return 0;
}
if (pos > hdr->header_size)
pos = hdr->header_size;
/* place the base header into memory. */
buffer_reset(map->hdr_copy_buf);
buffer_append(map->hdr_copy_buf, buf, pos);
if (pos != hdr->header_size) {
/* @UNSAFE: read the rest of the header into memory */
data = buffer_append_space_unsafe(map->hdr_copy_buf,
hdr->header_size -
pos);
ret = pread_full(index->fd, data,
hdr->header_size - pos, pos);
}
}
if (ret > 0) {
/* header read, read the records now. */
records_size = hdr->messages_count * hdr->record_size;
if (map->buffer == NULL) {
map->buffer = buffer_create_dynamic(default_pool,
records_size);
}
/* @UNSAFE */
buffer_set_used_size(map->buffer, 0);
data = buffer_append_space_unsafe(map->buffer, records_size);
ret = pread_full(index->fd, data, records_size,
hdr->header_size);
}
if (ret < 0) {
if (errno == ESTALE && try_retry) {
/* a new index file was renamed over this one. */
*retry_r = TRUE;
return 0;
}
mail_index_set_syscall_error(index, "pread_full()");
return -1;
}
if (ret == 0) {
mail_index_set_error(index,
"Corrupted index file %s: File too small",
index->filepath);
return 0;
}
map->records = data;
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
map->hdr_base = map->hdr_copy_buf->data;
index->sync_log_file_seq = hdr->log_file_seq;
index->sync_log_file_offset = hdr->log_file_int_offset;
return 1;
}
bool mail_index_is_ext_synced(struct mail_transaction_log_view *log_view,
struct mail_index_map *map)
{
uint32_t prev_seq;
uoff_t prev_offset;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
return prev_seq < map->hdr.log_file_seq ||
(prev_seq == map->hdr.log_file_seq &&
prev_offset < map->hdr.log_file_ext_offset);
}
static int mail_index_sync_from_transactions(struct mail_index *index,
struct mail_index_map **map,
bool sync_to_index)
{
const struct mail_index_header *map_hdr = &(*map)->hdr;
struct mail_index_view *view;
struct mail_transaction_log_view *log_view;
struct mail_index_sync_map_ctx sync_map_ctx;
struct mail_index_header hdr;
const struct mail_transaction_header *thdr;
const void *tdata;
uint32_t prev_seq, max_seq;
uoff_t prev_offset, max_offset;
size_t pos;
int ret;
bool skipped, check_ext_offsets, broken;
if (sync_to_index) {
/* read the real log position where we are supposed to be
synced */
ret = mail_index_read_header(index, &hdr, sizeof(hdr), &pos);
if (ret < 0 && errno != ESTALE) {
mail_index_set_syscall_error(index, "pread()");
return -1;
}
if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
return 0;
if (map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset == hdr.log_file_int_offset) {
/* nothing to do */
return 1;
}
if (map_hdr->log_file_seq > hdr.log_file_seq ||
(map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset > hdr.log_file_int_offset)) {
/* we went too far, have to re-read the file */
return 0;
}
if (map_hdr->log_file_ext_offset !=
map_hdr->log_file_int_offset ||
hdr.log_file_ext_offset != hdr.log_file_int_offset) {
/* too much trouble to get this right. */
return 0;
}
max_seq = hdr.log_file_seq;
max_offset = hdr.log_file_int_offset;
} else {
/* sync everything there is */
max_seq = (uint32_t)-1;
max_offset = (uoff_t)-1;
}
log_view = mail_transaction_log_view_open(index->log);
if (mail_transaction_log_view_set(log_view,
map_hdr->log_file_seq,
map_hdr->log_file_int_offset,
max_seq, max_offset,
MAIL_TRANSACTION_TYPE_MASK) <= 0) {
/* can't use it. sync by re-reading index. */
mail_transaction_log_view_close(&log_view);
return 0;
}
index->map = *map;
view = mail_index_view_open(index);
mail_index_sync_map_init(&sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_HEAD);
check_ext_offsets = TRUE; broken = FALSE;
while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,
&skipped)) > 0) {
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0 &&
check_ext_offsets) {
if (mail_index_is_ext_synced(log_view, index->map))
continue;
check_ext_offsets = FALSE;
}
if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {
ret = 0;
broken = TRUE;
break;
}
}
if (ret == 0 && !broken)
ret = 1;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
i_assert(prev_seq <= max_seq &&
(prev_seq != max_seq || prev_offset <= max_offset));
index->map->hdr.log_file_seq = prev_seq;
index->map->hdr.log_file_int_offset =
index->map->hdr.log_file_ext_offset = prev_offset;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(&view);
mail_transaction_log_view_close(&log_view);
*map = index->map;
index->map = NULL;
if (sync_to_index && ret > 0) {
/* make sure we did everything right. note that although the
message counts should be equal, the flag counters may not */
i_assert(hdr.messages_count == (*map)->hdr.messages_count);
i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq);
i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset);
i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset);
}
return ret;
}
static int mail_index_read_map_with_retry(struct mail_index *index,
struct mail_index_map **map,
bool sync_to_index)
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
int ret;
bool retry;
if (index->log_locked) {
/* we're most likely syncing the index and we really don't
want to read more than what was synced last time. */
sync_to_index = TRUE;
}
if ((*map)->hdr.indexid != 0 && index->log != NULL) {
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
index->sync_update = TRUE;
ret = mail_index_sync_from_transactions(index, map,
sync_to_index);
index->sync_update = FALSE;
if (ret != 0)
return ret;
/* transaction log lost/broken, fallback to re-reading it */
}
/* notify all "sync lost" handlers */
handlers = array_get(&index->sync_lost_handlers, &count);
for (i = 0; i < count; i++)
(*handlers[i])(index);
for (i = 0;; i++) {
ret = mail_index_read_map(index, *map, &retry,
i < MAIL_INDEX_ESTALE_RETRY_COUNT);
if (ret != 0 || !retry)
return ret;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
}
static int mail_index_map_try_existing(struct mail_index_map *map)
{
const struct mail_index_header *hdr;
size_t used_size;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
return 0;
hdr = map->mmap_base;
/* always check corrupted-flag to avoid errors later */
if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)
return -1;
used_size = hdr->header_size + hdr->messages_count * hdr->record_size;
if (map->mmap_size >= used_size && map->hdr_base == hdr) {
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
return 1;
}
return 0;
}
int mail_index_map(struct mail_index *index, bool force)
{
struct mail_index_map *map;
int ret;
i_assert(!index->mapping);
i_assert(index->map == NULL || index->map->refcount > 0);
i_assert(index->lock_type != F_UNLCK);
if (MAIL_INDEX_IS_IN_MEMORY(index)) {
if (index->map == NULL)
mail_index_create_in_memory(index, NULL);
return 1;
}
index->mapping = TRUE;
if (!force && index->map != NULL) {
i_assert(index->hdr != NULL);
ret = mail_index_map_try_existing(index->map);
if (ret != 0) {
index->mapping = FALSE;
return ret;
}
if (index->lock_type == F_WRLCK) {
/* we're syncing, don't break the mapping */
index->mapping = FALSE;
return 1;
}
}
if (index->map != NULL && index->map->refcount > 1) {
/* this map is already used by some views and they may have
pointers into it. leave them and create a new mapping. */
if (!index->mmap_disable) {
map = NULL;
} else {
/* create a copy of the mapping instead so we don't
have to re-read it */
map = mail_index_map_clone(index->map,
index->map->hdr.record_size);
}
index->map->refcount--;
index->map = NULL;
} else {
map = index->map;
}
if (map == NULL) {
map = i_new(struct mail_index_map, 1);
map->refcount = 1;
map->hdr_copy_buf =
buffer_create_dynamic(default_pool, sizeof(map->hdr));
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
i_assert(!map->write_to_disk);
} else if (map->mmap_base != NULL) {
i_assert(map->buffer == NULL);
if (munmap(map->mmap_base, map->mmap_size) < 0)
mail_index_set_syscall_error(index, "munmap()");
map->mmap_base = NULL;
}
index->hdr = NULL;
index->map = NULL;
if (!index->mmap_disable)
ret = mail_index_mmap(index, map);
else
ret = mail_index_read_map_with_retry(index, &map, force);
i_assert(index->map == NULL);
if (ret > 0) {
ret = mail_index_check_header(index, map);
if (ret < 0)
ret = 0;
else if (ret == 0) {
index->fsck = TRUE;
ret = 1;
}
}
if (ret <= 0) {
mail_index_map_clear(index, map);
mail_index_unmap(index, &map);
index->mapping = FALSE;
return ret;
}
index->hdr = &map->hdr;
index->map = map;
i_assert(map->hdr.messages_count == map->records_count);
index->mapping = FALSE;
return 1;
}
int mail_index_get_latest_header(struct mail_index *index,
struct mail_index_header *hdr_r)
{
size_t pos;
unsigned int i;
int ret;
if (!index->mmap_disable) {
ret = mail_index_map(index, FALSE);
if (ret > 0)
*hdr_r = *index->hdr;
else
memset(hdr_r, 0, sizeof(*hdr_r));
return ret;
}
for (i = 0;; i++) {
ret = mail_index_read_header(index, hdr_r, sizeof(*hdr_r),
&pos);
if (ret <= 0 || errno != ESTALE ||
i == MAIL_INDEX_ESTALE_RETRY_COUNT)
break;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
if (ret < 0)
mail_index_set_syscall_error(index, "pread_full()");
return ret;
}
struct mail_index_map *
mail_index_map_clone(struct mail_index_map *map, uint32_t new_record_size)
{
struct mail_index_map *mem_map;
struct mail_index_header *hdr;
struct mail_index_ext *extensions;
void *src, *dest;
size_t size, copy_size;
unsigned int i, count;
size = map->records_count * new_record_size;
mem_map = i_new(struct mail_index_map, 1);
mem_map->refcount = 1;
mem_map->buffer = buffer_create_dynamic(default_pool, size);
if (map->hdr.record_size == new_record_size)
buffer_append(mem_map->buffer, map->records, size);
else {
copy_size = I_MIN(map->hdr.record_size, new_record_size);
src = map->records;
for (i = 0; i < map->records_count; i++) {
dest = buffer_append_space_unsafe(mem_map->buffer,
new_record_size);
memcpy(dest, src, copy_size);
src = PTR_OFFSET(src, map->hdr.record_size);
}
}
mem_map->records = buffer_get_modifiable_data(mem_map->buffer, NULL);
mem_map->records_count = map->records_count;
mem_map->hdr_copy_buf =
buffer_create_dynamic(default_pool, map->hdr.header_size);
if (map->hdr.base_header_size < sizeof(*hdr))
buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr));
buffer_write(mem_map->hdr_copy_buf, 0,
&map->hdr, map->hdr.base_header_size);
buffer_append(mem_map->hdr_copy_buf,
CONST_PTR_OFFSET(map->hdr_base,
map->hdr.base_header_size),
map->hdr.header_size - map->hdr.base_header_size);
hdr = buffer_get_modifiable_data(mem_map->hdr_copy_buf, NULL);
if (hdr->base_header_size < sizeof(*hdr))
hdr->base_header_size = sizeof(*hdr);
hdr->record_size = new_record_size;
mem_map->hdr = *hdr;
mem_map->hdr_base = hdr;
/* if we're syncing transaction log into memory and later use the
mapping for updating the index, we need to remember what has
changed */
mem_map->write_atomic = map->write_atomic;
if (map->write_to_disk) {
mem_map->write_seq_first = map->write_seq_first;
mem_map->write_seq_last = map->write_seq_last;
}
/* copy extensions */
if (array_is_created(&map->ext_id_map)) {
count = array_count(&map->ext_id_map);
mail_index_map_init_extbufs(mem_map, count + 2);
array_append_array(&mem_map->extensions, &map->extensions);
array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
/* fix the name pointers to use our own pool */
extensions = array_get_modifiable(&mem_map->extensions, &count);
for (i = 0; i < count; i++) {
extensions[i].name = p_strdup(mem_map->extension_pool,
extensions[i].name);
}
}
return mem_map;
}
int mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r)
{
const uint32_t *id;
if (!array_is_created(&map->ext_id_map) ||
ext_id >= array_count(&map->ext_id_map))
return 0;
id = array_idx(&map->ext_id_map, ext_id);
*idx_r = *id;
return *idx_r != (uint32_t)-1;
}
static int mail_index_try_open_only(struct mail_index *index)
{
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
/* Note that our caller must close index->fd by itself.
mail_index_reopen() for example wants to revert back to old
index file if opening the new one fails. */
index->fd = nfs_safe_open(index->filepath, O_RDWR);
index->readonly = FALSE;
if (index->fd == -1 && errno == EACCES) {
index->fd = open(index->filepath, O_RDONLY);
index->readonly = TRUE;
}
if (index->fd == -1) {
if (errno != ENOENT)
return mail_index_set_syscall_error(index, "open()");
/* have to create it */
return 0;
}
return 1;
}
static int
mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)
{
unsigned int lock_id;
int ret;
i_assert(index->fd == -1);
i_assert(index->lock_type == F_UNLCK);
if (lock_id_r != NULL)
*lock_id_r = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
ret = mail_index_try_open_only(index);
if (ret <= 0)
return ret;
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) {
(void)close(index->fd);
index->fd = -1;
return -1;
}
ret = mail_index_map(index, FALSE);
if (ret == 0) {
/* it's corrupted - recreate it */
mail_index_unlock(index, lock_id);
if (lock_id_r != NULL)
*lock_id_r = 0;
i_assert(index->file_lock == NULL);
(void)close(index->fd);
index->fd = -1;
} else {
if (lock_id_r != NULL)
*lock_id_r = lock_id;
else
mail_index_unlock(index, lock_id);
}
return ret;
}
int mail_index_write_base_header(struct mail_index *index,
const struct mail_index_header *hdr)
{
size_t hdr_size;
hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
memcpy(index->map->mmap_base, hdr, hdr_size);
if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
return mail_index_set_syscall_error(index, "msync()");
index->map->hdr = *hdr;
} else {
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
mail_index_set_syscall_error(index,
"pwrite_full()");
return -1;
}
}
index->map->hdr = *hdr;
buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size);
}
return 0;
}
int mail_index_create_tmp_file(struct mail_index *index, const char **path_r)
{
mode_t old_mask;
const char *path;
int fd;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
path = *path_r = t_strconcat(index->filepath, ".tmp", NULL);
old_mask = umask(0);
fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode);
umask(old_mask);
if (fd == -1)
return mail_index_file_set_syscall_error(index, path, "open()");
if (index->gid != (gid_t)-1 && fchown(fd, (uid_t)-1, index->gid) < 0) {
mail_index_file_set_syscall_error(index, path, "fchown()");
return -1;
}
return fd;
}
static int mail_index_create(struct mail_index *index,
struct mail_index_header *hdr)
{
const char *path;
uint32_t seq;
uoff_t offset;
int ret;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->lock_type == F_UNLCK);
/* log file lock protects index creation */
if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
return -1;
ret = mail_index_try_open(index, NULL);
if (ret != 0) {
mail_transaction_log_sync_unlock(index->log);
return ret < 0 ? -1 : 0;
}
/* mark the existing log file as synced */
hdr->log_file_seq = seq;
hdr->log_file_int_offset = offset;
hdr->log_file_ext_offset = offset;
/* create it fully in index.tmp first */
index->fd = mail_index_create_tmp_file(index, &path);
if (index->fd == -1)
ret = -1;
else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
mail_index_file_set_syscall_error(index, path, "write_full()");
ret = -1;
} else {
index->lock_type = F_WRLCK;
ret = mail_index_map(index, FALSE);
index->lock_type = F_UNLCK;
}
if (ret == 0) {
/* it's corrupted even while we just created it,
should never happen unless someone pokes the file directly */
mail_index_set_error(index,
"Newly created index file is corrupted: %s", path);
ret = -1;
}
if (ret < 0) {
if (unlink(path) < 0 && errno != ENOENT) {
mail_index_file_set_syscall_error(index, path,
"unlink()");
}
} else {
/* make it visible to others */
if (rename(path, index->filepath) < 0) {
mail_index_set_error(index, "rename(%s, %s) failed: %m",
path, index->filepath);
ret = -1;
}
}
mail_transaction_log_sync_unlock(index->log);
return ret;
}
static void mail_index_header_init(struct mail_index_header *hdr)
{
time_t now = time(NULL);
i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
memset(hdr, 0, sizeof(*hdr));
hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
hdr->base_header_size = sizeof(*hdr);
hdr->header_size = sizeof(*hdr);
hdr->record_size = sizeof(struct mail_index_record);
#ifndef WORDS_BIGENDIAN
hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
hdr->indexid = now;
hdr->next_uid = 1;
}
static void mail_index_create_in_memory(struct mail_index *index,
const struct mail_index_header *hdr)
{
struct mail_index_header tmp_hdr;
struct mail_index_map tmp_map;
if (hdr == NULL) {
mail_index_header_init(&tmp_hdr);
hdr = &tmp_hdr;
}
memset(&tmp_map, 0, sizeof(tmp_map));
tmp_map.hdr = *hdr;
tmp_map.hdr_base = hdr;
/* a bit kludgy way to do this, but it initializes everything
nicely and correctly */
index->map = mail_index_map_clone(&tmp_map, hdr->record_size);
index->hdr = &index->map->hdr;
}
/* returns -1 = error, 0 = won't create, 1 = ok */
static int mail_index_open_files(struct mail_index *index,
enum mail_index_open_flags flags)
{
struct mail_index_header hdr;
unsigned int lock_id = 0;
int ret;
bool created = FALSE;
ret = mail_index_try_open(index, &lock_id);
if (ret > 0)
hdr = *index->hdr;
else if (ret == 0) {
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0 &&
!MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
mail_index_header_init(&hdr);
index->hdr = &hdr;
} else if (ret < 0)
return -1;
index->indexid = hdr.indexid;
index->log = mail_transaction_log_open_or_create(index);
if (index->log == NULL)
return -1;
if (index->fd == -1) {
if (index->indexid != hdr.indexid) {
/* looks like someone else created the transaction log
before we had the chance. use its indexid so we
don't try to create conflicting ones. */
hdr.indexid = index->indexid;
}
if (lock_id != 0) {
mail_index_unlock(index, lock_id);
lock_id = 0;
}
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (mail_index_create(index, &hdr) < 0) {
/* fallback to in-memory index */
mail_index_move_to_memory(index);
mail_index_create_in_memory(index, &hdr);
}
} else {
mail_index_create_in_memory(index, &hdr);
}
created = TRUE;
}
if (lock_id == 0) {
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0)
return -1;
}
index->cache = created ? mail_cache_create(index) :
mail_cache_open_or_create(index);
mail_index_unlock(index, lock_id);
return 1;
}
int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags,
enum file_lock_method lock_method)
{
int i = 0, ret;
if (index->opened) {
if (index->hdr != NULL &&
(index->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
/* corrupted, reopen files */
mail_index_close(index);
} else {
return 1;
}
}
index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
i_strdup("(in-memory index)") :
i_strconcat(index->dir, "/", index->prefix, NULL);
for (;;) {
index->shared_lock_count = 0;
index->excl_lock_count = 0;
index->lock_type = F_UNLCK;
index->lock_id = 2;
index->readonly = FALSE;
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
index->log_locked = FALSE;
index->mmap_disable =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
index->mmap_no_write =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0;
index->fsync_disable =
(flags & MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE) != 0;
index->lock_method = lock_method;
/* don't even bother to handle dotlocking without mmap being
disabled. that combination simply doesn't make any sense */
if (lock_method == FILE_LOCK_METHOD_DOTLOCK &&
!index->mmap_disable) {
i_fatal("lock_method=dotlock and mmap_disable=no "
"combination isn't supported. "
"You don't _really_ want it anyway.");
}
ret = mail_index_open_files(index, flags);
if (ret <= 0)
break;
index->opened = TRUE;
if (index->fsck) {
index->fsck = FALSE;
ret = mail_index_fsck(index);
if (ret == 0) {
/* completely broken, reopen */
if (i++ < 3)
continue;
/* too many tries */
ret = -1;
}
}
break;
}
if (ret <= 0)
mail_index_close(index);
return ret;
}
void mail_index_close(struct mail_index *index)
{
if (index->log != NULL)
mail_transaction_log_close(&index->log);
if (index->map != NULL)
mail_index_unmap(index, &index->map);
if (index->cache != NULL)
mail_cache_free(&index->cache);
if (index->file_lock != NULL)
file_lock_free(&index->file_lock);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
}
i_free_and_null(index->copy_lock_path);
i_free_and_null(index->filepath);
index->indexid = 0;
index->opened = FALSE;
}
int mail_index_reopen(struct mail_index *index, int fd)
{
struct mail_index_map *old_map;
struct file_lock *old_file_lock;
unsigned int old_shared_locks, old_lock_id, lock_id = 0;
int ret, old_fd, old_lock_type;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->excl_lock_count == 0);
old_map = index->map;
old_fd = index->fd;
old_map->refcount++;
/* new file, new locks. the old fd can keep its locks, they don't
matter anymore as no-one's going to modify the file. */
old_lock_type = index->lock_type;
old_lock_id = index->lock_id;
old_shared_locks = index->shared_lock_count;
old_file_lock = index->file_lock;
if (index->lock_type == F_RDLCK)
index->lock_type = F_UNLCK;
index->lock_id += 2;
index->shared_lock_count = 0;
index->file_lock = NULL;
if (fd != -1) {
index->fd = fd;
ret = 0;
} else {
i_assert(index->excl_lock_count == 0);
ret = mail_index_try_open_only(index);
if (ret > 0)
ret = mail_index_lock_shared(index, FALSE, &lock_id);
else if (ret == 0) {
/* index file is lost */
ret = -1;
}
}
if (ret == 0) {
/* read the new mapping. note that with mmap_disable we want
to keep the old mapping in index->map so we can update it
by reading transaction log. */
if (mail_index_map(index, TRUE) <= 0)
ret = -1;
}
if (lock_id != 0)
mail_index_unlock(index, lock_id);
if (ret == 0) {
mail_index_unmap(index, &old_map);
if (old_file_lock != NULL)
file_lock_free(&old_file_lock);
if (close(old_fd) < 0)
mail_index_set_syscall_error(index, "close()");
} else {
if (index->map != NULL)
mail_index_unmap(index, &index->map);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
}
index->map = old_map;
index->hdr = &index->map->hdr;
index->fd = old_fd;
index->file_lock = old_file_lock;
index->lock_type = old_lock_type;
index->lock_id = old_lock_id;
index->shared_lock_count = old_shared_locks;
}
return ret;
}
int mail_index_reopen_if_needed(struct mail_index *index)
{
struct stat st1, st2;
i_assert(index->copy_lock_path == NULL);
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (fstat(index->fd, &st1) < 0) {
if (errno == ESTALE) {
/* deleted, reopen */
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
}
return mail_index_set_syscall_error(index, "fstat()");
}
if (nfs_safe_stat(index->filepath, &st2) < 0) {
mail_index_set_syscall_error(index, "stat()");
if (errno != ENOENT)
return -1;
/* lost it? recreate later */
mail_index_mark_corrupted(index);
return -1;
}
if (st1.st_ino != st2.st_ino ||
!CMP_DEV_T(st1.st_dev, st2.st_dev)) {
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
} else {
return 0;
}
}
int mail_index_refresh(struct mail_index *index)
{
unsigned int lock_id;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (index->excl_lock_count > 0) {
/* we have index exclusively locked, nothing could
have changed. */
return 0;
}
if (!index->mmap_disable) {
/* reopening is all we need */
return mail_index_reopen_if_needed(index);
}
i_assert(!index->mapping);
/* mail_index_map() simply reads latest changes from transaction log,
which makes us fully refreshed. */
if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
return -1;
ret = mail_index_map(index, FALSE);
mail_index_unlock(index, lock_id);
return ret <= 0 ? -1 : 0;
}
struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
return index->cache;
}
int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
va_list va;
i_free(index->error);
if (fmt == NULL)
index->error = NULL;
else {
va_start(va, fmt);
index->error = i_strdup_vprintf(fmt, va);
va_end(va);
i_error("%s", index->error);
}
return -1;
}
void mail_index_set_inconsistent(struct mail_index *index)
{
index->indexid = 0;
}
int mail_index_move_to_memory(struct mail_index *index)
{
struct mail_index_map *map;
int ret = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (index->map == NULL) {
/* mbox file was never even opened. just mark it as being in
memory and let the caller re-open the index. */
i_assert(index->fd == -1);
return -1;
}
/* set the index as being into memory */
i_free_and_null(index->dir);
/* move index map to memory */
map = mail_index_map_clone(index->map, index->map->hdr.record_size);
mail_index_unmap(index, &index->map);
index->map = map;
index->hdr = &map->hdr;
if (index->log != NULL) {
/* move transaction log to memory */
if (mail_transaction_log_move_to_memory(index->log) < 0)
ret = -1;
}
if (index->file_lock != NULL)
file_lock_free(&index->file_lock);
/* close the index file. */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
return ret;
}
void mail_index_mark_corrupted(struct mail_index *index)
{
struct mail_index_header hdr;
mail_index_set_inconsistent(index);
if (index->readonly || index->map == NULL)
return;
hdr = *index->hdr;
hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
if (mail_index_write_base_header(index, &hdr) == 0) {
if (!MAIL_INDEX_IS_IN_MEMORY(index) && fsync(index->fd) < 0)
mail_index_set_syscall_error(index, "fsync()");
}
}
int mail_index_set_syscall_error(struct mail_index *index,
const char *function)
{
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with index file %s: %m",
function, index->filepath);
}
int mail_index_file_set_syscall_error(struct mail_index *index,
const char *filepath,
const char *function)
{
i_assert(filepath != NULL);
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with file %s: %m",
function, filepath);
}
enum mail_index_error mail_index_get_last_error(struct mail_index *index)
{
if (index->nodiskspace)
return MAIL_INDEX_ERROR_DISKSPACE;
if (index->error != NULL)
return MAIL_INDEX_ERROR_INTERNAL;
return MAIL_INDEX_ERROR_NONE;
}
const char *mail_index_get_error_message(struct mail_index *index)
{
return index->error;
}
void mail_index_reset_error(struct mail_index *index)
{
if (index->error != NULL) {
i_free(index->error);
index->error = NULL;
}
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
}
uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
unsigned char buf[4];
i_assert(offset < 0x40000000);
i_assert((offset & 3) == 0);
offset >>= 2;
buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
buf[3] = 0x80 | (offset & 0x0000007f);
return *((uint32_t *) buf);
}
uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
const unsigned char *buf = (const unsigned char *) &offset;
if ((offset & 0x80808080) != 0x80808080)
return 0;
return (((uint32_t)buf[3] & 0x7f) << 2) |
(((uint32_t)buf[2] & 0x7f) << 9) |
(((uint32_t)buf[1] & 0x7f) << 16) |
(((uint32_t)buf[0] & 0x7f) << 23);
}