2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "lib.h"

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "array.h"

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "buffer.h"

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "hash.h"

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "mmap-util.h"

5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen#include "nfs-workarounds.h"

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "read-full.h"

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "write-full.h"

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "mail-index-private.h"

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen#include "mail-index-sync-private.h"

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen#include "mail-transaction-log.h"

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen#include "mail-cache.h"

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include <stdio.h>

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include <stddef.h>

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include <time.h>

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include <sys/stat.h>

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic void mail_index_create_in_memory(struct mail_index *index,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const struct mail_index_header *hdr);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen struct mail_index *index;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen index = i_new(struct mail_index, 1);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen index->dir = i_strdup(dir);

5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen index->prefix = i_strdup(prefix);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen index->fd = -1;

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen index->extension_pool = pool_alloconly_create("extension", 512);

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen ARRAY_CREATE(&index->extensions, index->extension_pool,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen struct mail_index_registered_ext, 5);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ARRAY_CREATE(&index->sync_lost_handlers, default_pool,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_sync_lost_handler_t *, 4);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen index->mode = 0600;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen index->gid = (gid_t)-1;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen index->keywords_ext_id =

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_ext_register(index, "keywords", 128, 2, 1);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen index->keywords_pool = pool_alloconly_create("keywords", 512);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen ARRAY_CREATE(&index->keywords, default_pool, const char *, 16);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen index->keywords_hash =

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen hash_create(default_pool, index->keywords_pool, 0,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen strcase_hash, (hash_cmp_callback_t *)strcasecmp);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return index;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid mail_index_free(struct mail_index **_index)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen struct mail_index *index = *_index;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen *_index = NULL;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_close(index);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen hash_destroy(index->keywords_hash);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen pool_unref(index->extension_pool);

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen pool_unref(index->keywords_pool);

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen array_free(&index->sync_lost_handlers);

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen array_free(&index->keywords);

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen i_free(index->error);

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen i_free(index->dir);

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen i_free(index->prefix);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_free(index);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2726de0d31ed0f1a1425e4c8d167f7681e07fc25Timo Sirainenvoid mail_index_set_permissions(struct mail_index *index,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mode_t mode, gid_t gid)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen index->mode = mode & 0666;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen index->gid = gid;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint32_t default_hdr_size,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint16_t default_record_size,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint16_t default_record_align)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen const struct mail_index_registered_ext *extensions;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen struct mail_index_registered_ext rext;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen unsigned int i, ext_count;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen extensions = array_get(&index->extensions, &ext_count);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen /* see if it's already there */

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen for (i = 0; i < ext_count; i++) {

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen if (strcmp(extensions[i].name, name) == 0)

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen return i;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen }

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen memset(&rext, 0, sizeof(rext));

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext.name = p_strdup(index->extension_pool, name);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext.index_idx = ext_count;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext.hdr_size = default_hdr_size;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext.record_size = default_record_size;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext.record_align = default_record_align;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen array_append(&index->extensions, &rext, 1);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen return ext_count;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen}

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainenvoid mail_index_register_expunge_handler(struct mail_index *index,

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen uint32_t ext_id,

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen mail_index_expunge_handler_t *cb)

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen{

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen struct mail_index_registered_ext *rext;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext = array_idx_modifyable(&index->extensions, ext_id);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen i_assert(rext->expunge_handler == NULL);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext->expunge_handler = cb;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen}

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainenvoid mail_index_unregister_expunge_handler(struct mail_index *index,

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen uint32_t ext_id)

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen{

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen struct mail_index_registered_ext *rext;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext = array_idx_modifyable(&index->extensions, ext_id);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen i_assert(rext->expunge_handler != NULL);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen rext->expunge_handler = NULL;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainenvoid mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen mail_index_sync_handler_t *cb,

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen enum mail_index_sync_handler_type type)

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen{

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen struct mail_index_registered_ext *rext;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext = array_idx_modifyable(&index->extensions, ext_id);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen i_assert(rext->sync_handler.callback == NULL);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen rext->sync_handler.callback = cb;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen rext->sync_handler.type = type;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenvoid mail_index_unregister_sync_handler(struct mail_index *index,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint32_t ext_id)

b75feeb580a78d76188f66d589939b298da4b61bTimo Sirainen{

00b6a40fd9f032914b41fe2cfe52d902db29829bTimo Sirainen struct mail_index_registered_ext *rext;

b75feeb580a78d76188f66d589939b298da4b61bTimo Sirainen

b75feeb580a78d76188f66d589939b298da4b61bTimo Sirainen rext = array_idx_modifyable(&index->extensions, ext_id);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen i_assert(rext->sync_handler.callback != NULL);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext->sync_handler.callback = NULL;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen rext->sync_handler.type = 0;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen}

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainenvoid mail_index_register_sync_lost_handler(struct mail_index *index,

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen mail_index_sync_lost_handler_t *cb)

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen{

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen array_append(&index->sync_lost_handlers, &cb, 1);

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen}

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainenvoid mail_index_unregister_sync_lost_handler(struct mail_index *index,

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen mail_index_sync_lost_handler_t *cb)

b75feeb580a78d76188f66d589939b298da4b61bTimo Sirainen{

b75feeb580a78d76188f66d589939b298da4b61bTimo Sirainen mail_index_sync_lost_handler_t *const *handlers;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int i, count;

447bf65ddb82ec279e7386828748ef47e199a6afTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen handlers = array_get(&index->sync_lost_handlers, &count);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (handlers[i] == cb) {

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen array_delete(&index->sync_lost_handlers, i, 1);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen break;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int initial_count)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#define EXTENSION_NAME_APPROX_LEN 20

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen size_t size;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (map->extension_pool == NULL) {

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen size = (sizeof(array_t) + BUFFER_APPROX_SIZE) * 2 +

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen initial_count * (EXTENSION_NAME_APPROX_LEN +

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen sizeof(struct mail_index_ext) +

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen sizeof(uint32_t));

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen map->extension_pool =

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen pool_alloconly_create("extensions",

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen nearest_power(size));

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen } else {

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen p_clear(map->extension_pool);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }

5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen

5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen ARRAY_CREATE(&map->extensions, map->extension_pool,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen struct mail_index_ext, initial_count);

5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen ARRAY_CREATE(&map->ext_id_map, map->extension_pool,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint32_t, initial_count);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainenuint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

7dd73e056c7d9976080393d9cd731f69111818eeTimo Sirainen const struct mail_index_ext *extensions;

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen unsigned int i, size;

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen if (!array_is_created(&map->extensions))

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen return (uint32_t)-1;

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen extensions = array_get(&map->extensions, &size);

dc1bc1685e4a0d58ae7bacaecc282d0ebde2d7daTimo Sirainen for (i = 0; i < size; i++) {

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen if (strcmp(extensions[i].name, name) == 0)

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen return i;

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen }

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return (uint32_t)-1;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenmail_index_map_register_ext(struct mail_index *index,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen struct mail_index_map *map, const char *name,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint32_t hdr_offset, uint32_t hdr_size,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint32_t record_offset, uint32_t record_size,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint32_t record_align, uint32_t reset_id)

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen{

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen struct mail_index_ext *ext;

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen uint32_t idx, empty_idx = (uint32_t)-1;

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen if (!array_is_created(&map->extensions)) {

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen mail_index_map_init_extbufs(map, 5);

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen idx = 0;

5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen } else {

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen idx = array_count(&map->extensions);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext = array_append_space(&map->extensions);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext->name = p_strdup(map->extension_pool, name);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext->hdr_offset = hdr_offset;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext->hdr_size = hdr_size;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext->record_offset = record_offset;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext->record_size = record_size;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext->record_align = record_align;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext->reset_id = reset_id;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext->index_idx = mail_index_ext_register(index, name, hdr_size,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen record_size, record_align);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* Update index ext_id -> map ext_id mapping. Fill non-used

5733207dc3ec10e6e5a6e0a8b30fbd1b061062b9Timo Sirainen while (array_count(&map->ext_id_map) < ext->index_idx)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen array_append(&map->ext_id_map, &empty_idx, 1);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen array_idx_set(&map->ext_id_map, ext->index_idx, &idx);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return idx;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic bool size_check(size_t *size_left, size_t size)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (size > *size_left)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return FALSE;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen *size_left -= size;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return TRUE;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

b6b9c99fefbbc662bd9a0006566133c4480bf0e8Timo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic size_t get_align(size_t name_len)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen size_t size = sizeof(struct mail_index_ext_header) + name_len;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic int mail_index_parse_extensions(struct mail_index *index,

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen struct mail_index_map *map)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const struct mail_index_ext_header *ext_hdr;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int i, old_count;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const char *name;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uint32_t ext_id, offset, name_offset;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen size_t size_left;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* extension headers always start from 64bit offsets, so if base header

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (offset >= map->hdr.header_size && map->extension_pool == NULL) {

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* nothing to do, skip allocatations and all */

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return 1;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen old_count = array_count(&index->extensions);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_map_init_extbufs(map, old_count + 5);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext_id = (uint32_t)-1;

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < old_count; i++)

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen array_append(&map->ext_id_map, &ext_id, 1);

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen while (offset < map->hdr.header_size) {

2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);

/* Extension header contains:

size_left = map->hdr.header_size - offset;

if (!size_check(&size_left, sizeof(*ext_hdr)) ||

!size_check(&size_left, ext_hdr->name_size) ||

!size_check(&size_left, get_align(ext_hdr->name_size)) ||

!size_check(&size_left, ext_hdr->hdr_size)) {

mail_index_set_error(index, "Corrupted index file %s: "

"Header extension goes outside header",

index->filepath);

return -1;

}

offset += sizeof(*ext_hdr);

name_offset = offset;

offset += ext_hdr->name_size + get_align(ext_hdr->name_size);

t_push();

name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),

ext_hdr->name_size);

if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {

mail_index_set_error(index, "Corrupted index file %s: "

"Duplicate header extension %s",

index->filepath, name);

t_pop();

return -1;

}

if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||

(map->hdr.record_size % ext_hdr->record_align) != 0) {

mail_index_set_error(index, "Corrupted index file %s: "

"Record field %s alignmentation %u not used",

index->filepath, name, ext_hdr->record_align);

t_pop();

return -1;

}

mail_index_map_register_ext(index, map, name,

offset, ext_hdr->hdr_size,

ext_hdr->record_offset,

ext_hdr->record_size,

ext_hdr->record_align,

ext_hdr->reset_id);

t_pop();

offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);

}

return 1;

}

bool mail_index_keyword_lookup(struct mail_index *index,

const char *keyword, bool autocreate,

unsigned int *idx_r)

{

char *keyword_dup;

void *value;

/* keywords_hash keeps a name => index mapping of keywords.

if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) {

*idx_r = POINTER_CAST_TO(value, unsigned int);

return TRUE;

}

if (!autocreate) {

*idx_r = (unsigned int)-1;

return FALSE;

}

keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);

*idx_r = array_count(&index->keywords);

hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r));

array_append(&index->keywords, &keyword, 1);

return TRUE;

}

int mail_index_map_parse_keywords(struct mail_index *index,

struct mail_index_map *map)

{

const struct mail_index_ext *ext;

const struct mail_index_keyword_header *kw_hdr;

const struct mail_index_keyword_header_rec *kw_rec;

const char *name;

unsigned int i, name_area_end_offset, old_count;

uint32_t ext_id;

ext_id = mail_index_map_lookup_ext(map, "keywords");

if (ext_id == (uint32_t)-1) {

if (array_is_created(&map->keyword_idx_map))

array_clear(&map->keyword_idx_map);

return 0;

}

ext = array_idx(&map->extensions, ext_id);

/* Extension header contains:

i_assert(ext->hdr_offset < map->hdr.header_size);

kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);

kw_rec = (const void *)(kw_hdr + 1);

name = (const char *)(kw_rec + kw_hdr->keywords_count);

old_count = !array_is_created(&map->keyword_idx_map) ? 0 :

array_count(&map->keyword_idx_map);

/* Keywords can only be added into same mapping. Removing requires a

if (kw_hdr->keywords_count == old_count) {

/* nothing changed */

return 0;

}

/* make sure the header is valid */

if (kw_hdr->keywords_count < old_count) {

mail_index_set_error(index, "Corrupted index file %s: "

"Keywords removed unexpectedly",

index->filepath);

return -1;

}

if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {

mail_index_set_error(index, "Corrupted index file %s: "

"keywords_count larger than header size",

index->filepath);

return -1;

}

name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;

for (i = 0; i < kw_hdr->keywords_count; i++) {

if (kw_rec[i].name_offset > name_area_end_offset) {

mail_index_set_error(index, "Corrupted index file %s: "

"name_offset points outside allocated header",

index->filepath);

return -1;

}

}

if (name[name_area_end_offset-1] != '\0') {

mail_index_set_error(index, "Corrupted index file %s: "

"Keyword header doesn't end with NUL",

index->filepath);

return -1;

}

/* create file -> index mapping */

if (!array_is_created(&map->keyword_idx_map)) {

ARRAY_CREATE(&map->keyword_idx_map, default_pool,

unsigned int, kw_hdr->keywords_count);

}

#ifdef DEBUG

/* Check that existing headers are still the same. It's behind DEBUG

for (i = 0; i < array_count(&map->keyword_idx_map); i++) {

const char *keyword = name + kw_rec[i].name_offset;

const unsigned int *old_idx;

unsigned int idx;

old_idx = array_idx(&map->keyword_idx_map, i);

if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) ||

idx != *old_idx) {

mail_index_set_error(index, "Corrupted index file %s: "

"Keywords changed unexpectedly",

index->filepath);

return -1;

}

}

#endif

/* Register the newly seen keywords */

i = array_count(&map->keyword_idx_map);

for (; i < kw_hdr->keywords_count; i++) {

const char *keyword = name + kw_rec[i].name_offset;

unsigned int idx;

(void)mail_index_keyword_lookup(index, keyword, TRUE, &idx);

array_append(&map->keyword_idx_map, &idx, 1);

}

return 0;

}

const array_t *mail_index_get_keywords(struct mail_index *index)

{

/* Make sure all the keywords are in index->keywords. It's quick to do

(void)mail_index_map_parse_keywords(index, index->map);

return &index->keywords;

}

static int mail_index_check_header(struct mail_index *index,

struct mail_index_map *map)

{

const struct mail_index_header *hdr = &map->hdr;

enum mail_index_header_compat_flags compat_flags = 0;

#ifndef WORDS_BIGENDIAN

compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;

#endif

if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {

/* major version change - handle silently(?) */

return -1;

}

if (hdr->compat_flags != compat_flags) {

/* architecture change - handle silently(?) */

return -1;

}

if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {

/* we've already complained about it */

return -1;

}

/* following some extra checks that only take a bit of CPU */

if (hdr->uid_validity == 0 && hdr->next_uid != 1) {

mail_index_set_error(index, "Corrupted index file %s: "

"uid_validity = 0, next_uid = %u",

index->filepath, hdr->next_uid);

return -1;

}

if (hdr->record_size < sizeof(struct mail_index_record)) {

mail_index_set_error(index, "Corrupted index file %s: "

"record_size too small: %u < %"PRIuSIZE_T,

index->filepath, hdr->record_size,

sizeof(struct mail_index_record));

return -1;

}

if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)

return 0;

if (hdr->next_uid == 0)

return 0;

if (hdr->recent_messages_count > hdr->messages_count ||

hdr->seen_messages_count > hdr->messages_count ||

hdr->deleted_messages_count > hdr->messages_count)

return 0;

if (hdr->first_recent_uid_lowwater > hdr->next_uid ||

hdr->first_unseen_uid_lowwater > hdr->next_uid ||

hdr->first_deleted_uid_lowwater > hdr->next_uid)

return 0;

return mail_index_parse_extensions(index, map);

}

static void mail_index_map_clear(struct mail_index *index,

struct mail_index_map *map)

{

if (map->buffer != NULL) {

i_assert(map->mmap_base == NULL);

buffer_free(map->buffer);

map->buffer = NULL;

} else if (map->mmap_base != NULL) {

i_assert(map->buffer == NULL);

if (munmap(map->mmap_base, map->mmap_size) < 0)

mail_index_set_syscall_error(index, "munmap()");

map->mmap_base = NULL;

}

if (map->refcount > 0) {

memset(&map->hdr, 0, sizeof(map->hdr));

map->mmap_size = 0;

map->mmap_used_size = 0;

map->records = NULL;

map->records_count = 0;

}

}

void mail_index_unmap(struct mail_index *index, struct mail_index_map **_map)

{

struct mail_index_map *map = *_map;

*_map = NULL;

if (--map->refcount > 0)

return;

i_assert(map->refcount == 0);

mail_index_map_clear(index, map);

if (map->extension_pool != NULL)

pool_unref(map->extension_pool);

if (array_is_created(&map->keyword_idx_map))

array_free(&map->keyword_idx_map);

buffer_free(map->hdr_copy_buf);

i_free(map);

}

static void mail_index_map_copy_hdr(struct mail_index_map *map,

const struct mail_index_header *hdr)

{

if (hdr->base_header_size < sizeof(map->hdr)) {

/* header smaller than ours, make a copy so our newer headers

memset(&map->hdr, 0, sizeof(map->hdr));

memcpy(&map->hdr, hdr, hdr->base_header_size);

} else {

map->hdr = *hdr;

}

}

static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)

{

const struct mail_index_header *hdr;

unsigned int records_count;

i_assert(!map->write_to_disk);

if (map->buffer != NULL) {

/* we had temporarily used a buffer, eg. for updating index */

buffer_free(map->buffer);

map->buffer = NULL;

}

map->mmap_base = index->readonly ?

mmap_ro_file(index->fd, &map->mmap_size) :

mmap_rw_file(index->fd, &map->mmap_size);

if (map->mmap_base == MAP_FAILED) {

map->mmap_base = NULL;

mail_index_set_syscall_error(index, "mmap()");

return -1;

}

hdr = map->mmap_base;

if (map->mmap_size >

offsetof(struct mail_index_header, major_version) &&

hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {

/* major version change - handle silently */

return 0;

}

if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {

mail_index_set_error(index, "Corrupted index file %s: "

"File too small (%"PRIuSIZE_T")",

index->filepath, map->mmap_size);

return 0;

}

map->mmap_used_size = hdr->header_size +

hdr->messages_count * hdr->record_size;

if (map->mmap_used_size > map->mmap_size) {

records_count = (map->mmap_size - hdr->header_size) /

hdr->record_size;

mail_index_set_error(index, "Corrupted index file %s: "

"messages_count too large (%u > %u)",

index->filepath, hdr->messages_count,

records_count);

return 0;

}

mail_index_map_copy_hdr(map, hdr);

map->hdr_base = map->mmap_base;

map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);

map->records_count = map->hdr.messages_count;

return 1;

}

static int mail_index_read_header(struct mail_index *index,

void *buf, size_t buf_size, size_t *pos_r)

{

size_t pos;

int ret;

memset(buf, 0, sizeof(struct mail_index_header));

/* try to read the whole header, but it's not necessarily an error to

pos = 0;

do {

ret = pread(index->fd, PTR_OFFSET(buf, pos),

buf_size - pos, pos);

if (ret > 0)

pos += ret;

} while (ret > 0 && pos < sizeof(struct mail_index_header));

*pos_r = pos;

return ret;

}

static int

mail_index_read_map(struct mail_index *index, struct mail_index_map *map,

bool *retry_r, bool try_retry)

{

const struct mail_index_header *hdr;

unsigned char buf[512];

void *data = NULL;

ssize_t ret;

size_t pos, records_size;

i_assert(map->mmap_base == NULL);

*retry_r = FALSE;

ret = mail_index_read_header(index, buf, sizeof(buf), &pos);

hdr = (const struct mail_index_header *)buf;

if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&

hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {

/* major version change - handle silently */

return 0;

}

if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&

(ret > 0 || pos >= hdr->base_header_size)) {

if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||

hdr->header_size < hdr->base_header_size) {

mail_index_set_error(index, "Corrupted index file %s: "

"Corrupted header sizes (base %u, full %u)",

index->filepath, hdr->base_header_size,

hdr->header_size);

return 0;

}

if (pos > hdr->header_size)

pos = hdr->header_size;

/* place the base header into memory. */

buffer_reset(map->hdr_copy_buf);

buffer_append(map->hdr_copy_buf, buf, pos);

if (pos != hdr->header_size) {

/* @UNSAFE: read the rest of the header into memory */

data = buffer_append_space_unsafe(map->hdr_copy_buf,

hdr->header_size -

pos);

ret = pread_full(index->fd, data,

hdr->header_size - pos, pos);

}

}

if (ret > 0) {

/* header read, read the records now. */

records_size = hdr->messages_count * hdr->record_size;

if (map->buffer == NULL) {

map->buffer = buffer_create_dynamic(default_pool,

records_size);

}

/* @UNSAFE */

buffer_set_used_size(map->buffer, 0);

data = buffer_append_space_unsafe(map->buffer, records_size);

ret = pread_full(index->fd, data, records_size,

hdr->header_size);

}

if (ret < 0) {

if (errno == ESTALE && try_retry) {

/* a new index file was renamed over this one. */

*retry_r = TRUE;

return 0;

}

mail_index_set_syscall_error(index, "pread_full()");

return -1;

}

if (ret == 0) {

mail_index_set_error(index,

"Corrupted index file %s: File too small",

index->filepath);

return 0;

}

map->records = data;

map->records_count = hdr->messages_count;

mail_index_map_copy_hdr(map, hdr);

map->hdr_base = map->hdr_copy_buf->data;

index->sync_log_file_seq = hdr->log_file_seq;

index->sync_log_file_offset = hdr->log_file_int_offset;

return 1;

}

bool mail_index_is_ext_synced(struct mail_transaction_log_view *log_view,

struct mail_index_map *map)

{

uint32_t prev_seq;

uoff_t prev_offset;

mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,

&prev_offset);

return prev_seq < map->hdr.log_file_seq ||

(prev_seq == map->hdr.log_file_seq &&

prev_offset < map->hdr.log_file_ext_offset);

}

static int mail_index_sync_from_transactions(struct mail_index *index,

struct mail_index_map **map,

bool sync_to_index)

{

const struct mail_index_header *map_hdr = &(*map)->hdr;

struct mail_index_view *view;

struct mail_transaction_log_view *log_view;

struct mail_index_sync_map_ctx sync_map_ctx;

struct mail_index_header hdr;

const struct mail_transaction_header *thdr;

const void *tdata;

uint32_t prev_seq, max_seq;

uoff_t prev_offset, max_offset;

size_t pos;

int ret;

bool skipped, check_ext_offsets, broken;

if (sync_to_index) {

/* read the real log position where we are supposed to be

ret = mail_index_read_header(index, &hdr, sizeof(hdr), &pos);

if (ret < 0 && errno != ESTALE) {

mail_index_set_syscall_error(index, "pread()");

return -1;

}

if (pos < MAIL_INDEX_HEADER_MIN_SIZE)

return 0;

if (map_hdr->log_file_seq == hdr.log_file_seq &&

map_hdr->log_file_int_offset == hdr.log_file_int_offset) {

/* nothing to do */

return 1;

}

if (map_hdr->log_file_seq > hdr.log_file_seq ||

(map_hdr->log_file_seq == hdr.log_file_seq &&

map_hdr->log_file_int_offset > hdr.log_file_int_offset)) {

/* we went too far, have to re-read the file */

return 0;

}

if (map_hdr->log_file_ext_offset !=

map_hdr->log_file_int_offset ||

hdr.log_file_ext_offset != hdr.log_file_int_offset) {

/* too much trouble to get this right. */

return 0;

}

max_seq = hdr.log_file_seq;

max_offset = hdr.log_file_int_offset;

} else {

/* sync everything there is */

max_seq = (uint32_t)-1;

max_offset = (uoff_t)-1;

}

log_view = mail_transaction_log_view_open(index->log);

if (mail_transaction_log_view_set(log_view,

map_hdr->log_file_seq,

map_hdr->log_file_int_offset,

max_seq, max_offset,

MAIL_TRANSACTION_TYPE_MASK) <= 0) {

/* can't use it. sync by re-reading index. */

mail_transaction_log_view_close(&log_view);

return 0;

}

index->map = *map;

view = mail_index_view_open(index);

mail_index_sync_map_init(&sync_map_ctx, view,

MAIL_INDEX_SYNC_HANDLER_HEAD);

check_ext_offsets = TRUE; broken = FALSE;

while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,

&skipped)) > 0) {

if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0 &&

check_ext_offsets) {

if (mail_index_is_ext_synced(log_view, index->map))

continue;

check_ext_offsets = FALSE;

}

if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {

ret = 0;

broken = TRUE;

break;

}

}

if (ret == 0 && !broken)

ret = 1;

mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,

&prev_offset);

i_assert(prev_seq <= max_seq &&

(prev_seq != max_seq || prev_offset <= max_offset));

index->map->hdr.log_file_seq = prev_seq;

index->map->hdr.log_file_int_offset =

index->map->hdr.log_file_ext_offset = prev_offset;

mail_index_sync_map_deinit(&sync_map_ctx);

mail_index_view_close(&view);

mail_transaction_log_view_close(&log_view);

*map = index->map;

index->map = NULL;

if (sync_to_index && ret > 0) {

/* make sure we did everything right. note that although the

i_assert(hdr.messages_count == (*map)->hdr.messages_count);

i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq);

i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset);

i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset);

}

return ret;

}

static int mail_index_read_map_with_retry(struct mail_index *index,

struct mail_index_map **map,

bool sync_to_index)

{

mail_index_sync_lost_handler_t *const *handlers;

unsigned int i, count;

int ret;

bool retry;

if (index->log_locked) {

/* we're most likely syncing the index and we really don't

sync_to_index = TRUE;

}

if ((*map)->hdr.indexid != 0 && index->log != NULL) {

/* we're not creating the index, or opening transaction log.

index->sync_update = TRUE;

ret = mail_index_sync_from_transactions(index, map,

sync_to_index);

index->sync_update = FALSE;

if (ret != 0)

return ret;

/* transaction log lost/broken, fallback to re-reading it */

}

/* notify all "sync lost" handlers */

handlers = array_get(&index->sync_lost_handlers, &count);

for (i = 0; i < count; i++)

(*handlers[i])(index);

for (i = 0;; i++) {

ret = mail_index_read_map(index, *map, &retry,

i < MAIL_INDEX_ESTALE_RETRY_COUNT);

if (ret != 0 || !retry)

return ret;

/* ESTALE - reopen index file */

if (close(index->fd) < 0)

mail_index_set_syscall_error(index, "close()");

index->fd = -1;

ret = mail_index_try_open_only(index);

if (ret <= 0) {

if (ret == 0) {

/* the file was lost */

errno = ENOENT;

mail_index_set_syscall_error(index, "open()");

}

return -1;

}

}

/* Too many ESTALE retries */

mail_index_set_syscall_error(index, "read_map()");

return -1;

}

static int mail_index_map_try_existing(struct mail_index_map *map)

{

const struct mail_index_header *hdr;

size_t used_size;

if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))

return 0;

hdr = map->mmap_base;

/* always check corrupted-flag to avoid errors later */

if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)

return -1;

used_size = hdr->header_size + hdr->messages_count * hdr->record_size;

if (map->mmap_size >= used_size && map->hdr_base == hdr) {

map->records_count = hdr->messages_count;

mail_index_map_copy_hdr(map, hdr);

return 1;

}

return 0;

}

int mail_index_map(struct mail_index *index, bool force)

{

struct mail_index_map *map;

int ret;

i_assert(!index->mapping);

i_assert(index->map == NULL || index->map->refcount > 0);

i_assert(index->lock_type != F_UNLCK);

if (MAIL_INDEX_IS_IN_MEMORY(index)) {

if (index->map == NULL)

mail_index_create_in_memory(index, NULL);

return 1;

}

index->mapping = TRUE;

if (!force && index->map != NULL) {

i_assert(index->hdr != NULL);

ret = mail_index_map_try_existing(index->map);

if (ret != 0) {

index->mapping = FALSE;

return ret;

}

if (index->lock_type == F_WRLCK) {

/* we're syncing, don't break the mapping */

index->mapping = FALSE;

return 1;

}

}

if (index->map != NULL && index->map->refcount > 1) {

/* this map is already used by some views and they may have

if (!index->mmap_disable) {

map = NULL;

} else {

/* create a copy of the mapping instead so we don't

map = mail_index_map_clone(index->map,

index->map->hdr.record_size);

}

index->map->refcount--;

index->map = NULL;

} else {

map = index->map;

}

if (map == NULL) {

map = i_new(struct mail_index_map, 1);

map->refcount = 1;

map->hdr_copy_buf =

buffer_create_dynamic(default_pool, sizeof(map->hdr));

} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {

i_assert(!map->write_to_disk);

} else if (map->mmap_base != NULL) {

i_assert(map->buffer == NULL);

if (munmap(map->mmap_base, map->mmap_size) < 0)

mail_index_set_syscall_error(index, "munmap()");

map->mmap_base = NULL;

}

index->hdr = NULL;

index->map = NULL;

if (!index->mmap_disable)

ret = mail_index_mmap(index, map);

else

ret = mail_index_read_map_with_retry(index, &map, force);

i_assert(index->map == NULL);

if (ret > 0) {

ret = mail_index_check_header(index, map);

if (ret < 0)

ret = 0;

else if (ret == 0) {

index->fsck = TRUE;

ret = 1;

}

}

if (ret <= 0) {

mail_index_map_clear(index, map);

mail_index_unmap(index, &map);

index->mapping = FALSE;

return ret;

}

index->hdr = &map->hdr;

index->map = map;

i_assert(map->hdr.messages_count == map->records_count);

index->mapping = FALSE;

return 1;

}

int mail_index_get_latest_header(struct mail_index *index,

struct mail_index_header *hdr_r)

{

size_t pos;

unsigned int i;

int ret;

if (!index->mmap_disable) {

ret = mail_index_map(index, FALSE);

if (ret > 0)

*hdr_r = *index->hdr;

else

memset(hdr_r, 0, sizeof(*hdr_r));

return ret;

}

for (i = 0;; i++) {

ret = mail_index_read_header(index, hdr_r, sizeof(*hdr_r),

&pos);

if (ret <= 0 || errno != ESTALE ||

i == MAIL_INDEX_ESTALE_RETRY_COUNT)

break;

/* ESTALE - reopen index file */

if (close(index->fd) < 0)

mail_index_set_syscall_error(index, "close()");

index->fd = -1;

ret = mail_index_try_open_only(index);

if (ret <= 0) {

if (ret == 0) {

/* the file was lost */

errno = ENOENT;

mail_index_set_syscall_error(index, "open()");

}

return -1;

}

}

if (ret < 0)

mail_index_set_syscall_error(index, "pread_full()");

return ret;

}

struct mail_index_map *

mail_index_map_clone(struct mail_index_map *map, uint32_t new_record_size)

{

struct mail_index_map *mem_map;

struct mail_index_header *hdr;

struct mail_index_ext *extensions;

void *src, *dest;

size_t size, copy_size;

unsigned int i, count;

size = map->records_count * new_record_size;

mem_map = i_new(struct mail_index_map, 1);

mem_map->refcount = 1;

mem_map->buffer = buffer_create_dynamic(default_pool, size);

if (map->hdr.record_size == new_record_size)

buffer_append(mem_map->buffer, map->records, size);

else {

copy_size = I_MIN(map->hdr.record_size, new_record_size);

src = map->records;

for (i = 0; i < map->records_count; i++) {

dest = buffer_append_space_unsafe(mem_map->buffer,

new_record_size);

memcpy(dest, src, copy_size);

src = PTR_OFFSET(src, map->hdr.record_size);

}

}

mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL);

mem_map->records_count = map->records_count;

mem_map->hdr_copy_buf =

buffer_create_dynamic(default_pool, map->hdr.header_size);

if (map->hdr.base_header_size < sizeof(*hdr))

buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr));

buffer_write(mem_map->hdr_copy_buf, 0,

&map->hdr, map->hdr.base_header_size);

buffer_append(mem_map->hdr_copy_buf,

CONST_PTR_OFFSET(map->hdr_base,

map->hdr.base_header_size),

map->hdr.header_size - map->hdr.base_header_size);

hdr = buffer_get_modifyable_data(mem_map->hdr_copy_buf, NULL);

if (hdr->base_header_size < sizeof(*hdr))

hdr->base_header_size = sizeof(*hdr);

hdr->record_size = new_record_size;

mem_map->hdr = *hdr;

mem_map->hdr_base = hdr;

/* if we're syncing transaction log into memory and later use the

mem_map->write_atomic = map->write_atomic;

if (map->write_to_disk) {

mem_map->write_seq_first = map->write_seq_first;

mem_map->write_seq_last = map->write_seq_last;

}

/* copy extensions */

if (array_is_created(&map->ext_id_map)) {

count = array_count(&map->ext_id_map);

mail_index_map_init_extbufs(mem_map, count + 2);

array_append_array(&mem_map->extensions, &map->extensions);

array_append_array(&mem_map->ext_id_map, &map->ext_id_map);

/* fix the name pointers to use our own pool */

extensions = array_get_modifyable(&mem_map->extensions, &count);

for (i = 0; i < count; i++) {

extensions[i].name = p_strdup(mem_map->extension_pool,

extensions[i].name);

}

}

return mem_map;

}

int mail_index_map_get_ext_idx(struct mail_index_map *map,

uint32_t ext_id, uint32_t *idx_r)

{

const uint32_t *id;

if (!array_is_created(&map->ext_id_map) ||

ext_id >= array_count(&map->ext_id_map))

return 0;

id = array_idx(&map->ext_id_map, ext_id);

*idx_r = *id;

return *idx_r != (uint32_t)-1;

}

static int mail_index_try_open_only(struct mail_index *index)

{

i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

/* Note that our caller must close index->fd by itself.

index->fd = nfs_safe_open(index->filepath, O_RDWR);

index->readonly = FALSE;

if (index->fd == -1 && errno == EACCES) {

index->fd = open(index->filepath, O_RDONLY);

index->readonly = TRUE;

}

if (index->fd == -1) {

if (errno != ENOENT)

return mail_index_set_syscall_error(index, "open()");

/* have to create it */

return 0;

}

return 1;

}

static int

mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)

{

unsigned int lock_id;

int ret;

i_assert(index->fd == -1);

if (lock_id_r != NULL)

*lock_id_r = 0;

if (MAIL_INDEX_IS_IN_MEMORY(index))

return 0;