c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "lib.h"

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "array.h"

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "buffer.h"

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "hash.h"

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen#include "mmap-util.h"

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "nfs-workarounds.h"

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen#include "read-full.h"

06fda713b84e857dbc3e80f401a54085c9b0ed16Timo Sirainen#include "write-full.h"

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen#include "mail-index-private.h"

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen#include "mail-index-sync-private.h"

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-transaction-log.h"

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-cache.h"

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen#include <stdio.h>

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include <stddef.h>

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include <time.h>

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include <sys/stat.h>

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenunsigned int mail_index_module_id = 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenstatic void mail_index_create_in_memory(struct mail_index *index,

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen const struct mail_index_header *hdr);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen{

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen struct mail_index *index;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen index = i_new(struct mail_index, 1);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen index->dir = i_strdup(dir);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->prefix = i_strdup(prefix);

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen index->fd = -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->extension_pool =

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pool_alloconly_create(MEMPOOL_GROWING"index extension", 1024);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen p_array_init(&index->extensions, index->extension_pool, 5);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_array_init(&index->sync_lost_handlers, 4);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_create(&index->mail_index_module_contexts, default_pool,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen sizeof(void *), I_MIN(5, mail_index_module_id));

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen index->mode = 0600;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen index->gid = (gid_t)-1;

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->keywords_ext_id =

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_ext_register(index, "keywords", 128, 2, 1);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->keywords_pool = pool_alloconly_create("keywords", 512);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_array_init(&index->keywords, 16);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->keywords_hash =

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hash_create(default_pool, index->keywords_pool, 0,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen strcase_hash, (hash_cmp_callback_t *)strcasecmp);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return index;

4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid mail_index_free(struct mail_index **_index)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen struct mail_index *index = *_index;

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen *_index = NULL;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_close(index);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hash_destroy(index->keywords_hash);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pool_unref(index->extension_pool);

4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen pool_unref(index->keywords_pool);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_free(&index->sync_lost_handlers);

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen array_free(&index->keywords);

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen array_free(&index->mail_index_module_contexts);

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_free(index->error);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_free(index->dir);

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen i_free(index->prefix);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_free(index);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

35ef661bd85c64834e3e34eeeb3c393b81108760Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid mail_index_set_permissions(struct mail_index *index,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mode_t mode, gid_t gid)

35ef661bd85c64834e3e34eeeb3c393b81108760Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->mode = mode & 0666;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->gid = gid;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uint32_t default_hdr_size,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uint16_t default_record_size,

c39c3d8089fbdd8eb34646c25167aa4551064cf4Timo Sirainen uint16_t default_record_align)

c39c3d8089fbdd8eb34646c25167aa4551064cf4Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const struct mail_index_registered_ext *extensions;

bd63b5b860658b01b1f46f26d406e1e4a9dc019aTimo Sirainen struct mail_index_registered_ext rext;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int i, ext_count;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen extensions = array_get(&index->extensions, &ext_count);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* see if it's already there */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen for (i = 0; i < ext_count; i++) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (strcmp(extensions[i].name, name) == 0)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return i;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen memset(&rext, 0, sizeof(rext));

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext.name = p_strdup(index->extension_pool, name);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext.index_idx = ext_count;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext.hdr_size = default_hdr_size;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext.record_size = default_record_size;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext.record_align = default_record_align;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_append(&index->extensions, &rext, 1);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return ext_count;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid mail_index_register_expunge_handler(struct mail_index *index,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uint32_t ext_id, bool call_always,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_expunge_handler_t *cb,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen void *context)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_registered_ext *rext;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen i_assert(rext->expunge_handler == NULL || rext->expunge_handler == cb);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext->expunge_handler = cb;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext->expunge_context = context;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rext->expunge_handler_call_always = call_always;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenvoid mail_index_unregister_expunge_handler(struct mail_index *index,

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen uint32_t ext_id)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_registered_ext *rext;

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen i_assert(rext->expunge_handler != NULL);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen rext->expunge_handler = NULL;

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen}

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainenvoid mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen mail_index_sync_handler_t *cb,

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen enum mail_index_sync_handler_type type)

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen{

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen struct mail_index_registered_ext *rext;

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen i_assert(rext->sync_handler.callback == NULL);

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen rext->sync_handler.callback = cb;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen rext->sync_handler.type = type;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid mail_index_unregister_sync_handler(struct mail_index *index,

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen uint32_t ext_id)

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen{

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen struct mail_index_registered_ext *rext;

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen i_assert(rext->sync_handler.callback != NULL);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen rext->sync_handler.callback = NULL;

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen rext->sync_handler.type = 0;

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen}

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainenvoid mail_index_register_sync_lost_handler(struct mail_index *index,

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen mail_index_sync_lost_handler_t *cb)

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen{

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen array_append(&index->sync_lost_handlers, &cb, 1);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen}

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainenvoid mail_index_unregister_sync_lost_handler(struct mail_index *index,

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen mail_index_sync_lost_handler_t *cb)

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen{

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen mail_index_sync_lost_handler_t *const *handlers;

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen unsigned int i, count;

5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen handlers = array_get(&index->sync_lost_handlers, &count);

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen for (i = 0; i < count; i++) {

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen if (handlers[i] == cb) {

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen array_delete(&index->sync_lost_handlers, i, 1);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen break;

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen }

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen }

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen

2f19f8ff906a0017b906763e0f7675d49ab0e58fTimo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,

06fda713b84e857dbc3e80f401a54085c9b0ed16Timo Sirainen unsigned int initial_count)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen#define EXTENSION_NAME_APPROX_LEN 20

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#define EXT_GLOBAL_ALLOC_SIZE \

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen#define EXT_PER_ALLOC_SIZE \

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (EXTENSION_NAME_APPROX_LEN + \

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen sizeof(struct mail_index_ext) + sizeof(uint32_t))

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen size_t size;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map->extension_pool == NULL) {

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen size = EXT_GLOBAL_ALLOC_SIZE +

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen initial_count * EXT_PER_ALLOC_SIZE;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->extension_pool =

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pool_alloconly_create("map extensions",

3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody nearest_power(size));

06fda713b84e857dbc3e80f401a54085c9b0ed16Timo Sirainen } else {

06fda713b84e857dbc3e80f401a54085c9b0ed16Timo Sirainen p_clear(map->extension_pool);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* try to use the existing pool's size for initial_count so

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen size = p_get_max_easy_alloc_size(map->extension_pool);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen EXT_PER_ALLOC_SIZE;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen p_array_init(&map->extensions, map->extension_pool, initial_count);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen p_array_init(&map->ext_id_map, map->extension_pool, initial_count);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenuint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen{

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen const struct mail_index_ext *extensions;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen unsigned int i, size;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen if (!array_is_created(&map->extensions))

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen return (uint32_t)-1;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen extensions = array_get(&map->extensions, &size);

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen for (i = 0; i < size; i++) {

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen if (strcmp(extensions[i].name, name) == 0)

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen return i;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen }

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen return (uint32_t)-1;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen}

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenmail_index_map_register_ext(struct mail_index *index,

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen struct mail_index_map *map, const char *name,

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen uint32_t hdr_offset, uint32_t hdr_size,

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen uint32_t record_offset, uint32_t record_size,

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen uint32_t record_align, uint32_t reset_id)

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen{

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen struct mail_index_ext *ext;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen uint32_t idx, empty_idx = (uint32_t)-1;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen if (!array_is_created(&map->extensions)) {

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen mail_index_map_init_extbufs(map, 5);

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen idx = 0;

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen } else {

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen idx = array_count(&map->extensions);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen }

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen ext = array_append_space(&map->extensions);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen ext->name = p_strdup(map->extension_pool, name);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen ext->hdr_offset = hdr_offset;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen ext->hdr_size = hdr_size;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen ext->record_offset = record_offset;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen ext->record_size = record_size;

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen ext->record_align = record_align;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen ext->reset_id = reset_id;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen ext->index_idx = mail_index_ext_register(index, name, hdr_size,

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen record_size, record_align);

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen /* Update index ext_id -> map ext_id mapping. Fill non-used

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen while (array_count(&map->ext_id_map) < ext->index_idx)

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen array_append(&map->ext_id_map, &empty_idx, 1);

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen array_idx_set(&map->ext_id_map, ext->index_idx, &idx);

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen return idx;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic bool size_check(size_t *size_left, size_t size)

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen{

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen if (size > *size_left)

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen return FALSE;

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen *size_left -= size;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen return TRUE;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic size_t get_align(size_t name_len)

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen{

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen size_t size = sizeof(struct mail_index_ext_header) + name_len;

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen}

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainenstatic int mail_index_parse_extensions(struct mail_index *index,

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen struct mail_index_map *map)

e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen{

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen const struct mail_index_ext_header *ext_hdr;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen unsigned int i, old_count;

31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen const char *name;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen uint32_t ext_id, offset, name_offset;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen size_t size_left;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen /* extension headers always start from 64bit offsets, so if base header

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen if (offset >= map->hdr.header_size && map->extension_pool == NULL) {

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen /* nothing to do, skip allocatations and all */

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen return 1;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen }

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen old_count = array_count(&index->extensions);

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen mail_index_map_init_extbufs(map, old_count + 5);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen ext_id = (uint32_t)-1;

57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen for (i = 0; i < old_count; i++)

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen array_append(&map->ext_id_map, &ext_id, 1);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen while (offset < map->hdr.header_size) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* Extension header contains:

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen size_left = map->hdr.header_size - offset;

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen if (!size_check(&size_left, sizeof(*ext_hdr)) ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen !size_check(&size_left, ext_hdr->name_size) ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen !size_check(&size_left, get_align(ext_hdr->name_size)) ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen !size_check(&size_left, ext_hdr->hdr_size)) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "Header extension goes outside header",

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->filepath);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen offset += sizeof(*ext_hdr);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen name_offset = offset;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen offset += ext_hdr->name_size + get_align(ext_hdr->name_size);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen t_push();

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen ext_hdr->name_size);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "Duplicate header extension %s",

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->filepath, name);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen t_pop();

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map->hdr.record_size <

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ext_hdr->record_offset + ext_hdr->record_size) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "Record field %s points outside record size "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "(%u < %u+%u)", index->filepath, name,

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen map->hdr.record_size,

b3070aca2aa0dbdcc7f30e6e2bb1f0455127a343Timo Sirainen ext_hdr->record_offset, ext_hdr->record_size);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen t_pop();

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen

7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||

7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen (map->hdr.record_size % ext_hdr->record_align) != 0) {

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen "Record field %s alignmentation %u not used",

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen index->filepath, name, ext_hdr->record_align);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen t_pop();

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_map_register_ext(index, map, name,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen offset, ext_hdr->hdr_size,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ext_hdr->record_offset,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ext_hdr->record_size,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ext_hdr->record_align,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ext_hdr->reset_id);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen t_pop();

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenbool mail_index_keyword_lookup(struct mail_index *index,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *keyword, bool autocreate,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int *idx_r)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen char *keyword_dup;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen void *value;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* keywords_hash keeps a name => index mapping of keywords.

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) {

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen *idx_r = POINTER_CAST_TO(value, unsigned int);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return TRUE;

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!autocreate) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen *idx_r = (unsigned int)-1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return FALSE;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen }

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen *idx_r = array_count(&index->keywords);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r));

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen array_append(&index->keywords, &keyword, 1);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen return TRUE;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen}

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenint mail_index_map_parse_keywords(struct mail_index *index,

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen struct mail_index_map *map)

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen{

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen const struct mail_index_ext *ext;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen const struct mail_index_keyword_header *kw_hdr;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen const struct mail_index_keyword_header_rec *kw_rec;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen const char *name;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen unsigned int i, name_area_end_offset, old_count;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen uint32_t ext_id;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen map->keywords_read = TRUE;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen ext_id = mail_index_map_lookup_ext(map, "keywords");

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen if (ext_id == (uint32_t)-1) {

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen if (array_is_created(&map->keyword_idx_map))

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen array_clear(&map->keyword_idx_map);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen return 0;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen }

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen ext = array_idx(&map->extensions, ext_id);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen /* Extension header contains:

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen i_assert(ext->hdr_offset < map->hdr.header_size);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen kw_rec = (const void *)(kw_hdr + 1);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen name = (const char *)(kw_rec + kw_hdr->keywords_count);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen old_count = !array_is_created(&map->keyword_idx_map) ? 0 :

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen array_count(&map->keyword_idx_map);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* Keywords can only be added into same mapping. Removing requires a

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (kw_hdr->keywords_count == old_count) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* nothing changed */

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen /* make sure the header is valid */

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen if (kw_hdr->keywords_count < old_count) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen "Keywords removed unexpectedly",

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen index->filepath);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen return -1;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen }

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "keywords_count larger than header size",

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->filepath);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen for (i = 0; i < kw_hdr->keywords_count; i++) {

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen if (kw_rec[i].name_offset > name_area_end_offset) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

cfb351c1d28fab5a461f6506471c6b6eb26c0bdaTimo Sirainen "name_offset points outside allocated header",

cfb351c1d28fab5a461f6506471c6b6eb26c0bdaTimo Sirainen index->filepath);

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen return -1;

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (name[name_area_end_offset-1] != '\0') {

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen "Keyword header doesn't end with NUL",

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen index->filepath);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen /* create file -> index mapping */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!array_is_created(&map->keyword_idx_map))

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen#ifdef DEBUG

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen /* Check that existing headers are still the same. It's behind DEBUG

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen for (i = 0; i < array_count(&map->keyword_idx_map); i++) {

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen const char *keyword = name + kw_rec[i].name_offset;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen const unsigned int *old_idx;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen unsigned int idx;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen old_idx = array_idx(&map->keyword_idx_map, i);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) ||

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen idx != *old_idx) {

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen "Keywords changed unexpectedly",

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen index->filepath);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen return -1;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen }

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen }

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen#endif

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen /* Register the newly seen keywords */

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen i = array_count(&map->keyword_idx_map);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen for (; i < kw_hdr->keywords_count; i++) {

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen const char *keyword = name + kw_rec[i].name_offset;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen unsigned int idx;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_append(&map->keyword_idx_map, &idx, 1);

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen }

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenconst ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* Make sure all the keywords are in index->keywords. It's quick to do

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen (void)mail_index_map_parse_keywords(index, index->map);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return &index->keywords;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic bool mail_index_check_header_compat(const struct mail_index_header *hdr)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen enum mail_index_header_compat_flags compat_flags = 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#ifndef WORDS_BIGENDIAN

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#endif

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen

c8c4bbf6b1415e9d0845bc8f1cd6d19b76ab0392Timo Sirainen if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* major version change - handle silently(?) */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return FALSE;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen if (hdr->compat_flags != compat_flags) {

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen /* architecture change - handle silently(?) */

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen return FALSE;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* we've already complained about it */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return FALSE;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return TRUE;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mail_index_check_header(struct mail_index *index,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_map *map)

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen{

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen const struct mail_index_header *hdr = &map->hdr;

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!mail_index_check_header_compat(hdr))

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* following some extra checks that only take a bit of CPU */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (hdr->uid_validity == 0 && hdr->next_uid != 1) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "uid_validity = 0, next_uid = %u",

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->filepath, hdr->next_uid);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen if (hdr->record_size < sizeof(struct mail_index_record)) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "record_size too small: %u < %"PRIuSIZE_T,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->filepath, hdr->record_size,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen sizeof(struct mail_index_record));

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (hdr->next_uid == 0)

414d4ee117b45834e6e02f32cda9c9472dd89e1dTimo Sirainen return 0;

414d4ee117b45834e6e02f32cda9c9472dd89e1dTimo Sirainen

414d4ee117b45834e6e02f32cda9c9472dd89e1dTimo Sirainen if (hdr->recent_messages_count > hdr->messages_count ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr->seen_messages_count > hdr->messages_count ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr->deleted_messages_count > hdr->messages_count)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (hdr->first_recent_uid_lowwater > hdr->next_uid ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr->first_unseen_uid_lowwater > hdr->next_uid ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr->first_deleted_uid_lowwater > hdr->next_uid)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (map->records_count > 0) {

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* last message's UID must be smaller than next_uid.

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen const struct mail_index_record *rec;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen rec = MAIL_INDEX_MAP_IDX(map, map->records_count-1);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (rec->uid == 0 || rec->uid >= hdr->next_uid)

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen return mail_index_parse_extensions(index, map);

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void mail_index_map_clear(struct mail_index *index,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_map *map)

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen{

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen if (map->buffer != NULL) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_assert(map->mmap_base == NULL);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen buffer_free(map->buffer);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->buffer = NULL;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen } else if (map->mmap_base != NULL) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_assert(map->buffer == NULL);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (munmap(map->mmap_base, map->mmap_size) < 0)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_syscall_error(index, "munmap()");

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->mmap_base = NULL;

7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map->refcount > 0) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen map->mmap_size = 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->mmap_used_size = 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->records = NULL;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->records_count = 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid mail_index_unmap(struct mail_index *index, struct mail_index_map **_map)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_map *map = *_map;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen *_map = NULL;

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen if (--map->refcount > 0)

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen return;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_assert(map->refcount == 0);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_map_clear(index, map);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen if (map->extension_pool != NULL)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pool_unref(map->extension_pool);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (array_is_created(&map->keyword_idx_map))

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_free(&map->keyword_idx_map);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen buffer_free(map->hdr_copy_buf);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen i_free(map);

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen}

9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void mail_index_map_copy_hdr(struct mail_index_map *map,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const struct mail_index_header *hdr)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (hdr->base_header_size < sizeof(map->hdr)) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* header smaller than ours, make a copy so our newer headers

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen memset(&map->hdr, 0, sizeof(map->hdr));

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen memcpy(&map->hdr, hdr, hdr->base_header_size);

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen } else {

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen map->hdr = *hdr;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen{

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen const struct mail_index_header *hdr;

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen unsigned int records_count;

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen

a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen i_assert(!map->write_to_disk);

7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen

12e32734b3fe762c647c1ae5552185517a988cf8Timo Sirainen if (map->buffer != NULL) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* we had temporarily used a buffer, eg. for updating index */

7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen buffer_free(map->buffer);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->buffer = NULL;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen map->mmap_base = index->readonly ?

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen mmap_ro_file(index->fd, &map->mmap_size) :

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen mmap_rw_file(index->fd, &map->mmap_size);

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen if (map->mmap_base == MAP_FAILED) {

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen map->mmap_base = NULL;

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen mail_index_set_syscall_error(index, "mmap()");

e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen return -1;

e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen }

e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen

e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen hdr = map->mmap_base;

2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen if (map->mmap_size >

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen offsetof(struct mail_index_header, major_version) &&

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {

2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen /* major version change - handle silently */

2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen return 0;

af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen }

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "File too small (%"PRIuSIZE_T")",

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen index->filepath, map->mmap_size);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!mail_index_check_header_compat(hdr)) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* Can't use this file */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen }

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen map->mmap_used_size = hdr->header_size +

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen hdr->messages_count * hdr->record_size;

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (map->mmap_used_size > map->mmap_size) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen records_count = (map->mmap_size - hdr->header_size) /

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen hdr->record_size;

0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen "messages_count too large (%u > %u)",

0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen index->filepath, hdr->messages_count,

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen records_count);

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen return 0;

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen mail_index_map_copy_hdr(map, hdr);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->hdr_base = map->mmap_base;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->records_count = map->hdr.messages_count;

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen return 1;

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen}

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int mail_index_read_header(struct mail_index *index,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen void *buf, size_t buf_size, size_t *pos_r)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen size_t pos;

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen int ret;

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen memset(buf, 0, sizeof(struct mail_index_header));

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* try to read the whole header, but it's not necessarily an error to

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen pos = 0;

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen do {

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen ret = pread(index->fd, PTR_OFFSET(buf, pos),

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen buf_size - pos, pos);

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (ret > 0)

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen pos += ret;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen } while (ret > 0 && pos < sizeof(struct mail_index_header));

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen *pos_r = pos;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return ret;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainenstatic int

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainenmail_index_read_map(struct mail_index *index, struct mail_index_map *map,

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen bool *retry_r, bool try_retry)

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen{

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen const struct mail_index_header *hdr;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen struct stat st;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen unsigned char buf[512];

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen void *data = NULL;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen ssize_t ret;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen size_t pos, records_size;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen unsigned int records_count;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen i_assert(map->mmap_base == NULL);

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen *retry_r = FALSE;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen ret = mail_index_read_header(index, buf, sizeof(buf), &pos);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr = (const struct mail_index_header *)buf;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* major version change - handle silently */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (fstat(index->fd, &st) < 0) {

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen mail_index_set_syscall_error(index, "fstat()");

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen return -1;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (ret > 0 || pos >= hdr->base_header_size)) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!mail_index_check_header_compat(hdr)) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* Can't use this file */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen hdr->header_size < hdr->base_header_size) {

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "Corrupted header sizes (base %u, full %u)",

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->filepath, hdr->base_header_size,

b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen hdr->header_size);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (hdr->header_size > (uoff_t)st.st_size) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "Corrupted header size (%u > %"PRIuUOFF_T")",

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->filepath, hdr->header_size,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen st.st_size);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (pos > hdr->header_size)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pos = hdr->header_size;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* place the base header into memory. */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen buffer_reset(map->hdr_copy_buf);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen buffer_append(map->hdr_copy_buf, buf, pos);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (pos != hdr->header_size) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* @UNSAFE: read the rest of the header into memory */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen data = buffer_append_space_unsafe(map->hdr_copy_buf,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr->header_size -

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pos);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = pread_full(index->fd, data,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr->header_size - pos, pos);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (ret > 0) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* header read, read the records now. */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen records_size = (size_t)hdr->messages_count * hdr->record_size;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if ((uoff_t)st.st_size - hdr->header_size < records_size ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (hdr->record_size != 0 &&

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen records_size / hdr->record_size != hdr->messages_count)) {

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen records_count = (st.st_size - hdr->header_size) /

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen hdr->record_size;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen "messages_count too large (%u > %u)",

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen index->filepath, hdr->messages_count,

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen records_count);

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen return 0;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen }

d430698b6d9d47ff9136ee137cc58a1c657baeddTimo Sirainen

d430698b6d9d47ff9136ee137cc58a1c657baeddTimo Sirainen if (map->buffer == NULL) {

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen map->buffer = buffer_create_dynamic(default_pool,

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen records_size);

d430698b6d9d47ff9136ee137cc58a1c657baeddTimo Sirainen }

d430698b6d9d47ff9136ee137cc58a1c657baeddTimo Sirainen

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen /* @UNSAFE */

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen buffer_set_used_size(map->buffer, 0);

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen data = buffer_append_space_unsafe(map->buffer, records_size);

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen ret = pread_full(index->fd, data, records_size,

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen hdr->header_size);

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen }

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen if (ret < 0) {

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen if (errno == ESTALE && try_retry) {

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen /* a new index file was renamed over this one. */

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen *retry_r = TRUE;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_syscall_error(index, "pread_full()");

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (ret == 0) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_error(index,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "Corrupted index file %s: File too small",

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->filepath);

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen return 0;

5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->records = data;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->records_count = hdr->messages_count;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_map_copy_hdr(map, hdr);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->hdr_base = map->hdr_copy_buf->data;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->sync_log_file_seq = hdr->log_file_seq;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->sync_log_file_offset = hdr->log_file_int_offset;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}

c6afd726060aae56b6622c6c52aec10231c4bf1cTimo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenbool mail_index_is_ext_synced(struct mail_transaction_log_view *log_view,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_map *map)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uint32_t prev_seq;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uoff_t prev_offset;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen &prev_offset);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return prev_seq < map->hdr.log_file_seq ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (prev_seq == map->hdr.log_file_seq &&

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen prev_offset < map->hdr.log_file_ext_offset);

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen}

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mail_index_sync_from_transactions(struct mail_index *index,

ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen struct mail_index_map **map,

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen bool sync_to_index)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const struct mail_index_header *map_hdr = &(*map)->hdr;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_view *view;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_transaction_log_view *log_view;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_index_header hdr;

1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen const struct mail_transaction_header *thdr;

71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang const void *tdata;

71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang uint32_t prev_seq, max_seq;

2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen uoff_t prev_offset, max_offset;

71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang size_t pos;

71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang int ret;

b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen bool skipped, check_ext_offsets, broken;

71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang

71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang if (sync_to_index) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* read the real log position where we are supposed to be

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = mail_index_read_header(index, &hdr, sizeof(hdr), &pos);

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (ret < 0 && errno != ESTALE) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_index_set_syscall_error(index, "pread()");

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (pos < MAIL_INDEX_HEADER_MIN_SIZE)

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map_hdr->log_file_seq == hdr.log_file_seq &&

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map_hdr->log_file_int_offset == hdr.log_file_int_offset) {

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* nothing to do */

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 1;

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map_hdr->log_file_seq > hdr.log_file_seq ||

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (map_hdr->log_file_seq == hdr.log_file_seq &&

c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map_hdr->log_file_int_offset > hdr.log_file_int_offset)) {

/* we went too far, have to re-read the file */

return 0;

}

if (map_hdr->log_file_ext_offset !=

map_hdr->log_file_int_offset ||

hdr.log_file_ext_offset != hdr.log_file_int_offset) {

/* too much trouble to get this right. */

return 0;

}

max_seq = hdr.log_file_seq;

max_offset = hdr.log_file_int_offset;

} else {

/* sync everything there is */

max_seq = (uint32_t)-1;

max_offset = (uoff_t)-1;

}

log_view = mail_transaction_log_view_open(index->log);

if (mail_transaction_log_view_set(log_view,

map_hdr->log_file_seq,

map_hdr->log_file_int_offset,

max_seq, max_offset,

MAIL_TRANSACTION_TYPE_MASK) <= 0) {

/* can't use it. sync by re-reading index. */

mail_transaction_log_view_close(&log_view);

return 0;

}

index->map = *map;

view = mail_index_view_open(index);

mail_index_sync_map_init(&sync_map_ctx, view,

MAIL_INDEX_SYNC_HANDLER_HEAD);

check_ext_offsets = TRUE; broken = FALSE;

while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,

&skipped)) > 0) {

if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0 &&

check_ext_offsets) {

if (mail_index_is_ext_synced(log_view, index->map))

continue;

check_ext_offsets = FALSE;

}

if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {

ret = 0;

broken = TRUE;

break;

}

}

if (ret == 0 && !broken)

ret = 1;

mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,

&prev_offset);

i_assert(prev_seq <= max_seq &&

(prev_seq != max_seq || prev_offset <= max_offset));

index->map->hdr.log_file_seq = prev_seq;

index->map->hdr.log_file_int_offset =

index->map->hdr.log_file_ext_offset = prev_offset;

mail_index_sync_map_deinit(&sync_map_ctx);

mail_index_view_close(&view);

mail_transaction_log_view_close(&log_view);

*map = index->map;

index->map = NULL;

if (sync_to_index && ret > 0) {

/* make sure we did everything right. note that although the

i_assert(hdr.messages_count == (*map)->hdr.messages_count);

i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq);

i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset);

i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset);

}

return ret;

}

static int mail_index_read_map_with_retry(struct mail_index *index,

struct mail_index_map **map,

bool sync_to_index)

{

mail_index_sync_lost_handler_t *const *handlers;

unsigned int i, count;

int ret;

bool retry;

if (index->log_locked) {

/* we're most likely syncing the index and we really don't

sync_to_index = TRUE;

}

if ((*map)->hdr.indexid != 0 && index->log != NULL) {

/* we're not creating the index, or opening transaction log.

index->sync_update = TRUE;

ret = mail_index_sync_from_transactions(index, map,

sync_to_index);

index->sync_update = FALSE;

if (ret != 0)

return ret;

/* transaction log lost/broken, fallback to re-reading it */

}

/* notify all "sync lost" handlers */

handlers = array_get(&index->sync_lost_handlers, &count);

for (i = 0; i < count; i++)

(*handlers[i])(index);

for (i = 0;; i++) {

ret = mail_index_read_map(index, *map, &retry,

i < MAIL_INDEX_ESTALE_RETRY_COUNT);

if (ret != 0 || !retry)

return ret;

/* ESTALE - reopen index file */

if (close(index->fd) < 0)

mail_index_set_syscall_error(index, "close()");

index->fd = -1;

ret = mail_index_try_open_only(index);

if (ret <= 0) {

if (ret == 0) {

/* the file was lost */

errno = ENOENT;

mail_index_set_syscall_error(index, "open()");

}

return -1;

}

}

}

static int mail_index_map_try_existing(struct mail_index *index)

{

struct mail_index_map *map = index->map;

const struct mail_index_header *hdr;

size_t used_size;

int ret;

if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))

return 0;

hdr = map->mmap_base;

/* always check corrupted-flag to avoid errors later */

if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)

return -1;

used_size = hdr->header_size + hdr->messages_count * hdr->record_size;

if (map->mmap_size >= used_size && map->hdr_base == hdr) {

map->records_count = hdr->messages_count;

mail_index_map_copy_hdr(map, hdr);

/* make sure the header is still valid. it also re-parses

ret = mail_index_check_header(index, map);

if (ret > 0)

return 1;

/* broken. fallback to re-mmaping which will catch it */

}

return 0;

}

int mail_index_map(struct mail_index *index, bool force)

{

struct mail_index_map *map;

int ret;

i_assert(!index->mapping);

i_assert(index->map == NULL || index->map->refcount > 0);

i_assert(index->lock_type != F_UNLCK);

if (MAIL_INDEX_IS_IN_MEMORY(index)) {

if (index->map == NULL)

mail_index_create_in_memory(index, NULL);

return 1;

}

index->mapping = TRUE;

if (!force && index->map != NULL) {

i_assert(index->hdr != NULL);

ret = mail_index_map_try_existing(index);

if (ret != 0) {

index->mapping = FALSE;

return ret;

}

if (index->lock_type == F_WRLCK) {

/* we're syncing, don't break the mapping */

index->mapping = FALSE;

return 1;

}

}

if (index->map != NULL && index->map->refcount > 1) {

/* this map is already used by some views and they may have

if (!index->mmap_disable) {

map = NULL;

} else {

/* create a copy of the mapping instead so we don't

map = mail_index_map_clone(index->map,

index->map->hdr.record_size);

}

index->map->refcount--;

index->map = NULL;

} else {

map = index->map;

}

if (map == NULL) {

map = i_new(struct mail_index_map, 1);

map->refcount = 1;

map->hdr_copy_buf =

buffer_create_dynamic(default_pool, sizeof(map->hdr));

} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {

i_assert(!map->write_to_disk);

} else if (map->mmap_base != NULL) {

i_assert(map->buffer == NULL);

if (munmap(map->mmap_base, map->mmap_size) < 0)

mail_index_set_syscall_error(index, "munmap()");

map->mmap_base = NULL;

}

index->hdr = NULL;

index->map = NULL;

if (!index->mmap_disable)

ret = mail_index_mmap(index, map);

else

ret = mail_index_read_map_with_retry(index, &map, force);

i_assert(index->map == NULL);

if (ret > 0) {

ret = mail_index_check_header(index, map);

if (ret < 0)

ret = 0;

else if (ret == 0) {

index->fsck = TRUE;

ret = 1;

}

}

if (ret <= 0) {

mail_index_map_clear(index, map);

mail_index_unmap(index, &map);

index->mapping = FALSE;

return ret;

}

index->hdr = &map->hdr;

index->map = map;

i_assert(map->hdr.messages_count == map->records_count);

index->mapping = FALSE;

return 1;

}

int mail_index_get_latest_header(struct mail_index *index,

struct mail_index_header *hdr_r)

{

size_t pos;

unsigned int i;

int ret;

if (!index->mmap_disable) {

ret = mail_index_map(index, FALSE);

if (ret > 0)

*hdr_r = *index->hdr;

else

memset(hdr_r, 0, sizeof(*hdr_r));

return ret;

}

for (i = 0;; i++) {

ret = mail_index_read_header(index, hdr_r, sizeof(*hdr_r),

&pos);

if (ret <= 0 || errno != ESTALE ||

i == MAIL_INDEX_ESTALE_RETRY_COUNT)

break;

/* ESTALE - reopen index file */

if (close(index->fd) < 0)

mail_index_set_syscall_error(index, "close()");

index->fd = -1;

ret = mail_index_try_open_only(index);

if (ret <= 0) {

if (ret == 0) {

/* the file was lost */

errno = ENOENT;

mail_index_set_syscall_error(index, "open()");

}

return -1;

}

}

if (ret < 0)

mail_index_set_syscall_error(index, "pread_full()");

return ret;

}

struct mail_index_map *

mail_index_map_clone(const struct mail_index_map *map, uint32_t new_record_size)

{

struct mail_index_map *mem_map;

struct mail_index_header *hdr;

struct mail_index_ext *extensions;

void *src, *dest;

size_t size, copy_size;

unsigned int i, count;

size = map->records_count * new_record_size;

mem_map = i_new(struct mail_index_map, 1);

mem_map->refcount = 1;

mem_map->buffer = buffer_create_dynamic(default_pool, size);

if (map->hdr.record_size == new_record_size)

buffer_append(mem_map->buffer, map->records, size);

else {

copy_size = I_MIN(map->hdr.record_size, new_record_size);

src = map->records;

for (i = 0; i < map->records_count; i++) {

dest = buffer_append_space_unsafe(mem_map->buffer,

new_record_size);

memcpy(dest, src, copy_size);

src = PTR_OFFSET(src, map->hdr.record_size);

}

}

mem_map->records = buffer_get_modifiable_data(mem_map->buffer, NULL);

mem_map->records_count = map->records_count;

mem_map->hdr_copy_buf =

buffer_create_dynamic(default_pool, map->hdr.header_size);

if (map->hdr.base_header_size < sizeof(*hdr))

buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr));

buffer_write(mem_map->hdr_copy_buf, 0,

&map->hdr, map->hdr.base_header_size);

buffer_append(mem_map->hdr_copy_buf,

CONST_PTR_OFFSET(map->hdr_base,

map->hdr.base_header_size),

map->hdr.header_size - map->hdr.base_header_size);

hdr = buffer_get_modifiable_data(mem_map->hdr_copy_buf, NULL);