src/lib-index/mail-index.c

	mail-index.c revision ef5fb27361cc5e15766e85e28355750ff04b13c9
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen#include "lib.h"
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen#include "ioloop.h"
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen#include "array.h"
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen#include "buffer.h"
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen#include "hash.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "str-sanitize.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "mmap-util.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "nfs-workarounds.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "read-full.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "write-full.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "mail-index-private.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "mail-index-view-private.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "mail-index-sync-private.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "mail-index-modseq.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "mail-transaction-log.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include "mail-cache.h"
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include <stdio.h>
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include <stddef.h>
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include <time.h>
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen#include <sys/stat.h>
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainenstruct mail_index_module_register mail_index_module_register = { 0 };
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen{
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    struct mail_index *index;
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    index = i_new(struct mail_index, 1);
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    index->dir = i_strdup(dir);
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    index->prefix = i_strdup(prefix);
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    index->fd = -1;
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    index->extension_pool =
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen        pool_alloconly_create(MEMPOOL_GROWING"index extension", 1024);
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    p_array_init(&index->extensions, index->extension_pool, 5);
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    i_array_init(&index->sync_lost_handlers, 4);
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen    i_array_init(&index->module_contexts,
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen             I_MIN(5, mail_index_module_register.id));
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen    index->mode = 0600;
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen    index->gid = (gid_t)-1;
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen    index->keywords_ext_id =
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen        mail_index_ext_register(index, MAIL_INDEX_EXT_KEYWORDS,
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen                    128, 2, 1);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen    index->keywords_pool = pool_alloconly_create("keywords", 512);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen    i_array_init(&index->keywords, 16);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen    index->keywords_hash =
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen        hash_table_create(default_pool, index->keywords_pool, 0,
c8bac7666cdc780a3390110e420350fffb62b909Timo Sirainen                  strcase_hash, (hash_cmp_callback_t *)strcasecmp);
c8bac7666cdc780a3390110e420350fffb62b909Timo Sirainen    index->log = mail_transaction_log_alloc(index);
c8bac7666cdc780a3390110e420350fffb62b909Timo Sirainen    mail_index_modseq_init(index);
c8bac7666cdc780a3390110e420350fffb62b909Timo Sirainen    return index;
c8bac7666cdc780a3390110e420350fffb62b909Timo Sirainen}
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainenvoid mail_index_free(struct mail_index **_index)
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen{
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    struct mail_index *index = *_index;
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    *_index = NULL;
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    mail_index_close(index);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    mail_transaction_log_free(&index->log);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    hash_table_destroy(&index->keywords_hash);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    pool_unref(&index->extension_pool);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    pool_unref(&index->keywords_pool);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    array_free(&index->sync_lost_handlers);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    array_free(&index->keywords);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    array_free(&index->module_contexts);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    i_free(index->error);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    i_free(index->dir);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    i_free(index->prefix);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    i_free(index);
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen}
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainenvoid mail_index_set_fsync_types(struct mail_index *index,
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen                enum mail_index_sync_type fsync_mask)
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen{
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen    index->fsync_mask = fsync_mask;
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen}
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenvoid mail_index_set_permissions(struct mail_index *index,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                mode_t mode, gid_t gid)
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody{
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody    index->mode = mode & 0666;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    index->gid = gid;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen}
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                 uint32_t default_hdr_size,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                 uint16_t default_record_size,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                 uint16_t default_record_align)
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen{
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    struct mail_index_registered_ext rext;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    uint32_t ext_id;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    if (*name == '\0' || strcmp(name, str_sanitize(name, -1)) != 0)
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        i_panic("mail_index_ext_register(%s): Invalid name", name);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    if (default_record_size != 0 && default_record_align == 0) {
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        i_panic("mail_index_ext_register(%s): "
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody            "Invalid record alignment", name);
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody    }
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    if (mail_index_ext_lookup(index, name, &ext_id))
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        return ext_id;
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody    memset(&rext, 0, sizeof(rext));
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody    rext.name = p_strdup(index->extension_pool, name);
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody    rext.index_idx = array_count(&index->extensions);
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody    rext.hdr_size = default_hdr_size;
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody    rext.record_size = default_record_size;
ba66ac5557ca97d8a6fe5d524056264a9f92243cPhil Carmody    rext.record_align = default_record_align;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    array_append(&index->extensions, &rext, 1);
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen    return rext.index_idx;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen}
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenbool mail_index_ext_lookup(struct mail_index *index, const char *name,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen               uint32_t *ext_id_r)
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen{
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        const struct mail_index_registered_ext *extensions;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    unsigned int i, count;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    extensions = array_get(&index->extensions, &count);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    for (i = 0; i < count; i++) {
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        if (strcmp(extensions[i].name, name) == 0) {
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen            *ext_id_r = i;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen            return TRUE;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        }
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    }
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    *ext_id_r = (uint32_t)-1;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    return FALSE;
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen}
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenvoid mail_index_register_expunge_handler(struct mail_index *index,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                     uint32_t ext_id, bool call_always,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                     mail_index_expunge_handler_t *cb,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                     void *context)
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen{
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    struct mail_index_registered_ext *rext;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    rext = array_idx_modifiable(&index->extensions, ext_id);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    i_assert(rext->expunge_handler == NULL || rext->expunge_handler == cb);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    rext->expunge_handler = cb;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    rext->expunge_context = context;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    rext->expunge_handler_call_always = call_always;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen}
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenvoid mail_index_unregister_expunge_handler(struct mail_index *index,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                       uint32_t ext_id)
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen{
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    struct mail_index_registered_ext *rext;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    rext = array_idx_modifiable(&index->extensions, ext_id);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    i_assert(rext->expunge_handler != NULL);
9f5db9cf866a978aab404772a0daf8794069bba5Phil Carmody
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    rext->expunge_handler = NULL;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen}
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenvoid mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen                      mail_index_sync_handler_t *cb,
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen                      enum mail_index_sync_handler_type type)
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen{
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    struct mail_index_registered_ext *rext;
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    rext = array_idx_modifiable(&index->extensions, ext_id);
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    i_assert(rext->sync_handler.callback == NULL);
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    rext->sync_handler.callback = cb;
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    rext->sync_handler.type = type;
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen}
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenvoid mail_index_unregister_sync_handler(struct mail_index *index,
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen                    uint32_t ext_id)
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen{
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    struct mail_index_registered_ext *rext;
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    rext = array_idx_modifiable(&index->extensions, ext_id);
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    i_assert(rext->sync_handler.callback != NULL);
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    rext->sync_handler.callback = NULL;
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    rext->sync_handler.type = 0;
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen}
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainenvoid mail_index_register_sync_lost_handler(struct mail_index *index,
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen                       mail_index_sync_lost_handler_t *cb)
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen{
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    array_append(&index->sync_lost_handlers, &cb, 1);
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen}
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainenvoid mail_index_unregister_sync_lost_handler(struct mail_index *index,
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen                         mail_index_sync_lost_handler_t *cb)
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen{
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    mail_index_sync_lost_handler_t *const *handlers;
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    unsigned int i, count;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    handlers = array_get(&index->sync_lost_handlers, &count);
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    for (i = 0; i < count; i++) {
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen        if (handlers[i] == cb) {
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen            array_delete(&index->sync_lost_handlers, i, 1);
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen            break;
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen        }
f7bd22354fb9311dfbf3e340a69c15d8b00facc1Timo Sirainen    }
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen}
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainenbool mail_index_keyword_lookup(struct mail_index *index,
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen                   const char *keyword, unsigned int *idx_r)
b7324e421e2132cbbf753e6fdbe675bbaecdf929Timo Sirainen{
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    void *value;
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    /* keywords_hash keeps a name => index mapping of keywords.
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen       Keywords are never removed from it, so the index values are valid
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen       for the lifetime of the mail_index. */
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    if (hash_table_lookup_full(index->keywords_hash, keyword,
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen                   NULL, &value)) {
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen        *idx_r = POINTER_CAST_TO(value, unsigned int);
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen        return TRUE;
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    }
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    *idx_r = (unsigned int)-1;
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    return FALSE;
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen}
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainenvoid mail_index_keyword_lookup_or_create(struct mail_index *index,
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen                     const char *keyword,
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen                     unsigned int *idx_r)
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen{
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    char *keyword_dup;
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    i_assert(*keyword != '\0');
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    if (mail_index_keyword_lookup(index, keyword, idx_r))
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen        return;
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    *idx_r = array_count(&index->keywords);
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    hash_table_insert(index->keywords_hash,
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen              keyword_dup, POINTER_CAST(*idx_r));
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    array_append(&index->keywords, &keyword, 1);
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen}
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainenconst ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index)
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen{
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    return &index->keywords;
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen}
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainenint mail_index_try_open_only(struct mail_index *index)
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen{
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    i_assert(index->fd == -1);
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen        /* Note that our caller must close index->fd by itself. */
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    if (index->readonly)
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen        errno = EACCES;
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    else {
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        index->fd = nfs_safe_open(index->filepath, O_RDWR);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        index->readonly = FALSE;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    }
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    if (index->fd == -1 && errno == EACCES) {
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        index->fd = open(index->filepath, O_RDONLY);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        index->readonly = TRUE;
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen    }
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    if (index->fd == -1) {
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        if (errno != ENOENT)
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen            return mail_index_set_syscall_error(index, "open()");
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        /* have to create it */
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        return 0;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    }
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    return 1;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen}
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenstatic int
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainenmail_index_try_open(struct mail_index *index)
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen{
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    int ret;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        i_assert(index->fd == -1);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    if (MAIL_INDEX_IS_IN_MEMORY(index))
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        return 0;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    i_assert(index->map == NULL || index->map->rec_map->lock_id == 0);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD);
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    if (ret == 0) {
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        /* it's corrupted - recreate it */
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        if (index->fd != -1) {
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen            if (close(index->fd) < 0)
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen                mail_index_set_syscall_error(index, "close()");
199566f5a171b2c43b9a5254634f6bf47b8baca8Timo Sirainen            index->fd = -1;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen        }
369764cc0856a4c5e9b4081de8c4e29e90f11ccdTimo Sirainen    }
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen    return ret;
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen}
48acc31adebfdd4e4945ee76e1f5259e4b1b6fffTimo Sirainen
int mail_index_create_tmp_file(struct mail_index *index, const char **path_r)
{
        mode_t old_mask;
    const char *path;
    int fd;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

    path = *path_r = t_strconcat(index->filepath, ".tmp", NULL);
    old_mask = umask(0);
    fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode);
    umask(old_mask);
    if (fd == -1)
        return mail_index_file_set_syscall_error(index, path, "open()");

    if (index->gid != (gid_t)-1 && fchown(fd, (uid_t)-1, index->gid) < 0) {
        mail_index_file_set_syscall_error(index, path, "fchown()");
        return -1;
    }

    return fd;
}

static int mail_index_open_files(struct mail_index *index,
                 enum mail_index_open_flags flags)
{
    int ret;
    bool created = FALSE;

    ret = mail_transaction_log_open(index->log);
    if (ret == 0) {
        if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
            return 0;

        /* if dovecot.index exists, read it first so that we can get
           the correct indexid and log sequence */
        (void)mail_index_try_open(index);

        if (index->indexid == 0) {
            /* Create a new indexid for us. If we're opening index
               into memory, index->map doesn't exist yet. */
            index->indexid = ioloop_time;
            index->initial_create = TRUE;
            if (index->map != NULL)
                index->map->hdr.indexid = index->indexid;
        }

        ret = mail_transaction_log_create(index->log, FALSE);
        index->initial_create = FALSE;
        created = TRUE;
    }
    if (ret >= 0) {
        ret = index->map != NULL ? 1 : mail_index_try_open(index);
        if (ret == 0) {
            /* corrupted */
            mail_transaction_log_close(index->log);
            ret = mail_transaction_log_create(index->log, TRUE);
            if (ret == 0) {
                if (index->map != NULL)
                    mail_index_unmap(&index->map);
                index->map = mail_index_map_alloc(index);
            }
        }
    }
    if (ret < 0) {
        /* open/create failed, fallback to in-memory indexes */
        if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
            return -1;

        if (mail_index_move_to_memory(index) < 0)
            return -1;
    }

    index->cache = created ? mail_cache_create(index) :
        mail_cache_open_or_create(index);
    return 1;
}

int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags,
            enum file_lock_method lock_method)
{
    int ret;

    if (index->opened) {
        if (index->map != NULL &&
            (index->map->hdr.flags &
             MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
            /* corrupted, reopen files */
                        mail_index_close(index);
        } else {
            i_assert(index->map != NULL);
            return 1;
        }
    }

    index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
        i_strdup("(in-memory index)") :
        i_strconcat(index->dir, "/", index->prefix, NULL);

    index->shared_lock_count = 0;
    index->excl_lock_count = 0;
    index->lock_type = F_UNLCK;
    index->lock_id_counter = 2;

    index->readonly = FALSE;
    index->nodiskspace = FALSE;
    index->index_lock_timeout = FALSE;
    index->log_locked = FALSE;
    index->mmap_disable = (flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
    index->use_excl_dotlocks =
        (flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
    index->fsync_disable =
        (flags & MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE) != 0;
    index->nfs_flush = (flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;
    index->readonly = (flags & MAIL_INDEX_OPEN_FLAG_READONLY) != 0;
    index->keep_backups = (flags & MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS) != 0;
    index->never_in_memory =
        (flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0;
    index->lock_method = lock_method;

    if (index->nfs_flush && index->fsync_disable)
        i_fatal("nfs flush requires fsync_disable=no");
    if (index->nfs_flush && !index->mmap_disable)
        i_fatal("nfs flush requires mmap_disable=yes");

    if ((ret = mail_index_open_files(index, flags)) <= 0) {
        /* doesn't exist and create flag not used */
        mail_index_close(index);
        return ret;
    }

    i_assert(index->map != NULL);
    index->opened = TRUE;
    return 1;
}

int mail_index_open_or_create(struct mail_index *index,
                  enum mail_index_open_flags flags,
                  enum file_lock_method lock_method)
{
    int ret;

    flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
    ret = mail_index_open(index, flags, lock_method);
    i_assert(ret != 0);
    return ret < 0 ? -1 : 0;
}

void mail_index_close_file(struct mail_index *index)
{
    if (index->file_lock != NULL)
        file_lock_free(&index->file_lock);

    if (index->fd != -1) {
        if (close(index->fd) < 0)
            mail_index_set_syscall_error(index, "close()");
        index->fd = -1;
    }

    index->lock_id_counter += 2;
    index->lock_type = F_UNLCK;
    index->shared_lock_count = 0;
    index->excl_lock_count = 0;
}

void mail_index_close(struct mail_index *index)
{
    if (index->map != NULL)
        mail_index_unmap(&index->map);

    mail_index_close_file(index);
    mail_transaction_log_close(index->log);
    if (index->cache != NULL)
        mail_cache_free(&index->cache);

    i_free_and_null(index->filepath);

    index->indexid = 0;
    index->opened = FALSE;
}

int mail_index_reopen_if_changed(struct mail_index *index)
{
    struct stat st1, st2;

    i_assert(index->shared_lock_count == 0 || !index->nfs_flush);
    i_assert(index->excl_lock_count == 0);

    if (MAIL_INDEX_IS_IN_MEMORY(index))
        return 0;

    if (index->fd == -1)
        return mail_index_try_open_only(index);

    if (index->nfs_flush)
        nfs_flush_file_handle_cache(index->filepath);
    if (nfs_safe_stat(index->filepath, &st2) < 0) {
        if (errno == ENOENT)
            return 0;
        return mail_index_set_syscall_error(index, "stat()");
    }

    if (fstat(index->fd, &st1) < 0) {
        if (errno != ESTALE)
            return mail_index_set_syscall_error(index, "fstat()");
        /* deleted/recreated, reopen */
    } else if (st1.st_ino == st2.st_ino &&
           CMP_DEV_T(st1.st_dev, st2.st_dev)) {
        /* the same file */
        return 1;
    }

    /* new file, new locks. the old fd can keep its locks, they don't
       matter anymore as no-one's going to modify the file. */
    mail_index_close_file(index);

    return mail_index_try_open_only(index);
}

int mail_index_refresh(struct mail_index *index)
{
    int ret;

    ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD);
    return ret <= 0 ? -1 : 0;
}

struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
    return index->cache;
}

int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
    va_list va;

    i_free(index->error);

    if (fmt == NULL)
        index->error = NULL;
    else {
        va_start(va, fmt);
        index->error = i_strdup_vprintf(fmt, va);
        va_end(va);

        i_error("%s", index->error);
    }

    return -1;
}

int mail_index_move_to_memory(struct mail_index *index)
{
    struct mail_index_map *map;

    if (MAIL_INDEX_IS_IN_MEMORY(index))
        return index->map == NULL ? -1 : 0;

    if (index->never_in_memory)
        return -1;

    /* set the index as being into memory */
    i_free_and_null(index->dir);

    i_free(index->filepath);
    index->filepath = i_strdup("(in-memory index)");

    if (index->map == NULL) {
        /* index was never even opened. just mark it as being in
           memory and let the caller re-open the index. */
        i_assert(index->fd == -1);
        return -1;
    }

    /* move index map to memory */
    if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
        map = mail_index_map_clone(index->map);
        mail_index_unmap(&index->map);
        index->map = map;
    }

    if (index->log != NULL) {
        /* move transaction log to memory */
        mail_transaction_log_move_to_memory(index->log);
    }

    if (index->file_lock != NULL)
        file_lock_free(&index->file_lock);

    if (index->fd != -1) {
        if (close(index->fd) < 0)
            mail_index_set_syscall_error(index, "close()");
        index->fd = -1;
    }
    return 0;
}

bool mail_index_is_in_memory(struct mail_index *index)
{
    return MAIL_INDEX_IS_IN_MEMORY(index);
}

void mail_index_mark_corrupted(struct mail_index *index)
{
    index->indexid = 0;

    index->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
    if (unlink(index->filepath) < 0 && errno != ENOENT && errno != ESTALE)
        mail_index_set_syscall_error(index, "unlink()");
}

int mail_index_set_syscall_error(struct mail_index *index,
                 const char *function)
{
    i_assert(function != NULL);

    if (ENOSPACE(errno)) {
        index->nodiskspace = TRUE;
        if (!index->never_in_memory)
            return -1;
    }

    return mail_index_set_error(index, "%s failed with index file %s: %m",
                    function, index->filepath);
}

int mail_index_file_set_syscall_error(struct mail_index *index,
                      const char *filepath,
                      const char *function)
{
    i_assert(filepath != NULL);
    i_assert(function != NULL);

    if (ENOSPACE(errno)) {
        index->nodiskspace = TRUE;
        if (!index->never_in_memory)
            return -1;
    }

    return mail_index_set_error(index, "%s failed with file %s: %m",
                    function, filepath);
}

const char *mail_index_get_error_message(struct mail_index *index)
{
    return index->error;
}

void mail_index_reset_error(struct mail_index *index)
{
    if (index->error != NULL) {
        i_free(index->error);
        index->error = NULL;
    }

    index->nodiskspace = FALSE;
        index->index_lock_timeout = FALSE;
}

#if WORDS_BIGENDIAN
/* FIXME: Unfortunately these functions were originally written to use
   endian-specific code and we can't avoid that without breaking backwards
   compatibility. When we do break it, just select one of these. */
uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
    i_assert(offset < 0x40000000);
    i_assert((offset & 3) == 0);

    offset >>= 2;
    return  0x00000080 | ((offset & 0x0000007f)) |
        0x00008000 | ((offset & 0x00003f80) >> 7 << 8) |
        0x00800000 | ((offset & 0x001fc000) >> 14 << 16) |
        0x80000000 | ((offset & 0x0fe00000) >> 21 << 24);
}

uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
    if ((offset & 0x80808080) != 0x80808080)
        return 0;

    return  (((offset & 0x0000007f)) |
         ((offset & 0x00007f00) >> 8 << 7) |
         ((offset & 0x007f0000) >> 16 << 14) |
         ((offset & 0x7f000000) >> 24 << 21)) << 2;
}
#else
uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
    i_assert(offset < 0x40000000);
    i_assert((offset & 3) == 0);

    offset >>= 2;
    return  0x80000000 | ((offset & 0x0000007f) << 24) |
        0x00800000 | ((offset & 0x00003f80) >> 7 << 16) |
        0x00008000 | ((offset & 0x001fc000) >> 14 << 8) |
        0x00000080 | ((offset & 0x0fe00000) >> 21);
}

uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
    if ((offset & 0x80808080) != 0x80808080)
        return 0;

    return  (((offset & 0x0000007f) << 21) |
         ((offset & 0x00007f00) >> 8 << 14) |
         ((offset & 0x007f0000) >> 16 << 7) |
         ((offset & 0x7f000000) >> 24)) << 2;
}
#endif

void mail_index_pack_num(uint8_t **p, uint32_t num)
{
    /* number continues as long as the highest bit is set */
    while (num >= 0x80) {
        **p = (num & 0x7f) | 0x80;
        *p += 1;
        num >>= 7;
    }

    **p = num;
    *p += 1;
}

int mail_index_unpack_num(const uint8_t **p, const uint8_t *end,
              uint32_t *num_r)
{
    const uint8_t *c = *p;
    uint32_t value = 0;
    unsigned int bits = 0;

    for (;;) {
        if (unlikely(c == end)) {
            /* we should never see EOF */
            *num_r = 0;
            return -1;
        }

        value |= (*c & 0x7f) << bits;
        if (*c < 0x80)
            break;

        bits += 7;
        c++;
    }

    if (unlikely(bits >= 32)) {
        /* broken input */
        *p = end;
        *num_r = 0;
        return -1;
    }

    *p = c + 1;
    *num_r = value;
    return 0;
}