src/lib-index/mail-index.c

	mail-index.c revision 006c7de040ef939a4a6bb6c75913a41a7791001a
e59faf65ce864fe95dc00f5d52b8323cdbd0608aTimo Sirainen/* Copyright (C) 2002 Timo Sirainen */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "lib.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "ioloop.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "hostpid.h"
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen#include "mmap-util.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "write-full.h"
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen#include "mail-index.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "mail-index-data.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "mail-index-util.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "mail-hash.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "mail-lockdir.h"
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen#include "mail-modifylog.h"
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include <stdio.h>
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen#include <stdlib.h>
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen#include <unistd.h>
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen#include <fcntl.h>
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen#include <dirent.h>
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen#include <utime.h>
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainenstatic int mmap_update(MailIndex *index)
373492be949e159fda651807b3acda2c5c077027Timo Sirainen{
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    unsigned int extra;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    if (!index->dirty_mmap) {
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen        index->header = (MailIndexHeader *) index->mmap_base;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen        return TRUE;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    }
373492be949e159fda651807b3acda2c5c077027Timo Sirainen
373492be949e159fda651807b3acda2c5c077027Timo Sirainen    if (index->mmap_base != NULL)
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen        (void)munmap(index->mmap_base, index->mmap_length);
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->mmap_base = mmap_rw_file(index->fd, &index->mmap_length);
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    if (index->mmap_base == MAP_FAILED) {
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen        index->mmap_base = NULL;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen        index_set_error(index, "index: mmap() failed with file %s: %m",
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen                index->filepath);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        return FALSE;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    if (index->mmap_length < sizeof(MailIndexHeader)) {
635df5b4cbcd7b24c825e01d9dd66d3a4274c4c7Timo Sirainen        index_set_error(index, "truncated index file %s",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen                index->filepath);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen                INDEX_MARK_CORRUPTED(index);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        return FALSE;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    extra = (index->mmap_length - sizeof(MailIndexHeader)) %
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        sizeof(MailIndexRecord);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    if (extra != 0) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        /* partial write or corrupted -
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen           truncate the file to valid length */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        index->mmap_length -= extra;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        (void)ftruncate(index->fd, (off_t)index->mmap_length);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    index->header = (MailIndexHeader *) index->mmap_base;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    index->dirty_mmap = FALSE;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    return TRUE;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen}
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainenvoid mail_index_close(MailIndex *index)
635df5b4cbcd7b24c825e01d9dd66d3a4274c4c7Timo Sirainen{
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->set_flags = 0;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->set_cache_fields = 0;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->opened = FALSE;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->updating = FALSE;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->inconsistent = FALSE;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->dirty_mmap = TRUE;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->lock_type = MAIL_LOCK_UNLOCK;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen    index->header = NULL;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    if (index->fd != -1) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        (void)close(index->fd);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        index->fd = -1;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    if (index->filepath != NULL) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        i_free(index->filepath);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        index->filepath = NULL;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    if (index->mmap_base != NULL) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        (void)munmap(index->mmap_base, index->mmap_length);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        index->mmap_base = NULL;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    if (index->data != NULL) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen                mail_index_data_free(index->data);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        index->data = NULL;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    if (index->hash != NULL) {
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen                mail_hash_free(index->hash);
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen        index->hash = NULL;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (index->modifylog != NULL) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen                mail_modifylog_free(index->modifylog);
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen        index->modifylog = NULL;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (index->error != NULL) {
380dbb60ae291cbe39d1f710284562ca9167150bTimo Sirainen        i_free(index->error);
380dbb60ae291cbe39d1f710284562ca9167150bTimo Sirainen        index->error = NULL;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenint mail_index_sync_file(MailIndex *index)
1b823b2b7790a1e1b7974fcf11a4c48a28e70f37Timo Sirainen{
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen    struct utimbuf ut;
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen    int failed;
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen    if (!mail_index_data_sync_file(index->data))
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen        return FALSE;
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen    if (index->mmap_base != NULL) {
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen        if (msync(index->mmap_base, index->mmap_length, MS_SYNC) == -1) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            index_set_error(index, "msync() failed for %s: %m",
1b823b2b7790a1e1b7974fcf11a4c48a28e70f37Timo Sirainen                    index->filepath);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            return FALSE;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    failed = FALSE;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    if (!mail_hash_sync_file(index->hash))
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        failed = TRUE;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    if (!mail_modifylog_sync_file(index->modifylog))
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        failed = TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    /* keep index's modify stamp same as the sync file's stamp */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    ut.actime = ioloop_time;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    ut.modtime = index->file_sync_stamp;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (utime(index->filepath, &ut) == -1) {
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen        index_set_error(index, "utime() failed for %s: %m",
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen                index->filepath);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        return FALSE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (fsync(index->fd) == -1) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        index_set_error(index, "fsync() failed for %s: %m",
1b823b2b7790a1e1b7974fcf11a4c48a28e70f37Timo Sirainen                index->filepath);
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen        return FALSE;
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen    }
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen    return !failed;
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen}
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
1b823b2b7790a1e1b7974fcf11a4c48a28e70f37Timo Sirainenint mail_index_fmsync(MailIndex *index, size_t size)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (msync(index->mmap_base, size, MS_SYNC) == -1) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        index_set_error(index, "msync() failed for %s: %m",
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen                index->filepath);
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        return FALSE;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (fsync(index->fd) == -1) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        index_set_error(index, "fsync() failed for %s: %m",
57d2429fae575e96ca276355af675deb66b76d00Timo Sirainen                index->filepath);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        return FALSE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
57d2429fae575e96ca276355af675deb66b76d00Timo Sirainen    return TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenint mail_index_rebuild_all(MailIndex *index)
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (!index->rebuild(index))
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        return FALSE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
61618d4c58080570f689614fec204ae14e90cef2Timo Sirainen    if (!mail_hash_rebuild(index->hash))
50e20db49f29917fe9adcf1b56b11badf28bd0e4Timo Sirainen        return FALSE;
50e20db49f29917fe9adcf1b56b11badf28bd0e4Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    return TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenstatic void mail_index_update_header_changes(MailIndex *index)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (index->set_flags != 0) {
50e20db49f29917fe9adcf1b56b11badf28bd0e4Timo Sirainen        index->header->flags |= index->set_flags;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        index->set_flags = 0;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (index->set_cache_fields != 0) {
c6f894e1522f7b0b6068c228900914073c145175Timo Sirainen        index->header->cache_fields = index->set_cache_fields;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        index->set_cache_fields = 0;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#define MAIL_LOCK_TO_FLOCK(lock_type) \
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        ((lock_type) == MAIL_LOCK_UNLOCK ? F_UNLCK : \
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        (lock_type) == MAIL_LOCK_SHARED ? F_RDLCK : F_WRLCK)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenint mail_index_try_lock(MailIndex *index, MailLockType lock_type)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    struct flock fl;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen    if (index->lock_type == lock_type)
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        return TRUE;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    /* lock whole file */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    fl.l_type = MAIL_LOCK_TO_FLOCK(lock_type);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    fl.l_whence = SEEK_SET;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    fl.l_start = 0;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    fl.l_len = 0;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    if (fcntl(index->fd, F_SETLK, &fl) == -1) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        if (errno != EINTR && errno != EACCES) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            index_set_error(index, "fcntl(F_SETLKW, %d) "
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen                    "failed for file %s: %m", fl.l_type,
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen                    index->filepath);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        return FALSE;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    return TRUE;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen}
c6f894e1522f7b0b6068c228900914073c145175Timo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenint mail_index_set_lock(MailIndex *index, MailLockType lock_type)
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen{
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    /* yeah, this function is a bit messy. besides locking, it keeps
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen       the index synced and in a good shape. */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    MailLockType old_lock_type;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    struct flock fl;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    int ret;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (index->inconsistent) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        /* index is in inconsistent state and nothing else than
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen           free() is allowed for it. */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        return FALSE;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (index->lock_type == lock_type)
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        return TRUE;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
383d0e8c24451468d6bea17e4b55d74de744abe6Timo Sirainen    /* shared -> exclusive isn't allowed */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    i_assert(lock_type != MAIL_LOCK_EXCLUSIVE ||
383d0e8c24451468d6bea17e4b55d74de744abe6Timo Sirainen         index->lock_type != MAIL_LOCK_SHARED);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        /* releasing exclusive lock */
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen        index->header->flags &= ~MAIL_INDEX_FLAG_FSCK;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen
2024157e8de36edd31f5fd72f5ea7364a0955fa7Timo Sirainen        mail_index_update_header_changes(index);
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen        /* sync mmaped memory */
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen        (void)mail_index_sync_file(index);
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen    }
15f526e5ac611b4532568d131fcd0abf664abe41Timo Sirainen
15f526e5ac611b4532568d131fcd0abf664abe41Timo Sirainen    if (lock_type != MAIL_LOCK_UNLOCK &&
15f526e5ac611b4532568d131fcd0abf664abe41Timo Sirainen        index->lock_type == MAIL_LOCK_UNLOCK && !index->updating) {
15f526e5ac611b4532568d131fcd0abf664abe41Timo Sirainen        /* unlock -> lock */
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen        index->updating = TRUE;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen        (void)index->sync(index);
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen        ret = mail_index_set_lock(index, lock_type);
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen        index->updating = FALSE;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen        return ret;
0f62889d833767acf9c2ad010c3269806b4cfae3Timo Sirainen    }
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen    /* lock whole file */
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen    fl.l_type = MAIL_LOCK_TO_FLOCK(lock_type);
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen    fl.l_whence = SEEK_SET;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen    fl.l_start = 0;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen    fl.l_len = 0;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen    while (fcntl(index->fd, F_SETLKW, &fl) == -1) {
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen        if (errno != EINTR) {
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen            index_set_error(index, "fcntl(F_SETLKW, %d) "
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen                    "failed for file %s: %m", fl.l_type,
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen                    index->filepath);
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen            return FALSE;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        }
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    }
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    if (lock_type == MAIL_LOCK_UNLOCK) {
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        /* reset last_lookup so rebuilds don't try to use it */
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        index->last_lookup = NULL;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    }
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    old_lock_type = index->lock_type;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    index->lock_type = lock_type;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen    if (lock_type != MAIL_LOCK_UNLOCK) {
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        /* we're always mmap()ed when we're locked */
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        if (!mmap_update(index)) {
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen            (void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK);
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen            return FALSE;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        }
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen        if (index->indexid != index->header->indexid) {
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen            /* index was rebuilt, there's no way we can maintain
272aca0a772140d3a45a425a3fd67854ae2ccec2Timo Sirainen               consistency */
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen            index_set_error(index, "Warning: Inconsistency - Index "
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen                    "%s was rebuilt while we had it open",
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen                    index->filepath);
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen            index->inconsistent = TRUE;
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen            return FALSE;
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen        }
1d22eaac93de41319918a1fc6de42bb302e25c1aTimo Sirainen    } else if (old_lock_type == MAIL_LOCK_SHARED) {
1d22eaac93de41319918a1fc6de42bb302e25c1aTimo Sirainen        /* releasing shared lock */
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen        unsigned int old_flags, old_cache;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        old_flags = index->header->flags;
2024157e8de36edd31f5fd72f5ea7364a0955fa7Timo Sirainen        old_cache = index->header->cache_fields;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        if ((old_flags | index->set_flags) != old_flags ||
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen            (old_cache | index->set_cache_fields) != old_cache) {
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen            /* need to update the header */
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen            index->updating = TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen            if (mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE))
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen                mail_index_update_header_changes(index);
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen            index->updating = FALSE;
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen            return mail_index_set_lock(index, MAIL_LOCK_UNLOCK);
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen        }
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen    }
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    if (lock_type == MAIL_LOCK_EXCLUSIVE) {
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen        /* while holding exclusive lock, keep the FSCK flag on.
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen           when the lock is released, the FSCK flag will also be
2024157e8de36edd31f5fd72f5ea7364a0955fa7Timo Sirainen           removed. */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        index->header->flags |= MAIL_INDEX_FLAG_FSCK;
        if (!mail_index_fmsync(index, sizeof(MailIndexHeader))) {
            (void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK);
            return FALSE;
        }
    }

    if (index->header != NULL && !index->updating &&
        (index->header->flags & MAIL_INDEX_FLAG_REBUILD) != 0) {
        /* index is corrupted, rebuild it */
        index->updating = TRUE;

        if (lock_type == MAIL_LOCK_SHARED)
            (void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK);

        if (!mail_index_rebuild_all(index))
            return FALSE;

        ret = mail_index_set_lock(index, lock_type);
        index->updating = FALSE;
        return ret;
    }

    if (lock_type == MAIL_LOCK_UNLOCK) {
        /* reset header so it's not used while being unlocked */
        index->last_lookup = NULL;
    }

    return TRUE;
}

static int read_and_verify_header(int fd, MailIndexHeader *hdr)
{
    /* read the header */
    if (lseek(fd, 0, SEEK_SET) != 0)
        return FALSE;

    if (read(fd, hdr, sizeof(MailIndexHeader)) != sizeof(MailIndexHeader))
        return FALSE;

    /* check the compatibility */
    if (hdr->compat_data[0] != MAIL_INDEX_COMPAT_FLAGS ||
        hdr->compat_data[1] != sizeof(unsigned int) ||
        hdr->compat_data[2] != sizeof(time_t) ||
        hdr->compat_data[3] != sizeof(uoff_t))
        return FALSE;

    /* check the version */
    return hdr->version == MAIL_INDEX_VERSION;
}

/* Returns TRUE if we're compatible with given index file */
static int mail_is_compatible_index(const char *path)
{
        MailIndexHeader hdr;
    int fd, compatible;

    fd = open(path, O_RDONLY);
    if (fd == -1)
        return FALSE;

    compatible = read_and_verify_header(fd, &hdr);

    (void)close(fd);
    return compatible;
}

/* Returns a file name of compatible index */
static const char *mail_find_index(MailIndex *index)
{
    DIR *dir;
    struct dirent *d;
    const char *name;
    char path[1024];
    unsigned int len;

    /* first try the primary name */
    i_snprintf(path, sizeof(path), "%s/" INDEX_FILE_PREFIX, index->dir);
    if (mail_is_compatible_index(path))
        return INDEX_FILE_PREFIX;

    dir = opendir(index->dir);
    if (dir == NULL) {
        /* path doesn't exist */
        index_set_error(index, "Can't open dir %s: %m",
                index->dir);
        return NULL;
    }

    len = strlen(INDEX_FILE_PREFIX);
    name = NULL;
    while ((d = readdir(dir)) != NULL) {
        if (strncmp(d->d_name, INDEX_FILE_PREFIX, len) == 0) {
            /* index found, check if we're compatible */
            i_snprintf(path, sizeof(path), "%s/%s",
                   index->dir, d->d_name);
            if (mail_is_compatible_index(path)) {
                name = t_strdup(d->d_name);
                break;
            }
        }
    }

    (void)closedir(dir);
    return name;
}

static int mail_index_open_init(MailIndex *index, int update_recent,
                MailIndexHeader *hdr)
{
    /* update \Recent message counters */
    if (update_recent && hdr->last_nonrecent_uid != hdr->next_uid-1) {
        /* keep last_recent_uid to next_uid-1 */
        if (index->lock_type == MAIL_LOCK_SHARED) {
            if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
                return FALSE;
        }

        if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
            return FALSE;

        index->first_recent_uid = index->header->last_nonrecent_uid+1;
        index->header->last_nonrecent_uid = index->header->next_uid-1;
    } else {
        index->first_recent_uid = hdr->last_nonrecent_uid+1;
    }

    if (hdr->next_uid >= INT_MAX-1024) {
        /* UID values are getting too high, rebuild index */
        index->set_flags |= MAIL_INDEX_FLAG_REBUILD;
    }

    return TRUE;
}

static int mail_index_open_file(MailIndex *index, const char *filename,
                int update_recent)
{
        MailIndexHeader hdr;
    const char *path;
    int fd, failed;

    /* the index file should already be checked that it exists and
       we're compatible with it. */

    path = t_strconcat(index->dir, "/", filename, NULL);
    fd = open(path, O_RDWR);
    if (fd == -1) {
        index_set_error(index, "Can't open index %s: %m", path);
        return FALSE;
    }

    /* check the compatibility anyway just to be sure */
    if (!read_and_verify_header(fd, &hdr)) {
        index_set_error(index, "Non-compatible index file %s", path);
        return FALSE;
    }

    if (index->fd != -1)
        mail_index_close(index);

    index->fd = fd;
    index->filepath = i_strdup(path);
    index->indexid = hdr.indexid;
    index->dirty_mmap = TRUE;
    index->updating = TRUE;

    failed = TRUE;
    do {
        /* open/create the index files */
        if (!mail_index_data_open(index)) {
            if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) == 0)
                break;

            /* data file is corrupted, need to rebuild index */
            hdr.flags |= MAIL_INDEX_FLAG_REBUILD;
            index->set_flags = 0;

            if (!mail_index_data_create(index))
                break;
        }
        if (!mail_hash_open_or_create(index))
            break;
        if (!mail_modifylog_open_or_create(index))
            break;

        if (hdr.flags & MAIL_INDEX_FLAG_REBUILD) {
            /* index is corrupted, rebuild */
            if (!mail_index_rebuild_all(index))
                break;
        }

        if (hdr.flags & MAIL_INDEX_FLAG_FSCK) {
            /* index needs fscking */
            if (!index->fsck(index))
                break;
        }

        if (hdr.flags & MAIL_INDEX_FLAG_COMPRESS) {
            /* remove deleted blocks from index file */
            if (!mail_index_compress(index))
                break;
        }

        if (hdr.flags & MAIL_INDEX_FLAG_REBUILD_HASH) {
            if (!mail_hash_rebuild(index->hash))
                break;
        }

        if (hdr.flags & MAIL_INDEX_FLAG_CACHE_FIELDS) {
            /* need to update cached fields */
            if (!mail_index_update_cache(index))
                break;
        }

        if (hdr.flags & MAIL_INDEX_FLAG_COMPRESS_DATA) {
            /* remove unused space from index data file.
               keep after cache_fields which may move data
               and create unused space.. */
            if (!mail_index_compress_data(index))
                break;
        }

        if (!index->sync(index))
            break;
        if (!mail_index_open_init(index, update_recent, &hdr))
            break;

        failed = FALSE;
    } while (FALSE);

    index->updating = FALSE;

    if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
        failed = TRUE;

    if (failed)
        mail_index_close(index);

    return !failed;
}

void mail_index_init_header(MailIndexHeader *hdr)
{
    memset(hdr, 0, sizeof(MailIndexHeader));
    hdr->compat_data[0] = MAIL_INDEX_COMPAT_FLAGS;
    hdr->compat_data[1] = sizeof(unsigned int);
    hdr->compat_data[2] = sizeof(time_t);
    hdr->compat_data[3] = sizeof(uoff_t);
    hdr->version = MAIL_INDEX_VERSION;
    hdr->indexid = ioloop_time;

    /* mark the index being rebuilt - rebuild() removes this flag
       when it succeeds */
    hdr->flags = MAIL_INDEX_FLAG_REBUILD;

    /* set the fields we always want to cache - currently nothing
       except the location. many clients aren't interested about
       any of the fields. */
    hdr->cache_fields = FIELD_TYPE_LOCATION;

    hdr->uid_validity = ioloop_time;
    hdr->next_uid = 1;
}

static int mail_index_create(MailIndex *index, int *dir_unlocked,
                 int update_recent)
{
        MailIndexHeader hdr;
    const char *path;
    char index_path[1024];
    int fd, len;

    *dir_unlocked = FALSE;

    /* first create the index into temporary file. */
    fd = mail_index_create_temp_file(index, &path);
    if (fd == -1)
        return FALSE;

    /* fill the header */
        mail_index_init_header(&hdr);

    /* write header */
    if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
        index_set_error(index, "Error writing to temp index %s: %m",
                path);
        (void)close(fd);
        (void)unlink(path);
        return FALSE;
    }

    /* move the temp index into the real one. we also need to figure
       out what to call ourself on the way. */
    len = i_snprintf(index_path, sizeof(index_path),
             "%s/" INDEX_FILE_PREFIX, index->dir);
    if (link(path, index_path) == 0)
        (void)unlink(path);
    else {
        if (errno != EEXIST) {
            /* fatal error */
            index_set_error(index, "link(%s, %s) failed: %m",
                    path, index_path);
            (void)close(fd);
            (void)unlink(path);
            return FALSE;
        }

        /* fallback to index.hostname - we require each system to
           have a different hostname so it's safe to override
           previous index as well */
        hostpid_init();
        i_snprintf(index_path + len, sizeof(index_path)-len,
               "-%s", my_hostname);

        if (rename(path, index_path) == -1) {
            index_set_error(index, "rename(%s, %s) failed: %m",
                    path, index_path);
            (void)close(fd);
            (void)unlink(path);
            return FALSE;
        }

        /* FIXME: race condition here! index may be opened before
           it's rebuilt. maybe set it locked here, and make it require
           shared lock when finding the indexes.. */
    }

    if (index->fd != -1)
        mail_index_close(index);

    index->fd = fd;
    index->filepath = i_strdup(index_path);
    index->indexid = hdr.indexid;
    index->updating = TRUE;
    index->dirty_mmap = TRUE;

    /* lock the index file and unlock the directory */
    if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) {
        index->updating = FALSE;
        return FALSE;
    }

    if (mail_index_lock_dir(index, MAIL_LOCK_UNLOCK))
        *dir_unlocked = TRUE;

    /* create the data file, build the index and hash */
    if (!mail_index_data_create(index) || !index->rebuild(index) ||
        !mail_hash_create(index) || !mail_modifylog_create(index)) {
        index->updating = FALSE;
        mail_index_close(index);
        return FALSE;
    }
    index->updating = FALSE;

    if (!mail_index_open_init(index, update_recent, index->header)) {
        mail_index_close(index);
        return FALSE;
    }

    /* unlock finally */
    if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) {
        mail_index_close(index);
        return FALSE;
    }

        return TRUE;
}

int mail_index_open(MailIndex *index, int update_recent)
{
    const char *name;

    i_assert(!index->opened);

    name = mail_find_index(index);
    if (name == NULL)
        return FALSE;

    if (!mail_index_open_file(index, name, update_recent))
        return FALSE;

    index->opened = TRUE;
    return TRUE;
}

int mail_index_open_or_create(MailIndex *index, int update_recent)
{
    const char *name;
    int failed, dir_unlocked;

    i_assert(!index->opened);

    /* first see if it's already there */
    name = mail_find_index(index);
    if (name != NULL && mail_index_open_file(index, name, update_recent)) {
        index->opened = TRUE;
        return TRUE;
    }

    /* index wasn't found or it was broken. get exclusive lock and check
       again, just to make sure we don't end up having two index files
       due to race condition with another process. */
    if (!mail_index_lock_dir(index, MAIL_LOCK_EXCLUSIVE))
        return FALSE;

    name = mail_find_index(index);
    if (name == NULL || !mail_index_open_file(index, name, update_recent)) {
        /* create/rebuild index */
        failed = !mail_index_create(index, &dir_unlocked,
                        update_recent);
    } else {
        dir_unlocked = FALSE;
        failed = FALSE;
    }

    if (!dir_unlocked && !mail_index_lock_dir(index, MAIL_LOCK_UNLOCK))
        return FALSE;

    if (failed)
        return FALSE;

    index->opened = TRUE;
    return TRUE;
}

int mail_index_verify_hole_range(MailIndex *index)
{
    MailIndexHeader *hdr;
    unsigned int max_records, first_records;

    hdr = index->header;
    if (hdr->first_hole_position == 0)
        return TRUE;

    /* make sure position is valid */
    if (hdr->first_hole_position < sizeof(MailIndexHeader) ||
        (hdr->first_hole_position -
         sizeof(MailIndexHeader)) % sizeof(MailIndexRecord) != 0) {
        index_set_error(index, "Error in index file %s: "
                "first_hole_position contains invalid value",
                index->filepath);
        INDEX_MARK_CORRUPTED(index);
        return FALSE;
    }

    /* make sure position is in range.. */
    if (hdr->first_hole_position >= index->mmap_length) {
        index_set_error(index, "Error in index file %s: "
                "first_hole_position points outside file",
                index->filepath);
        INDEX_MARK_CORRUPTED(index);
        return FALSE;
    }

    /* and finally check that first_hole_records is in valid range */
    max_records = MAIL_INDEX_RECORD_COUNT(index);
    first_records = (hdr->first_hole_position -
             sizeof(MailIndexHeader)) / sizeof(MailIndexRecord);
    if (index->header->first_hole_records > max_records ||
        first_records + index->header->first_hole_records > max_records) {
        index_set_error(index, "Error in index file %s: "
                "first_hole_records points outside file",
                index->filepath);
        INDEX_MARK_CORRUPTED(index);
        return FALSE;
    }

    return TRUE;
}

static MailIndexRecord *mail_index_lookup_mapped(MailIndex *index,
                         unsigned int lookup_seq)
{
    MailIndexHeader *hdr;
    MailIndexRecord *rec, *last_rec;
    unsigned int seq;
    uoff_t seekpos;

    if (lookup_seq == index->last_lookup_seq &&
        index->last_lookup != NULL && index->last_lookup->uid != 0) {
        /* wanted the same record as last time */
        return index->last_lookup;
    }

    hdr = index->header;
    if (lookup_seq > hdr->messages_count) {
        /* out of range */
        return NULL;
    }

    if (!mail_index_verify_hole_range(index))
        return NULL;

    seekpos = sizeof(MailIndexHeader) +
        (uoff_t)(lookup_seq-1) * sizeof(MailIndexRecord);
    if (seekpos + sizeof(MailIndexRecord) > index->mmap_length) {
        /* out of range */
        return NULL;
    }

    rec = (MailIndexRecord *) ((char *) index->mmap_base +
                   sizeof(MailIndexHeader));
    last_rec = (MailIndexRecord *) ((char *) index->mmap_base +
                    index->mmap_length -
                    sizeof(MailIndexRecord));

    if (hdr->first_hole_position == 0 ||
        hdr->first_hole_position > seekpos) {
        /* easy, it's just at the expected index */
        rec += lookup_seq-1;
        i_assert(rec <= last_rec);

        if (rec->uid == 0) {
            index_set_error(index, "Error in index file %s: "
                    "first_hole_position wasn't updated "
                    "properly", index->filepath);
            INDEX_MARK_CORRUPTED(index);
            return NULL;
        }
        return rec;
    }

    /* we need to walk through the index to get to wanted position */
    if (lookup_seq > index->last_lookup_seq && index->last_lookup != NULL) {
        /* we want to lookup data after last lookup -
           this helps us some */
        rec = index->last_lookup;
        seq = index->last_lookup_seq;
    } else {
        /* some mails are deleted, jump after the first known hole
           and start counting non-deleted messages.. */
        seq = INDEX_POSITION_INDEX(hdr->first_hole_position + 1) + 1;
        rec += seq-1 + hdr->first_hole_records;
    }

    while (seq < lookup_seq && rec <= last_rec) {
        if (rec->uid != 0)
            seq++;
        rec++;
    }

    return rec;
}

MailIndexHeader *mail_index_get_header(MailIndex *index)
{
    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);

    return index->header;
}

MailIndexRecord *mail_index_lookup(MailIndex *index, unsigned int seq)
{
    i_assert(seq > 0);
    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);

    if (!mmap_update(index))
        return NULL;

    index->last_lookup = mail_index_lookup_mapped(index, seq);
    index->last_lookup_seq = seq;
    return index->last_lookup;
}

MailIndexRecord *mail_index_next(MailIndex *index, MailIndexRecord *rec)
{
    MailIndexRecord *end_rec;

    i_assert(!index->dirty_mmap);
    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);

    if (rec == NULL)
        return NULL;

    /* go to the next non-deleted record */
    end_rec = (MailIndexRecord *) ((char *) index->mmap_base +
                       index->mmap_length);
    while (++rec < end_rec) {
        if (rec->uid != 0)
            return rec;
    }

    return NULL;
}

MailIndexRecord *mail_index_lookup_uid_range(MailIndex *index,
                         unsigned int first_uid,
                         unsigned int last_uid)
{
    MailIndexRecord *rec, *end_rec;
    unsigned int uid, last_try_uid;
    uoff_t pos;

    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
    i_assert(first_uid > 0 && last_uid > 0);
    i_assert(first_uid <= last_uid);

    if (!mmap_update(index))
        return NULL;

    /* try the few first with hash lookups */
    last_try_uid = last_uid - first_uid < 10 ? last_uid : first_uid + 4;
    for (uid = first_uid; uid <= last_try_uid; uid++) {
        pos = mail_hash_lookup_uid(index->hash, uid);
        if (pos != 0) {
            return (MailIndexRecord *)
                ((char *) index->mmap_base + pos);
        }
    }

    if (last_try_uid == last_uid)
        return NULL;

    /* fallback to looking through the whole index - this shouldn't be
       needed often, so don't bother trying anything too fancy. */
    rec = (MailIndexRecord *) ((char *) index->mmap_base +
                   sizeof(MailIndexHeader));
    end_rec = (MailIndexRecord *) ((char *) index->mmap_base +
                       index->mmap_length);
    while (rec < end_rec) {
        if (rec->uid != 0) {
            if (rec->uid > last_uid)
                return NULL;

            if (rec->uid >= first_uid)
                return rec;
        }
        rec++;
    }

    return NULL;
}

const char *mail_index_lookup_field(MailIndex *index, MailIndexRecord *rec,
                    MailField field)
{
    MailIndexDataRecord *datarec;

    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);

    /* first check if the field even could be in the file */
    if ((rec->cached_fields & field) != field) {
        if ((index->header->cache_fields & field) == 0) {
            /* no, but make sure the future records will have it.
               we don't immediately mark the index to cache this
               field for old messages as some clients never ask
               the info again */
            index->set_cache_fields |= field;
        } else {
            /* this is at least the second time it's being asked,
               make sure it'll be cached soon. */
            index->set_flags |= MAIL_INDEX_FLAG_CACHE_FIELDS;
        }

        return NULL;
    }

    datarec = mail_index_data_lookup(index->data, rec, field);
    if (datarec == NULL) {
        /* corrupted, the field should have been there */
        index_set_error(index, "Error in index file %s: "
                "Field not found from data file",
                index->filepath);
        INDEX_MARK_CORRUPTED(index);
        return NULL;
    }

    if (!mail_index_data_record_verify(index->data, datarec)) {
        /* index is corrupted, it will be rebuilt */
        return NULL;
    }

    return datarec->data;
}

unsigned int mail_index_get_sequence(MailIndex *index, MailIndexRecord *rec)
{
    MailIndexRecord *seekrec;
    unsigned int seq;

    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);

    if (rec == index->last_lookup) {
        /* same as last lookup sequence - too easy */
        return index->last_lookup_seq;
    }

    if (index->header->first_hole_position == 0) {
        /* easy, it's just at the expected index */
        return INDEX_POSITION_INDEX(
            INDEX_FILE_POSITION(index, rec)) + 1;
    }

    if (!mail_index_verify_hole_range(index))
        return 0;

    seekrec = (MailIndexRecord *) ((char *) index->mmap_base +
                       index->header->first_hole_position);
    if (rec < seekrec) {
        /* record before first hole */
        return INDEX_POSITION_INDEX(
            INDEX_FILE_POSITION(index, rec)) + 1;
    }

    /* we know the sequence after the first hole - skip to there and
       start browsing the records until ours is found */
    seq = INDEX_POSITION_INDEX(INDEX_FILE_POSITION(index, seekrec))+1;
    seekrec += index->header->first_hole_records;

    for (; seekrec < rec; seekrec++) {
        if (seekrec->uid != 0)
            seq++;
    }

    return seq;
}

static void index_mark_flag_changes(MailIndex *index, MailIndexRecord *rec,
                    MailFlags old_flags, MailFlags new_flags)
{
    if ((old_flags & MAIL_SEEN) == 0 && (new_flags & MAIL_SEEN)) {
        /* unseen -> seen */
        index->header->seen_messages_count++;
    } else if ((old_flags & MAIL_SEEN) && (new_flags & MAIL_SEEN) == 0) {
        /* seen -> unseen */
        if (index->header->seen_messages_count ==
            index->header->messages_count) {
            /* this is the first unseen message */
                        index->header->first_unseen_uid_lowwater = rec->uid;
        } else if (rec->uid < index->header->first_unseen_uid_lowwater)
            index->header->first_unseen_uid_lowwater = rec->uid;

        if (index->header->seen_messages_count == 0)
                        INDEX_MARK_CORRUPTED(index);
        else
            index->header->seen_messages_count--;
    } else if ((old_flags & MAIL_DELETED) == 0 &&
           (new_flags & MAIL_DELETED)) {
        /* undeleted -> deleted */
        index->header->deleted_messages_count++;

        if (index->header->deleted_messages_count == 1) {
            /* this is the first deleted message */
            index->header->first_deleted_uid_lowwater = rec->uid;
        } else if (rec->uid < index->header->first_deleted_uid_lowwater)
            index->header->first_deleted_uid_lowwater = rec->uid;
    } else if ((old_flags & MAIL_DELETED) &&
           (new_flags & MAIL_DELETED) == 0) {
        /* deleted -> undeleted */
        if (index->header->deleted_messages_count == 0)
                        INDEX_MARK_CORRUPTED(index);
        else
            index->header->deleted_messages_count--;
    }
}

static void update_first_hole_records(MailIndex *index)
{
        MailIndexRecord *rec, *end_rec;

    /* see if first_hole_records can be grown */
    rec = (MailIndexRecord *) ((char *) index->mmap_base +
                   index->header->first_hole_position) +
        index->header->first_hole_records;
    end_rec = (MailIndexRecord *) ((char *) index->mmap_base +
                       index->mmap_length);
    while (rec < end_rec && rec->uid == 0) {
        index->header->first_hole_records++;
        rec++;
    }
}

static int mail_index_truncate(MailIndex *index)
{
    /* truncate index file */
    if (ftruncate(index->fd, (off_t)index->header->first_hole_position) < 0)
        return FALSE;

    /* update header */
    index->header->first_hole_position = 0;
    index->header->first_hole_records = 0;

    if (index->header->messages_count == 0) {
        /* all mail was deleted, truncate data file */
        if (!mail_index_data_reset(index->data))
            return FALSE;
    }

    index->dirty_mmap = TRUE;
    if (!mmap_update(index))
        return FALSE;

    return TRUE;
}

int mail_index_expunge(MailIndex *index, MailIndexRecord *rec,
               unsigned int seq, int external_change)
{
    MailIndexHeader *hdr;
    uoff_t pos;

    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
    i_assert(rec->uid != 0);

    if (seq != 0) {
        if (!mail_modifylog_add_expunge(index->modifylog, seq,
                        rec->uid, external_change))
            return FALSE;
    }

    /* expunge() may be called while index is being rebuilt and when
       there's no hash yet */
    if (index->hash != NULL)
        mail_hash_update(index->hash, rec->uid, 0);

    /* setting UID to 0 is enough for deleting the mail from index */
    rec->uid = 0;

    /* update last_lookup_seq */
    if (seq != 0) {
        /* note that last_lookup can be left to point to
           invalid record so that next() works properly */
        if (seq == index->last_lookup_seq)
            index->last_lookup = NULL;
        else if (seq < index->last_lookup_seq)
            index->last_lookup_seq--;
    }

    if (!mail_index_verify_hole_range(index))
        return FALSE;

    hdr = index->header;

    /* update first hole */
    pos = INDEX_FILE_POSITION(index, rec);
    if (hdr->first_hole_position < sizeof(MailIndexRecord)) {
        /* first deleted message in index */
        hdr->first_hole_position = pos;
        hdr->first_hole_records = 1;
    } else if (hdr->first_hole_position - sizeof(MailIndexRecord) == pos) {
        /* deleted the previous record before hole */
        hdr->first_hole_position -= sizeof(MailIndexRecord);
        hdr->first_hole_records++;
    } else if (hdr->first_hole_position +
           (hdr->first_hole_records * sizeof(MailIndexRecord)) == pos) {
        /* deleted the next record after hole */
        hdr->first_hole_records++;
        update_first_hole_records(index);
    } else {
        /* second hole coming to index file, the index now needs to
           be compressed to keep high performance */
        index->set_flags |= MAIL_INDEX_FLAG_COMPRESS;

        if (hdr->first_hole_position > pos) {
            /* new hole before the old hole */
            hdr->first_hole_position = pos;
            hdr->first_hole_records = 1;
        }
    }

    /* update message counts */
    if (hdr->messages_count == 0) {
        /* corrupted */
        index_set_error(index, "Error in index file %s: "
                "Header says there's no mail while expunging",
                index->filepath);
        INDEX_MARK_CORRUPTED(index);
        return FALSE;
    }

    hdr->messages_count--;
    index_mark_flag_changes(index, rec, rec->msg_flags, 0);

    if ((hdr->first_hole_position - sizeof(MailIndexHeader)) /
        sizeof(MailIndexRecord) == hdr->messages_count) {
        /* the hole reaches end of file, truncate it */
        (void)mail_index_truncate(index);
    } else {
        /* update deleted_space in data file */
        (void)mail_index_data_add_deleted_space(index->data,
                            rec->data_size);
    }

    return TRUE;
}

int mail_index_update_flags(MailIndex *index, MailIndexRecord *rec,
                unsigned int seq, MailFlags flags,
                int external_change)
{
    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
    i_assert(seq != 0);

    if (flags == rec->msg_flags)
        return TRUE; /* no changes */

        index_mark_flag_changes(index, rec, rec->msg_flags, flags);

    rec->msg_flags = flags;
    return mail_modifylog_add_flags(index->modifylog, seq,
                    rec->uid, external_change);
}

int mail_index_append(MailIndex *index, MailIndexRecord **rec)
{
    off_t pos;

    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);

    (*rec)->uid = index->header->next_uid++;

    pos = lseek(index->fd, 0, SEEK_END);
    if (pos < 0) {
        index_set_error(index, "lseek() failed with file %s: %m",
                index->filepath);
        return FALSE;
    }

    if (write_full(index->fd, *rec, sizeof(MailIndexRecord)) < 0) {
        index_set_error(index, "Error appending to file %s: %m",
                index->filepath);
        return FALSE;
    }

    index->header->messages_count++;
        index_mark_flag_changes(index, *rec, 0, (*rec)->msg_flags);

    if (index->hash != NULL)
        mail_hash_update(index->hash, (*rec)->uid, (uoff_t)pos);

    index->dirty_mmap = TRUE;
    if (!mmap_update(index))
        return FALSE;

    *rec = (MailIndexRecord *) ((char *) index->mmap_base + pos);
    return TRUE;
}

const char *mail_index_get_last_error(MailIndex *index)
{
    return index->error;
}

int mail_index_is_inconsistency_error(MailIndex *index)
{
    return index->inconsistent;
}