src/lib-index/mail-cache.c

	mail-cache.c revision 131b073bdc3650083b00616dc778dd3017c2bbb5
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2003-2012 Dovecot authors, see the included COPYING file */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "lib.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "array.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "buffer.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "hash.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "nfs-workarounds.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "file-cache.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "mmap-util.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "read-full.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "write-full.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "mail-cache-private.h"
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include <unistd.h>
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#define MAIL_CACHE_MIN_HEADER_READ_SIZE 4096
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenvoid mail_cache_set_syscall_error(struct mail_cache *cache,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                  const char *function)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    mail_index_file_set_syscall_error(cache->index, cache->filepath,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                      function);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic void mail_cache_unlink(struct mail_cache *cache)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!cache->index->readonly)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        (void)unlink(cache->filepath);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenvoid mail_cache_reset(struct mail_cache *cache)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    mail_cache_unlink(cache);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    /* mark the cache as unusable */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->hdr = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenvoid mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    va_list va;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    mail_cache_reset(cache);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    va_start(va, fmt);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    T_BEGIN {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_index_set_error(cache->index,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                     "Corrupted index cache file %s: %s",
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen                     cache->filepath,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                     t_strdup_vprintf(fmt, va));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } T_END;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    va_end(va);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenvoid mail_cache_file_close(struct mail_cache *cache)
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cache->mmap_base != NULL) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (munmap(cache->mmap_base, cache->mmap_length) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            mail_cache_set_syscall_error(cache, "munmap()");
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cache->file_cache != NULL)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        file_cache_set_fd(cache->file_cache, -1);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->mmap_base = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->hdr = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->mmap_length = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->last_field_header_offset = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cache->file_lock != NULL)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        file_lock_free(&cache->file_lock);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->locked = FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cache->fd != -1) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (close(cache->fd) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            mail_cache_set_syscall_error(cache, "close()");
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        cache->fd = -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic void mail_cache_init_file_cache(struct mail_cache *cache)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    struct stat st;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cache->file_cache == NULL)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    file_cache_set_fd(cache->file_cache, cache->fd);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (fstat(cache->fd, &st) == 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        (void)file_cache_set_size(cache->file_cache, st.st_size);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    else if (!ESTALE_FSTAT(errno))
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_cache_set_syscall_error(cache, "fstat()");
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->st_ino = st.st_ino;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->st_dev = st.st_dev;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic bool mail_cache_need_reopen(struct mail_cache *cache)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    struct stat st;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (cache->need_compress_file_seq != 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            /* we're waiting for compression */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            /* disabled */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cache->fd == -1)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return TRUE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    /* see if the file has changed */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        i_assert(!cache->locked);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        nfs_flush_file_handle_cache(cache->filepath);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (nfs_safe_stat(cache->filepath, &st) < 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_cache_set_syscall_error(cache, "stat()");
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return TRUE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (st.st_ino != cache->st_ino ||
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        !CMP_DEV_T(st.st_dev, cache->st_dev)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* file changed */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return TRUE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* if the old file has been deleted, the new file may have
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen           the same inode as the old one. we'll catch this here by
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen           checking if fstat() fails with ESTALE */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (fstat(cache->fd, &st) < 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (ESTALE_FSTAT(errno))
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                return TRUE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            mail_cache_set_syscall_error(cache, "fstat()");
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenint mail_cache_reopen(struct mail_cache *cache)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    struct mail_index_view *view;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const struct mail_index_ext *ext;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const void *data;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    i_assert(!cache->locked);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!mail_cache_need_reopen(cache)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* reopening does no good */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    mail_cache_file_close(cache);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->fd = nfs_safe_open(cache->filepath,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                  cache->index->readonly ? O_RDONLY : O_RDWR);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cache->fd == -1) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (errno == ENOENT)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            cache->need_compress_file_seq = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        else
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            mail_cache_set_syscall_error(cache, "open()");
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    mail_cache_init_file_cache(cache);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (mail_cache_map(cache, 0, 0, &data) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (mail_cache_header_fields_read(cache) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    view = mail_index_view_open(cache->index);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    ext = mail_index_view_get_ext(view, cache->ext_id);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    if (ext == NULL || cache->hdr->file_seq != ext->reset_id) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* still different - maybe a race condition or maybe the
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen           file_seq really is corrupted. either way, this shouldn't
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen           happen often so we'll just mark cache to be compressed
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen           later which fixes this. */
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        cache->need_compress_file_seq = cache->hdr->file_seq;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_index_view_close(&view);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    mail_index_view_close(&view);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    return 1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic void mail_cache_update_need_compress(struct mail_cache *cache)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    const struct mail_cache_header *hdr = cache->hdr;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    struct stat st;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    unsigned int cont_percentage;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    uoff_t file_size, max_del_space;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (fstat(cache->fd, &st) < 0) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        if (!ESTALE_FSTAT(errno))
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen            mail_cache_set_syscall_error(cache, "fstat()");
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        return;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    file_size = st.st_size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        cont_percentage = hdr->continued_record_count * 100 /
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        (cache->index->map->rec_map->records_count == 0 ? 1 :
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen         cache->index->map->rec_map->records_count);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE &&
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* too many continued rows, compress */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        cache->need_compress_file_seq = hdr->file_seq;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    /* see if we've reached the max. deleted space in file */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    max_del_space = file_size / 100 * MAIL_CACHE_COMPRESS_PERCENTAGE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (hdr->deleted_space >= max_del_space &&
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        cache->need_compress_file_seq = hdr->file_seq;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
2f16d2e0b4408370cd44db359759b23a8c0656d3Phil Carmodystatic bool mail_cache_verify_header(struct mail_cache *cache,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                     const struct mail_cache_header *hdr)
2f16d2e0b4408370cd44db359759b23a8c0656d3Phil Carmody{
2f16d2e0b4408370cd44db359759b23a8c0656d3Phil Carmody    /* check that the header is still ok */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (cache->mmap_length < sizeof(struct mail_cache_header)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_cache_set_corrupted(cache, "File too small");
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    if (hdr->version != MAIL_CACHE_VERSION) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* version changed - upgrade silently */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_cache_unlink(cache);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    if (hdr->compat_sizeof_uoff_t != sizeof(uoff_t)) {
9644b7914445f0fb1098038218bfcb7d135a8698Timo Sirainen        /* architecture change - handle silently(?) */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_cache_unlink(cache);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    if (hdr->indexid != cache->index->indexid) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        /* index id changed - handle silently */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_cache_unlink(cache);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    if (hdr->file_seq == 0) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        mail_cache_set_corrupted(cache, "file_seq is 0");
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        return FALSE;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    return TRUE;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen}
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic int
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainenmail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size,
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen              const void *hdr_data, bool copy_hdr)
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen{
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    const struct mail_cache_header *hdr = hdr_data;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (offset == 0) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        /* verify the header validity only with offset=0. this way
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen           we won't waste time re-verifying it all the time */
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        if (!mail_cache_verify_header(cache, hdr)) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen            cache->need_compress_file_seq =
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen                !MAIL_CACHE_IS_UNUSABLE(cache) &&
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen                cache->hdr->file_seq != 0 ?
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                cache->hdr->file_seq : 0;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen            return -1;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    if (hdr_data != NULL) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        if (!copy_hdr)
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen            cache->hdr = hdr;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            memcpy(&cache->hdr_ro_copy, hdr,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                   sizeof(cache->hdr_ro_copy));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            cache->hdr = &cache->hdr_ro_copy;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        mail_cache_update_need_compress(cache);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        i_assert(cache->hdr != NULL);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (offset + size > cache->mmap_length)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    return 1;
795aeec896095aa8f08cc5d3282c88cc0921bff6Timo Sirainen}
795aeec896095aa8f08cc5d3282c88cc0921bff6Timo Sirainen
795aeec896095aa8f08cc5d3282c88cc0921bff6Timo Sirainenstatic int
795aeec896095aa8f08cc5d3282c88cc0921bff6Timo Sirainenmail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size,
795aeec896095aa8f08cc5d3282c88cc0921bff6Timo Sirainen             const void **data_r)
795aeec896095aa8f08cc5d3282c88cc0921bff6Timo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const void *hdr_data;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    void *data;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    ssize_t ret;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    if (cache->read_buf == NULL) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        cache->read_buf =
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen            buffer_create_dynamic(default_pool, size);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    } else if (cache->read_offset <= offset &&
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen           cache->read_offset + cache->read_buf->used >= offset+size) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        /* already mapped */
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        *data_r = CONST_PTR_OFFSET(cache->read_buf->data,
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen                       offset - cache->read_offset);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        hdr_data = offset == 0 ? *data_r : NULL;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    } else {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        buffer_set_used_size(cache->read_buf, 0);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    if (offset == 0 && size < MAIL_CACHE_MIN_HEADER_READ_SIZE) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        /* we can usually read the fields header after the cache
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen           header. we need them both, so try to read them all with one
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen           pread() call. */
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        size = MAIL_CACHE_MIN_HEADER_READ_SIZE;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    data = buffer_append_space_unsafe(cache->read_buf, size);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    ret = pread(cache->fd, data, size, offset);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (ret < 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (errno != ESTALE)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            mail_cache_set_syscall_error(cache, "read()");
1df526903ed039e8ff966a223c43b8d04eddf3c7Phil Carmody
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        buffer_set_used_size(cache->read_buf, 0);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        cache->hdr = NULL;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        cache->mmap_length = 0;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen        return -1;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    }
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    buffer_set_used_size(cache->read_buf, ret);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen    cache->read_offset = offset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    cache->mmap_length = offset + size;
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    *data_r = data;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    hdr_data = offset == 0 ? *data_r : NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE);
}

int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
           const void **data_r)
{
    const void *data;
    ssize_t ret;

    if (size == 0)
        size = sizeof(struct mail_cache_header);

    cache->remap_counter++;
    if (cache->map_with_read)
        return mail_cache_map_with_read(cache, offset, size, data_r);

    if (cache->file_cache != NULL) {
        cache->hdr = NULL;

        ret = file_cache_read(cache->file_cache, offset, size);
        if (ret < 0) {
                        /* In case of ESTALE we'll simply fail without error
                           messages. The caller will then just have to
                           fallback to generating the value itself.

                           We can't simply reopen the cache flie, because
                           using it requires also having updated file
                           offsets. */
                        if (errno != ESTALE)
                                mail_cache_set_syscall_error(cache, "read()");
            return -1;
        }

        data = file_cache_get_map(cache->file_cache,
                      &cache->mmap_length);
        *data_r = offset > cache->mmap_length ? NULL :
            CONST_PTR_OFFSET(data, offset);
        return mail_cache_map_finish(cache, offset, size, data, TRUE);
    }

    if (offset < cache->mmap_length &&
        size <= cache->mmap_length - offset) {
        /* already mapped */
        *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset);
        return 1;
    }

    if (cache->mmap_base != NULL) {
        if (munmap(cache->mmap_base, cache->mmap_length) < 0)
            mail_cache_set_syscall_error(cache, "munmap()");
    } else {
        if (cache->fd == -1) {
            /* unusable, waiting for compression or
               index is in memory */
            i_assert(cache->need_compress_file_seq != 0 ||
                 MAIL_INDEX_IS_IN_MEMORY(cache->index));
            return -1;
        }
    }

    /* map the whole file */
    cache->hdr = NULL;
    cache->mmap_length = 0;

    cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length);
    if (cache->mmap_base == MAP_FAILED) {
        cache->mmap_base = NULL;
        mail_cache_set_syscall_error(cache, "mmap()");
        return -1;
    }
    *data_r = offset > cache->mmap_length ? NULL :
        CONST_PTR_OFFSET(cache->mmap_base, offset);
    return mail_cache_map_finish(cache, offset, size,
                     cache->mmap_base, FALSE);
}

static int mail_cache_try_open(struct mail_cache *cache)
{
    const void *data;

    cache->opened = TRUE;

    if (MAIL_INDEX_IS_IN_MEMORY(cache->index))
        return 0;

    cache->fd = nfs_safe_open(cache->filepath,
                  cache->index->readonly ? O_RDONLY : O_RDWR);
    if (cache->fd == -1) {
        if (errno == ENOENT) {
            cache->need_compress_file_seq = 0;
            return 0;
        }

        mail_cache_set_syscall_error(cache, "open()");
        return -1;
    }

    mail_cache_init_file_cache(cache);

    if (mail_cache_map(cache, 0, sizeof(struct mail_cache_header),
               &data) < 0)
        return -1;
    return 1;
}

int mail_cache_open_and_verify(struct mail_cache *cache)
{
    int ret;

    ret = mail_cache_try_open(cache);
    if (ret > 0)
        ret = mail_cache_header_fields_read(cache);
    if (ret < 0) {
        /* failed for some reason - doesn't really matter,
           it's disabled for now. */
        mail_cache_file_close(cache);
    }
    return ret;
}

static struct mail_cache *mail_cache_alloc(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = i_new(struct mail_cache, 1);
    cache->index = index;
    cache->fd = -1;
    cache->filepath =
        i_strconcat(index->filepath, MAIL_CACHE_FILE_SUFFIX, NULL);
    cache->field_pool = pool_alloconly_create("Cache fields", 2048);
    hash_table_create(&cache->field_name_hash, cache->field_pool, 0,
              strcase_hash, strcasecmp);

    cache->dotlock_settings.use_excl_lock =
        (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
    cache->dotlock_settings.nfs_flush =
        (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;
    cache->dotlock_settings.timeout =
        I_MIN(MAIL_CACHE_LOCK_TIMEOUT, index->max_lock_timeout_secs);
    cache->dotlock_settings.stale_timeout = MAIL_CACHE_LOCK_CHANGE_TIMEOUT;

    if (!MAIL_INDEX_IS_IN_MEMORY(index) &&
        (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0)
        cache->file_cache = file_cache_new(-1);
    cache->map_with_read =
        (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0;

    cache->ext_id =
        mail_index_ext_register(index, "cache", 0,
                    sizeof(uint32_t), sizeof(uint32_t));
    mail_index_register_expunge_handler(index, cache->ext_id, FALSE,
                        mail_cache_expunge_handler, cache);
    return cache;
}

struct mail_cache *mail_cache_open_or_create(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = mail_cache_alloc(index);
    return cache;
}

struct mail_cache *mail_cache_create(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = mail_cache_alloc(index);
    if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
        if (unlink(cache->filepath) < 0 && errno != ENOENT)
            mail_cache_set_syscall_error(cache, "unlink()");
    }
    return cache;
}

void mail_cache_free(struct mail_cache **_cache)
{
    struct mail_cache *cache = *_cache;

    *_cache = NULL;
    if (cache->file_cache != NULL)
        file_cache_free(&cache->file_cache);

    mail_index_unregister_expunge_handler(cache->index, cache->ext_id);
    mail_cache_file_close(cache);

    hash_table_destroy(&cache->field_name_hash);
    pool_unref(&cache->field_pool);
    i_free(cache->field_file_map);
    i_free(cache->file_field_map);
    i_free(cache->fields);
    i_free(cache->filepath);
    i_free(cache);
}

static int mail_cache_lock_file(struct mail_cache *cache, bool nonblock)
{
    unsigned int timeout_secs;
    int ret;

    if (cache->last_lock_failed) {
        /* previous locking failed. don't waste time waiting on it
           again, just try once to see if it's available now. */
        nonblock = TRUE;
    }

    if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
        i_assert(cache->file_lock == NULL);
        timeout_secs = I_MIN(MAIL_CACHE_LOCK_TIMEOUT,
                     cache->index->max_lock_timeout_secs);

        ret = mail_index_lock_fd(cache->index, cache->filepath,
                     cache->fd, F_WRLCK,
                     nonblock ? 0 : timeout_secs,
                     &cache->file_lock);
    } else {
        enum dotlock_create_flags flags =
            nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0;

        i_assert(cache->dotlock == NULL);
        ret = file_dotlock_create(&cache->dotlock_settings,
                      cache->filepath, flags,
                      &cache->dotlock);
        if (ret < 0) {
            mail_cache_set_syscall_error(cache,
                             "file_dotlock_create()");
        }
    }
    cache->last_lock_failed = ret <= 0;

    /* don't bother warning if locking failed due to a timeout. since cache
       updating isn't all that important we're using a very short timeout
       so it can be triggered sometimes on heavy load */
    if (ret <= 0)
        return ret;

    mail_index_flush_read_cache(cache->index, cache->filepath, cache->fd,
                    TRUE);
    return 1;
}

static void mail_cache_unlock_file(struct mail_cache *cache)
{
    if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK)
        file_unlock(&cache->file_lock);
    else
        file_dotlock_delete(&cache->dotlock);
}

static int
mail_cache_lock_full(struct mail_cache *cache, bool require_same_reset_id,
             bool nonblock)
{
    const struct mail_index_ext *ext;
    const void *data;
    struct mail_index_view *iview;
    uint32_t reset_id;
    int i, ret;

    i_assert(!cache->locked);

    if (!cache->opened)
        (void)mail_cache_open_and_verify(cache);

    if (MAIL_CACHE_IS_UNUSABLE(cache) ||
        MAIL_INDEX_IS_IN_MEMORY(cache->index) ||
        cache->index->readonly)
        return 0;

    iview = mail_index_view_open(cache->index);
    ext = mail_index_view_get_ext(iview, cache->ext_id);
    reset_id = ext == NULL ? 0 : ext->reset_id;
    mail_index_view_close(&iview);

    if (ext == NULL && require_same_reset_id) {
        /* cache not used */
        return 0;
    }

    for (i = 0; i < 3; i++) {
        if (cache->hdr->file_seq != reset_id &&
            (require_same_reset_id || i == 0)) {
            /* we want the latest cache file */
            if (reset_id < cache->hdr->file_seq) {
                /* either we're still waiting for index to
                   catch up with a cache compression, or
                   that catching up is never going to happen */
                ret = 0;
                break;
            }
            ret = mail_cache_reopen(cache);
            if (ret < 0 || (ret == 0 && require_same_reset_id))
                break;
        }

        if ((ret = mail_cache_lock_file(cache, nonblock)) <= 0) {
            ret = -1;
            break;
        }
        cache->locked = TRUE;

        if (cache->hdr->file_seq == reset_id ||
            !require_same_reset_id) {
            /* got it */
            break;
        }

        /* okay, so it was just compressed. try again. */
        (void)mail_cache_unlock(cache);
        ret = 0;
    }

    if (ret > 0) {
        /* make sure our header is up to date */
        if (cache->file_cache != NULL) {
            file_cache_invalidate(cache->file_cache, 0,
                          sizeof(struct mail_cache_header));
        }
        if (mail_cache_map(cache, 0, 0, &data) > 0)
            cache->hdr_copy = *cache->hdr;
        else {
            (void)mail_cache_unlock(cache);
            ret = -1;
        }
    }

    i_assert((ret <= 0 && !cache->locked) || (ret > 0 && cache->locked));
    return ret;
}

int mail_cache_lock(struct mail_cache *cache, bool require_same_reset_id)
{
    return mail_cache_lock_full(cache, require_same_reset_id, FALSE);
}

int mail_cache_try_lock(struct mail_cache *cache)
{
    return mail_cache_lock_full(cache, FALSE, TRUE);
}

int mail_cache_unlock(struct mail_cache *cache)
{
    int ret = 0;

    i_assert(cache->locked);

    if (cache->field_header_write_pending)
                ret = mail_cache_header_fields_update(cache);

    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
        /* we found it to be broken during the lock. just clean up. */
        cache->hdr_modified = FALSE;
        cache->locked = FALSE;
        return -1;
    }

    if (cache->hdr_modified) {
        cache->hdr_modified = FALSE;
        if (mail_cache_write(cache, &cache->hdr_copy,
                     sizeof(cache->hdr_copy), 0) < 0)
            ret = -1;
        cache->hdr_ro_copy = cache->hdr_copy;
        mail_cache_update_need_compress(cache);
    }

    if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
        if (fdatasync(cache->fd) < 0)
            mail_cache_set_syscall_error(cache, "fdatasync()");
    }

    cache->locked = FALSE;
    mail_cache_unlock_file(cache);
    return ret;
}

int mail_cache_write(struct mail_cache *cache, const void *data, size_t size,
             uoff_t offset)
{
    i_assert(cache->locked);

    if (pwrite_full(cache->fd, data, size, offset) < 0) {
        mail_cache_set_syscall_error(cache, "pwrite_full()");
        return -1;
    }

    if (cache->file_cache != NULL)
        file_cache_write(cache->file_cache, data, size, offset);
    return 0;
}

int mail_cache_append(struct mail_cache *cache, const void *data, size_t size,
              uint32_t *offset)
{
    struct stat st;

    if (*offset == 0) {
        if (fstat(cache->fd, &st) < 0) {
            if (!ESTALE_FSTAT(errno))
                mail_cache_set_syscall_error(cache, "fstat()");
            return -1;
        }
        *offset = st.st_size;
    }
    if (*offset > (uint32_t)-1 || (uint32_t)-1 - *offset < size) {
        mail_cache_set_corrupted(cache, "Cache file too large");
        return -1;
    }
    if (mail_cache_write(cache, data, size, *offset) < 0)
        return -1;

    /* FIXME: this is updated only so that older Dovecot versions (<=v2.1)
       can read this file. we can remove this later. */
    cache->hdr_modified = TRUE;
    cache->hdr_copy.backwards_compat_used_file_size = *offset + size;
    return 0;
}

bool mail_cache_exists(struct mail_cache *cache)
{
    return !MAIL_CACHE_IS_UNUSABLE(cache);
}

struct mail_cache_view *
mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview)
{
    struct mail_cache_view *view;

    view = i_new(struct mail_cache_view, 1);
    view->cache = cache;
    view->view = iview;
    view->cached_exists_buf =
        buffer_create_dynamic(default_pool,
                      cache->file_fields_count + 10);
    return view;
}

void mail_cache_view_close(struct mail_cache_view **_view)
{
    struct mail_cache_view *view = *_view;

    i_assert(view->trans_view == NULL);

    *_view = NULL;
    if (view->cache->field_header_write_pending &&
        !view->cache->compressing)
                (void)mail_cache_header_fields_update(view->cache);

    buffer_free(&view->cached_exists_buf);
    i_free(view);
}

void mail_cache_view_update_cache_decisions(struct mail_cache_view *view,
                        bool update)
{
    view->no_decision_updates = !update;
}

uint32_t mail_cache_get_first_new_seq(struct mail_index_view *view)
{
    const struct mail_index_header *idx_hdr;
    uint32_t first_new_seq, message_count;

    idx_hdr = mail_index_get_header(view);
    if (idx_hdr->day_first_uid[7] == 0)
        return 1;

    if (!mail_index_lookup_seq_range(view, idx_hdr->day_first_uid[7],
                     (uint32_t)-1, &first_new_seq,
                     &message_count)) {
        /* all messages are too old */
        return message_count+1;
    }
    return first_new_seq;
}