src/lib-index/mail-cache.c

	mail-cache.c revision 97144a346898fb62f9fae44fa5c076986553c66b
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen/* Copyright (c) 2003-2012 Dovecot authors, see the included COPYING file */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "lib.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "array.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "buffer.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "hash.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "nfs-workarounds.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "file-cache.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "mmap-util.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "read-full.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "write-full.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include "mail-cache-private.h"
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#include <unistd.h>
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen#define MAIL_CACHE_MIN_HEADER_READ_SIZE 4096
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainenvoid mail_cache_set_syscall_error(struct mail_cache *cache,
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen                  const char *function)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    mail_index_file_set_syscall_error(cache->index, cache->filepath,
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen                      function);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen}
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainenstatic void mail_cache_unlink(struct mail_cache *cache)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (!cache->index->readonly)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        (void)unlink(cache->filepath);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen}
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainenvoid mail_cache_reset(struct mail_cache *cache)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    mail_cache_unlink(cache);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    /* mark the cache as unusable */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->hdr = NULL;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen}
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainenvoid mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    va_list va;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    mail_cache_reset(cache);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    va_start(va, fmt);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    T_BEGIN {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        mail_index_set_error(cache->index,
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen                     "Corrupted index cache file %s: %s",
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen                     cache->filepath,
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen                     t_strdup_vprintf(fmt, va));
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    } T_END;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    va_end(va);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen}
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainenvoid mail_cache_file_close(struct mail_cache *cache)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (cache->mmap_base != NULL) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        if (munmap(cache->mmap_base, cache->mmap_length) < 0)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            mail_cache_set_syscall_error(cache, "munmap()");
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (cache->file_cache != NULL)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        file_cache_set_fd(cache->file_cache, -1);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->mmap_base = NULL;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->hdr = NULL;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->mmap_length = 0;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->last_field_header_offset = 0;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (cache->file_lock != NULL)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        file_lock_free(&cache->file_lock);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->locked = FALSE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (cache->fd != -1) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        if (close(cache->fd) < 0)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            mail_cache_set_syscall_error(cache, "close()");
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        cache->fd = -1;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen}
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainenstatic void mail_cache_init_file_cache(struct mail_cache *cache)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    struct stat st;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (cache->file_cache == NULL)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        return;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    file_cache_set_fd(cache->file_cache, cache->fd);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (fstat(cache->fd, &st) == 0)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        file_cache_set_size(cache->file_cache, st.st_size);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    else if (!ESTALE_FSTAT(errno))
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        mail_cache_set_syscall_error(cache, "fstat()");
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->st_ino = st.st_ino;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->st_dev = st.st_dev;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen}
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainenstatic bool mail_cache_need_reopen(struct mail_cache *cache)
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    struct stat st;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        if (cache->need_compress_file_seq != 0) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            /* we're waiting for compression */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            return FALSE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            /* disabled */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            return FALSE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (cache->fd == -1)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        return TRUE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    /* see if the file has changed */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        i_assert(!cache->locked);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        nfs_flush_file_handle_cache(cache->filepath);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (nfs_safe_stat(cache->filepath, &st) < 0) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        mail_cache_set_syscall_error(cache, "stat()");
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        return TRUE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (st.st_ino != cache->st_ino ||
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        !CMP_DEV_T(st.st_dev, cache->st_dev)) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        /* file changed */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        return TRUE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        /* if the old file has been deleted, the new file may have
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen           the same inode as the old one. we'll catch this here by
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen           checking if fstat() fails with ESTALE */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        if (fstat(cache->fd, &st) < 0) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            if (ESTALE_FSTAT(errno))
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen                return TRUE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            mail_cache_set_syscall_error(cache, "fstat()");
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            return FALSE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    return FALSE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen}
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainenint mail_cache_reopen(struct mail_cache *cache)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    struct mail_index_view *view;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    const struct mail_index_ext *ext;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    const void *data;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    i_assert(!cache->locked);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (!mail_cache_need_reopen(cache)) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        /* reopening does no good */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        return 0;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    mail_cache_file_close(cache);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    cache->fd = nfs_safe_open(cache->filepath,
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen                  cache->index->readonly ? O_RDONLY : O_RDWR);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (cache->fd == -1) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        if (errno == ENOENT)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen            cache->need_compress_file_seq = 0;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        else
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen            mail_cache_set_syscall_error(cache, "open()");
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        return -1;
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen    }
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    mail_cache_init_file_cache(cache);
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen    if (mail_cache_map(cache, 0, 0, &data) < 0)
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen        return -1;
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen    if (mail_cache_header_fields_read(cache) < 0)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        return -1;
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    view = mail_index_view_open(cache->index);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    ext = mail_index_view_get_ext(view, cache->ext_id);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (ext == NULL || cache->hdr->file_seq != ext->reset_id) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        /* still different - maybe a race condition or maybe the
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen           file_seq really is corrupted. either way, this shouldn't
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen           happen often so we'll just mark cache to be compressed
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen           later which fixes this. */
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen        cache->need_compress_file_seq = cache->hdr->file_seq;
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen        mail_index_view_close(&view);
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen        return 0;
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen    }
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen
04dff721d2a63566a4dbe1c856f8218b6550aa3eTimo Sirainen    mail_index_view_close(&view);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    return 1;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen}
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainenstatic void mail_cache_update_need_compress(struct mail_cache *cache)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen{
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    const struct mail_cache_header *hdr = cache->hdr;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    unsigned int cont_percentage;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    uoff_t max_del_space;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        cont_percentage = hdr->continued_record_count * 100 /
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        (cache->index->map->rec_map->records_count == 0 ? 1 :
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen         cache->index->map->rec_map->records_count);
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE &&
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        hdr->used_file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) {
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        /* too many continued rows, compress */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        cache->need_compress_file_seq = hdr->file_seq;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    }
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    /* see if we've reached the max. deleted space in file */
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    max_del_space = hdr->used_file_size / 100 *
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        MAIL_CACHE_COMPRESS_PERCENTAGE;
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen    if (hdr->deleted_space >= max_del_space &&
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        hdr->used_file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE)
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen        cache->need_compress_file_seq = hdr->file_seq;
}

static bool mail_cache_verify_header(struct mail_cache *cache,
                     const struct mail_cache_header *hdr)
{
    /* check that the header is still ok */
    if (cache->mmap_length < sizeof(struct mail_cache_header)) {
        mail_cache_set_corrupted(cache, "File too small");
        return FALSE;
    }

    if (hdr->version != MAIL_CACHE_VERSION) {
        /* version changed - upgrade silently */
        mail_cache_unlink(cache);
        return FALSE;
    }
    if (hdr->compat_sizeof_uoff_t != sizeof(uoff_t)) {
        /* architecture change - handle silently(?) */
        mail_cache_unlink(cache);
        return FALSE;
    }

    if (hdr->indexid != cache->index->indexid) {
        /* index id changed - handle silently */
        mail_cache_unlink(cache);
        return FALSE;
    }
    if (hdr->file_seq == 0) {
        mail_cache_set_corrupted(cache, "file_seq is 0");
        return FALSE;
    }

    /* only check the header if we're locked */
    if (!cache->locked)
        return TRUE;

    if (hdr->used_file_size < sizeof(struct mail_cache_header)) {
        mail_cache_set_corrupted(cache, "used_file_size too small");
        return FALSE;
    }
    if ((hdr->used_file_size % sizeof(uint32_t)) != 0) {
        mail_cache_set_corrupted(cache, "used_file_size not aligned");
        return FALSE;
    }

    if (cache->mmap_base != NULL &&
        hdr->used_file_size > cache->mmap_length) {
        mail_cache_set_corrupted(cache, "used_file_size too large");
        return FALSE;
    }
    return TRUE;
}

static int
mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size,
              const void *hdr_data, bool copy_hdr)
{
    const struct mail_cache_header *hdr = hdr_data;

    if (offset == 0) {
        /* verify the header validity only with offset=0. this way
           we won't waste time re-verifying it all the time */
        if (!mail_cache_verify_header(cache, hdr)) {
            cache->need_compress_file_seq =
                !MAIL_CACHE_IS_UNUSABLE(cache) &&
                cache->hdr->file_seq != 0 ?
                cache->hdr->file_seq : 0;
            cache->hdr = NULL;
            return -1;
        }
    }
    if (hdr_data != NULL) {
        if (!copy_hdr)
            cache->hdr = hdr;
        else {
            memcpy(&cache->hdr_ro_copy, hdr,
                   sizeof(cache->hdr_ro_copy));
            cache->hdr = &cache->hdr_ro_copy;
        }
        mail_cache_update_need_compress(cache);
    } else {
        i_assert(cache->hdr != NULL);
    }
    i_assert(cache->hdr->file_seq != 0);

    if (offset + size > cache->mmap_length)
        return 0;
    return 1;
}

static int
mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size,
             const void **data_r)
{
    const void *hdr_data;
    void *data;
    ssize_t ret;

    if (cache->read_buf == NULL) {
        cache->read_buf =
            buffer_create_dynamic(default_pool, size);
    } else if (cache->read_offset <= offset &&
           cache->read_offset + cache->read_buf->used >= offset+size) {
        /* already mapped */
        *data_r = CONST_PTR_OFFSET(cache->read_buf->data,
                       offset - cache->read_offset);
        hdr_data = offset == 0 ? *data_r : NULL;
        return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE);
    } else {
        buffer_set_used_size(cache->read_buf, 0);
    }
    if (offset == 0 && size < MAIL_CACHE_MIN_HEADER_READ_SIZE) {
        /* we can usually read the fields header after the cache
           header. we need them both, so try to read them all with one
           pread() call. */
        size = MAIL_CACHE_MIN_HEADER_READ_SIZE;
    }

    data = buffer_append_space_unsafe(cache->read_buf, size);
    ret = pread(cache->fd, data, size, offset);
    if (ret < 0) {
        if (errno != ESTALE)
            mail_cache_set_syscall_error(cache, "read()");

        buffer_set_used_size(cache->read_buf, 0);
        cache->hdr = NULL;
        cache->mmap_length = 0;
        return -1;
    }
    buffer_set_used_size(cache->read_buf, ret);

    cache->read_offset = offset;
    cache->mmap_length = offset + cache->read_buf->used;

    *data_r = data;
    hdr_data = offset == 0 ? *data_r : NULL;
    return mail_cache_map_finish(cache, offset,
                     cache->read_buf->used, hdr_data, TRUE);
}

int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
           const void **data_r)
{
    struct stat st;
    const void *data;
    ssize_t ret;

    if (size == 0)
        size = sizeof(struct mail_cache_header);

    /* verify offset + size before trying to allocate a huge amount of
       memory due to them. note that we may be prefetching more than we
       actually need, so don't fail too early. */
    if ((size > cache->mmap_length || offset + size > cache->mmap_length) &&
        (offset > 0 || size > sizeof(struct mail_cache_header))) {
        if (fstat(cache->fd, &st) < 0) {
            i_error("fstat(%s) failed: %m", cache->filepath);
            return -1;
        }
        if (offset >= (uoff_t)st.st_size) {
            *data_r = NULL;
            return 0;
        }
        if (offset + size > (uoff_t)st.st_size)
            size = st.st_size - offset;
    }

    cache->remap_counter++;
    if (cache->map_with_read)
        return mail_cache_map_with_read(cache, offset, size, data_r);

    if (cache->file_cache != NULL) {
        ret = file_cache_read(cache->file_cache, offset, size);
        if (ret < 0) {
                        /* In case of ESTALE we'll simply fail without error
                           messages. The caller will then just have to
                           fallback to generating the value itself.

                           We can't simply reopen the cache flie, because
                           using it requires also having updated file
                           offsets. */
                        if (errno != ESTALE)
                                mail_cache_set_syscall_error(cache, "read()");
            cache->hdr = NULL;
            return -1;
        }

        data = file_cache_get_map(cache->file_cache,
                      &cache->mmap_length);
        *data_r = offset > cache->mmap_length ? NULL :
            CONST_PTR_OFFSET(data, offset);
        return mail_cache_map_finish(cache, offset, size,
                         offset == 0 ? data : NULL, TRUE);
    }

    if (offset < cache->mmap_length &&
        size <= cache->mmap_length - offset) {
        /* already mapped */
        i_assert(cache->mmap_base != NULL);
        *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset);
        return 1;
    }

    if (cache->mmap_base != NULL) {
        if (munmap(cache->mmap_base, cache->mmap_length) < 0)
            mail_cache_set_syscall_error(cache, "munmap()");
    } else {
        if (cache->fd == -1) {
            /* unusable, waiting for compression or
               index is in memory */
            i_assert(cache->need_compress_file_seq != 0 ||
                 MAIL_INDEX_IS_IN_MEMORY(cache->index));
            return -1;
        }
    }

    /* map the whole file */
    cache->hdr = NULL;
    cache->mmap_length = 0;

    cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length);
    if (cache->mmap_base == MAP_FAILED) {
        cache->mmap_base = NULL;
        mail_cache_set_syscall_error(cache, "mmap()");
        return -1;
    }
    *data_r = offset > cache->mmap_length ? NULL :
        CONST_PTR_OFFSET(cache->mmap_base, offset);
    return mail_cache_map_finish(cache, offset, size,
                     cache->mmap_base, FALSE);
}

static int mail_cache_try_open(struct mail_cache *cache)
{
    const void *data;

    cache->opened = TRUE;

    if (MAIL_INDEX_IS_IN_MEMORY(cache->index))
        return 0;

    cache->fd = nfs_safe_open(cache->filepath,
                  cache->index->readonly ? O_RDONLY : O_RDWR);
    if (cache->fd == -1) {
        if (errno == ENOENT) {
            cache->need_compress_file_seq = 0;
            return 0;
        }

        mail_cache_set_syscall_error(cache, "open()");
        return -1;
    }

    mail_cache_init_file_cache(cache);

    if (mail_cache_map(cache, 0, 0, &data) < 0)
        return -1;
    return 1;
}

int mail_cache_open_and_verify(struct mail_cache *cache)
{
    int ret;

    ret = mail_cache_try_open(cache);
    if (ret > 0)
        ret = mail_cache_header_fields_read(cache);
    if (ret < 0) {
        /* failed for some reason - doesn't really matter,
           it's disabled for now. */
        mail_cache_file_close(cache);
    }
    return ret;
}

static struct mail_cache *mail_cache_alloc(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = i_new(struct mail_cache, 1);
    cache->index = index;
    cache->fd = -1;
    cache->filepath =
        i_strconcat(index->filepath, MAIL_CACHE_FILE_SUFFIX, NULL);
    cache->field_pool = pool_alloconly_create("Cache fields", 2048);
    cache->field_name_hash =
        hash_table_create(default_pool, cache->field_pool, 0,
                  strcase_hash, (hash_cmp_callback_t *)strcasecmp);

    cache->dotlock_settings.use_excl_lock =
        (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
    cache->dotlock_settings.nfs_flush =
        (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;
    cache->dotlock_settings.timeout =
        I_MIN(MAIL_CACHE_LOCK_TIMEOUT, index->max_lock_timeout_secs);
    cache->dotlock_settings.stale_timeout = MAIL_CACHE_LOCK_CHANGE_TIMEOUT;

    if (!MAIL_INDEX_IS_IN_MEMORY(index) &&
        (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0)
        cache->file_cache = file_cache_new(-1);
    cache->map_with_read =
        (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0;

    cache->ext_id =
        mail_index_ext_register(index, "cache", 0,
                    sizeof(uint32_t), sizeof(uint32_t));
    mail_index_register_expunge_handler(index, cache->ext_id, FALSE,
                        mail_cache_expunge_handler, cache);
    mail_index_register_sync_handler(index, cache->ext_id,
                     mail_cache_sync_handler,
                                         MAIL_INDEX_SYNC_HANDLER_FILE |
                                         MAIL_INDEX_SYNC_HANDLER_HEAD |
                     (cache->file_cache == NULL ? 0 :
                      MAIL_INDEX_SYNC_HANDLER_VIEW));

    if (cache->file_cache != NULL) {
        mail_index_register_sync_lost_handler(index,
            mail_cache_sync_lost_handler);
    }
    return cache;
}

struct mail_cache *mail_cache_open_or_create(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = mail_cache_alloc(index);
    return cache;
}

struct mail_cache *mail_cache_create(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = mail_cache_alloc(index);
    if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
        if (unlink(cache->filepath) < 0 && errno != ENOENT)
            mail_cache_set_syscall_error(cache, "unlink()");
    }
    return cache;
}

void mail_cache_free(struct mail_cache **_cache)
{
    struct mail_cache *cache = *_cache;

    *_cache = NULL;
    if (cache->file_cache != NULL) {
        mail_index_unregister_sync_lost_handler(cache->index,
            mail_cache_sync_lost_handler);

        file_cache_free(&cache->file_cache);
    }

    mail_index_unregister_expunge_handler(cache->index, cache->ext_id);
    mail_index_unregister_sync_handler(cache->index, cache->ext_id);

    mail_cache_file_close(cache);

    if (cache->read_buf != NULL)
        buffer_free(&cache->read_buf);
    hash_table_destroy(&cache->field_name_hash);
    pool_unref(&cache->field_pool);
    i_free(cache->field_file_map);
    i_free(cache->file_field_map);
    i_free(cache->fields);
    i_free(cache->filepath);
    i_free(cache);
}

static int mail_cache_lock_file(struct mail_cache *cache, bool nonblock)
{
    unsigned int timeout_secs;
    int ret;

    if (cache->last_lock_failed) {
        /* previous locking failed. don't waste time waiting on it
           again, just try once to see if it's available now. */
        nonblock = TRUE;
    }

    if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
        i_assert(cache->file_lock == NULL);
        timeout_secs = I_MIN(MAIL_CACHE_LOCK_TIMEOUT,
                     cache->index->max_lock_timeout_secs);

        ret = mail_index_lock_fd(cache->index, cache->filepath,
                     cache->fd, F_WRLCK,
                     nonblock ? 0 : timeout_secs,
                     &cache->file_lock);
    } else {
        enum dotlock_create_flags flags =
            nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0;

        i_assert(cache->dotlock == NULL);
        ret = file_dotlock_create(&cache->dotlock_settings,
                      cache->filepath, flags,
                      &cache->dotlock);
        if (ret < 0) {
            mail_cache_set_syscall_error(cache,
                             "file_dotlock_create()");
        }
    }
    cache->last_lock_failed = ret <= 0;

    /* don't bother warning if locking failed due to a timeout. since cache
       updating isn't all that important we're using a very short timeout
       so it can be triggered sometimes on heavy load */
    if (ret <= 0)
        return ret;

    mail_index_flush_read_cache(cache->index, cache->filepath, cache->fd,
                    TRUE);
    return 1;
}

static void mail_cache_unlock_file(struct mail_cache *cache)
{
    if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK)
        file_unlock(&cache->file_lock);
    else
        (void)file_dotlock_delete(&cache->dotlock);
}

static int
mail_cache_lock_full(struct mail_cache *cache, bool require_same_reset_id,
             bool nonblock)
{
    const struct mail_index_ext *ext;
    const void *data;
    struct mail_index_view *iview;
    uint32_t reset_id;
    int i, ret;

    i_assert(!cache->locked);

    if (!cache->opened)
        (void)mail_cache_open_and_verify(cache);

    if (MAIL_CACHE_IS_UNUSABLE(cache) ||
        MAIL_INDEX_IS_IN_MEMORY(cache->index) ||
        cache->index->readonly)
        return 0;

    iview = mail_index_view_open(cache->index);
    ext = mail_index_view_get_ext(iview, cache->ext_id);
    reset_id = ext == NULL ? 0 : ext->reset_id;
    mail_index_view_close(&iview);

    if (ext == NULL && require_same_reset_id) {
        /* cache not used */
        return 0;
    }

    for (i = 0; i < 3; i++) {
        if (cache->hdr->file_seq != reset_id &&
            (require_same_reset_id || i == 0)) {
            /* we want the latest cache file */
            if (reset_id < cache->hdr->file_seq) {
                /* either we're still waiting for index to
                   catch up with a cache compression, or
                   that catching up is never going to happen */
                ret = 0;
                break;
            }
            ret = mail_cache_reopen(cache);
            if (ret < 0 || (ret == 0 && require_same_reset_id))
                break;
        }

        if ((ret = mail_cache_lock_file(cache, nonblock)) <= 0) {
            ret = -1;
            break;
        }
        cache->locked = TRUE;

        if (cache->hdr->file_seq == reset_id ||
            !require_same_reset_id) {
            /* got it */
            break;
        }

        /* okay, so it was just compressed. try again. */
        (void)mail_cache_unlock(cache);
        ret = 0;
    }

    if (ret > 0) {
        /* make sure our header is up to date */
        if (cache->file_cache != NULL) {
            file_cache_invalidate(cache->file_cache, 0,
                          sizeof(struct mail_cache_header));
        }
        if (cache->read_buf != NULL)
            buffer_set_used_size(cache->read_buf, 0);
        if (mail_cache_map(cache, 0, 0, &data) > 0)
            cache->hdr_copy = *cache->hdr;
        else {
            (void)mail_cache_unlock(cache);
            ret = -1;
        }
    }

    i_assert((ret <= 0 && !cache->locked) || (ret > 0 && cache->locked));
    return ret;
}

int mail_cache_lock(struct mail_cache *cache, bool require_same_reset_id)
{
    return mail_cache_lock_full(cache, require_same_reset_id, FALSE);
}

int mail_cache_try_lock(struct mail_cache *cache)
{
    return mail_cache_lock_full(cache, FALSE, TRUE);
}

int mail_cache_unlock(struct mail_cache *cache)
{
    int ret = 0;

    i_assert(cache->locked);

    if (cache->field_header_write_pending)
                ret = mail_cache_header_fields_update(cache);

    cache->locked = FALSE;

    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
        /* we found it to be broken during the lock. just clean up. */
        cache->hdr_modified = FALSE;
        return -1;
    }

    if (cache->hdr_modified) {
        cache->hdr_modified = FALSE;
        if (mail_cache_write(cache, &cache->hdr_copy,
                     sizeof(cache->hdr_copy), 0) < 0)
            ret = -1;
        cache->hdr_ro_copy = cache->hdr_copy;
        mail_cache_update_need_compress(cache);
    }

    if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
        if (fdatasync(cache->fd) < 0)
            mail_cache_set_syscall_error(cache, "fdatasync()");
    }

    mail_cache_unlock_file(cache);
    return ret;
}

int mail_cache_write(struct mail_cache *cache, const void *data, size_t size,
             uoff_t offset)
{
    if (pwrite_full(cache->fd, data, size, offset) < 0) {
        mail_cache_set_syscall_error(cache, "pwrite_full()");
        return -1;
    }

    if (cache->file_cache != NULL)
        file_cache_write(cache->file_cache, data, size, offset);
    if (cache->read_buf != NULL)
        buffer_set_used_size(cache->read_buf, 0);
    return 0;
}

bool mail_cache_exists(struct mail_cache *cache)
{
    return !MAIL_CACHE_IS_UNUSABLE(cache);
}

struct mail_cache_view *
mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview)
{
    struct mail_cache_view *view;

    view = i_new(struct mail_cache_view, 1);
    view->cache = cache;
    view->view = iview;
    view->cached_exists_buf =
        buffer_create_dynamic(default_pool,
                      cache->file_fields_count + 10);
    return view;
}

void mail_cache_view_close(struct mail_cache_view **_view)
{
    struct mail_cache_view *view = *_view;

    i_assert(view->trans_view == NULL);

    *_view = NULL;
    if (view->cache->field_header_write_pending &&
        !view->cache->compressing)
                (void)mail_cache_header_fields_update(view->cache);

    buffer_free(&view->cached_exists_buf);
    i_free(view);
}

void mail_cache_view_update_cache_decisions(struct mail_cache_view *view,
                        bool update)
{
    view->no_decision_updates = !update;
}

uint32_t mail_cache_get_first_new_seq(struct mail_index_view *view)
{
    const struct mail_index_header *idx_hdr;
    uint32_t first_new_seq, message_count;

    idx_hdr = mail_index_get_header(view);
    if (idx_hdr->day_first_uid[7] == 0)
        return 1;

    if (!mail_index_lookup_seq_range(view, idx_hdr->day_first_uid[7],
                     (uint32_t)-1, &first_new_seq,
                     &message_count)) {
        /* all messages are too old */
        return message_count+1;
    }
    return first_new_seq;
}