src/lib-index/mail-cache.c

	mail-cache.c revision 2131ef7a3390f15ea6a958256ea54908f1096350
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina/* Copyright (c) 2003-2007 Dovecot authors, see the included COPYING file */
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "lib.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "array.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "buffer.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "hash.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "nfs-workarounds.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "file-cache.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "mmap-util.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "write-full.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include "mail-cache-private.h"
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina#include <unistd.h>
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březinavoid mail_cache_set_syscall_error(struct mail_cache *cache,
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina                  const char *function)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina{
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    i_assert(function != NULL);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (ENOSPACE(errno)) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        cache->index->nodiskspace = TRUE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        return;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    mail_index_set_error(cache->index,
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina                 "%s failed with index cache file %s: %m",
e880949305cee3aca79441fe6113a9d79e7c98f2Jakub Hrozek                 function, cache->filepath);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina}
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2a25713afc6beefb11a799903a43f695c5d7a4f9Adam Tkacvoid mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina{
769347ad4d35d43488eb98f980143495b0db415dStef Walter    va_list va;
769347ad4d35d43488eb98f980143495b0db415dStef Walter
769347ad4d35d43488eb98f980143495b0db415dStef Walter    (void)unlink(cache->filepath);
769347ad4d35d43488eb98f980143495b0db415dStef Walter
769347ad4d35d43488eb98f980143495b0db415dStef Walter    /* mark the cache as unusable */
769347ad4d35d43488eb98f980143495b0db415dStef Walter    cache->hdr = NULL;
769347ad4d35d43488eb98f980143495b0db415dStef Walter
769347ad4d35d43488eb98f980143495b0db415dStef Walter    va_start(va, fmt);
b76419cf8830440b46c20a15585562343c7b1924Jakub Hrozek    t_push();
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    mail_index_set_error(cache->index, "Corrupted index cache file %s: %s",
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina                 cache->filepath, t_strdup_vprintf(fmt, va));
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    t_pop();
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    va_end(va);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina}
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březinavoid mail_cache_file_close(struct mail_cache *cache)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina{
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->mmap_base != NULL) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        if (munmap(cache->mmap_base, cache->mmap_length) < 0)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina            mail_cache_set_syscall_error(cache, "munmap()");
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->file_cache != NULL)
83a79d93035c2d75a1941f3b54426119174044a0Pavel Březina        file_cache_set_fd(cache->file_cache, -1);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    cache->mmap_base = NULL;
65976ea5e9767bfaced81dfb97dc87d59f50b57eSimo Sorce    cache->data = NULL;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    cache->hdr = NULL;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    cache->mmap_length = 0;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    cache->last_field_header_offset = 0;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->file_lock != NULL)
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov        file_lock_free(&cache->file_lock);
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    cache->locked = FALSE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->fd != -1) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        if (close(cache->fd) < 0)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina            mail_cache_set_syscall_error(cache, "close()");
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        cache->fd = -1;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina}
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březinastatic void mail_cache_init_file_cache(struct mail_cache *cache)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina{
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    struct stat st;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->file_cache == NULL)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        return;
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->index->nfs_flush) {
8bccd95e275fae760a991da394235e4e70e57bbdMichal Zidek        nfs_flush_attr_cache_fd(cache->filepath, cache->fd);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        nfs_flush_read_cache(cache->filepath, cache->fd,
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina                     F_UNLCK, FALSE);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
07e941c1bbdc752142bbd3b838c540bc7ecd0ed7Stef Walter
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    file_cache_set_fd(cache->file_cache, cache->fd);
04e870d99e72aa3160bdb6ab05d986fb4005c3edPavel Březina
4f3a9d837a55b49448eca3c713c85a406207e523Simo Sorce    if (fstat(cache->fd, &st) == 0)
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina        file_cache_set_size(cache->file_cache, st.st_size);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    else if (errno != ESTALE)
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov        mail_cache_set_syscall_error(cache, "fstat()");
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina    cache->st_ino = st.st_ino;
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina    cache->st_dev = st.st_dev;
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina}
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashovstatic bool mail_cache_need_reopen(struct mail_cache *cache)
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina{
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina    struct stat st;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina        if (cache->need_compress_file_seq != 0) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina            /* we're waiting for compression */
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina            return FALSE;
e3bb7b3fda4697fa9c6f80107cd01dd04a20c85fPetr Cech        }
2a25713afc6beefb11a799903a43f695c5d7a4f9Adam Tkac        if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) {
2a25713afc6beefb11a799903a43f695c5d7a4f9Adam Tkac            /* disabled */
2a25713afc6beefb11a799903a43f695c5d7a4f9Adam Tkac            return FALSE;
2a25713afc6beefb11a799903a43f695c5d7a4f9Adam Tkac        }
2a25713afc6beefb11a799903a43f695c5d7a4f9Adam Tkac    }
2a25713afc6beefb11a799903a43f695c5d7a4f9Adam Tkac
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->fd == -1)
3bea01f01d76e1e95a8239c0d3f67073992136a1Jan Zeleny        return TRUE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    /* see if the file has changed */
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->index->nfs_flush)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        nfs_flush_attr_cache(cache->filepath, TRUE);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (nfs_safe_stat(cache->filepath, &st) < 0) {
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov        mail_cache_set_syscall_error(cache, "stat()");
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina        return TRUE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->index->nfs_flush) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        /* if the old file has been deleted, the new file may have
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina           the same inode as the old one, but we'll catch this by
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina           flushing attribute cache and seeing if it fails with
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina           ESTALE */
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        if (!nfs_flush_attr_cache_fd(cache->filepath, cache->fd))
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina            return TRUE;
3bea01f01d76e1e95a8239c0d3f67073992136a1Jan Zeleny    }
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina    return st.st_ino != cache->st_ino ||
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina        !CMP_DEV_T(st.st_dev, cache->st_dev);
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina}
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březinaint mail_cache_reopen(struct mail_cache *cache)
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina{
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina    struct mail_index_view *view;
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina    const struct mail_index_ext *ext;
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina    i_assert(!cache->locked);
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina    if (!mail_cache_need_reopen(cache)) {
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina        /* reopening does no good */
52e3ee5c5ff2c5a4341041826a803ad42d2b2de7Pavel Březina        return 0;
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina    }
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina    mail_cache_file_close(cache);
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina    cache->fd = nfs_safe_open(cache->filepath,
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina                  cache->index->readonly ? O_RDONLY : O_RDWR);
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina    if (cache->fd == -1) {
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina        if (errno == ENOENT)
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina            cache->need_compress_file_seq = 0;
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina        else
a5f300adf19ec9c3087c62bd93a5175db799687aPavel Březina            mail_cache_set_syscall_error(cache, "open()");
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina        return -1;
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    }
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina    mail_cache_init_file_cache(cache);
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina    if (mail_cache_map(cache, 0, 0) < 0)
0528fdec17d0031996e919fcd852459e86592c35Jakub Hrozek        return -1;
909a86af4eb99f5d311d7136cab78dca535ae304Sumit Bose
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    if (mail_cache_header_fields_read(cache) < 0)
909a86af4eb99f5d311d7136cab78dca535ae304Sumit Bose        return -1;
909a86af4eb99f5d311d7136cab78dca535ae304Sumit Bose
909a86af4eb99f5d311d7136cab78dca535ae304Sumit Bose    view = mail_index_view_open(cache->index);
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    ext = mail_index_view_get_ext(view, cache->ext_id);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (ext == NULL || cache->hdr->file_seq != ext->reset_id) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        /* still different - maybe a race condition or maybe the
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina           file_seq really is corrupted. either way, this shouldn't
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina           happen often so we'll just mark cache to be compressed
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina           later which fixes this. */
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina        cache->need_compress_file_seq = cache->hdr->file_seq;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        mail_index_view_close(&view);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        return 0;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    mail_index_view_close(&view);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));
cb75b275d15beedd1fdecc1f8ced657fba282218Lukas Slebodnik    return 1;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina}
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
ac40d2f2b2b2fc35c95389f5e28febd580bd2b7aJakub Hrozekstatic bool mail_cache_verify_header(struct mail_cache *cache)
ac40d2f2b2b2fc35c95389f5e28febd580bd2b7aJakub Hrozek{
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    const struct mail_cache_header *hdr = cache->data;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    /* check that the header is still ok */
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->mmap_length < sizeof(struct mail_cache_header)) {
cb75b275d15beedd1fdecc1f8ced657fba282218Lukas Slebodnik        mail_cache_set_corrupted(cache, "File too small");
ac40d2f2b2b2fc35c95389f5e28febd580bd2b7aJakub Hrozek        return FALSE;
f37e795cd16310759dc9741c1ab1323b287a9101Fabiano Fidêncio    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (hdr->version != MAIL_CACHE_VERSION) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        /* version changed - upgrade silently */
677a31351c80453d9ce006481364399a96312052René Genz        return FALSE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (hdr->compat_sizeof_uoff_t != sizeof(uoff_t)) {
458f5245dd5130d12666cce6faf8ef1ec7f80169Pavel Reichl        /* architecture change - handle silently(?) */
458f5245dd5130d12666cce6faf8ef1ec7f80169Pavel Reichl        return FALSE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (hdr->indexid != cache->index->indexid) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        /* index id changed - handle silently */
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        return FALSE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (hdr->file_seq == 0) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        mail_cache_set_corrupted(cache, "file_seq is 0");
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        return FALSE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    /* only check the header if we're locked */
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (!cache->locked)
9e2c64c6d4f5560e27207193efea6536a566865eMichal Zidek        return TRUE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
677a31351c80453d9ce006481364399a96312052René Genz    if (hdr->used_file_size < sizeof(struct mail_cache_header)) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        mail_cache_set_corrupted(cache, "used_file_size too small");
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        return FALSE;
cb75b275d15beedd1fdecc1f8ced657fba282218Lukas Slebodnik    }
cb75b275d15beedd1fdecc1f8ced657fba282218Lukas Slebodnik    if ((hdr->used_file_size % sizeof(uint32_t)) != 0) {
3f9e2c24dbc14b2eafbe4f5a5ee16fe9af3c3f75Jakub Hrozek        mail_cache_set_corrupted(cache, "used_file_size not aligned");
ac40d2f2b2b2fc35c95389f5e28febd580bd2b7aJakub Hrozek        return FALSE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->mmap_base != NULL &&
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        hdr->used_file_size > cache->mmap_length) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        mail_cache_set_corrupted(cache, "used_file_size too large");
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        return FALSE;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    }
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    return TRUE;
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov}
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březinaint mail_cache_map(struct mail_cache *cache, size_t offset, size_t size)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina{
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    ssize_t ret;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    cache->remap_counter++;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (size == 0)
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        size = sizeof(struct mail_cache_header);
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina    if (cache->file_cache != NULL) {
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        cache->data = NULL;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina        cache->hdr = NULL;
2827b0d03f7b6bafa504d22a5d7ca39cbda048b3Pavel Březina
        ret = file_cache_read(cache->file_cache, offset, size);
        if (ret < 0) {
                        /* In case of ESTALE we'll simply fail without error
                           messages. The caller will then just have to
                           fallback to generating the value itself.

                           We can't simply reopen the cache flie, because
                           using it requires also having updated file
                           offsets. */
                        if (errno != ESTALE)
                                mail_cache_set_syscall_error(cache, "read()");
            return -1;
        }

        cache->data = file_cache_get_map(cache->file_cache,
                         &cache->mmap_length);

        if (offset == 0) {
            if (!mail_cache_verify_header(cache)) {
                cache->need_compress_file_seq =
                    !MAIL_CACHE_IS_UNUSABLE(cache) &&
                    cache->hdr->file_seq != 0 ?
                    cache->hdr->file_seq : 0;
                return -1;
            }
            memcpy(&cache->hdr_ro_copy, cache->data,
                   sizeof(cache->hdr_ro_copy));
        }
        cache->hdr = &cache->hdr_ro_copy;
        return 0;
    }

    if (offset < cache->mmap_length &&
        size <= cache->mmap_length - offset) {
        /* already mapped */
        return 0;
    }

    if (cache->mmap_base != NULL) {
        if (munmap(cache->mmap_base, cache->mmap_length) < 0)
            mail_cache_set_syscall_error(cache, "munmap()");
    } else {
        if (cache->fd == -1) {
            /* unusable, waiting for compression or
               index is in memory */
            i_assert(cache->need_compress_file_seq != 0 ||
                 MAIL_INDEX_IS_IN_MEMORY(cache->index));
            return -1;
        }
    }

    /* map the whole file */
    cache->hdr = NULL;
    cache->mmap_length = 0;

    cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length);
    if (cache->mmap_base == MAP_FAILED) {
        cache->mmap_base = NULL;
        cache->data = NULL;
        mail_cache_set_syscall_error(cache, "mmap()");
        return -1;
    }
    cache->data = cache->mmap_base;

    if (!mail_cache_verify_header(cache)) {
        cache->need_compress_file_seq =
            !MAIL_CACHE_IS_UNUSABLE(cache) &&
            cache->hdr->file_seq != 0 ?
            cache->hdr->file_seq : 0;
        return -1;
    }

    cache->hdr = cache->data;
    return 0;
}

static int mail_cache_try_open(struct mail_cache *cache)
{
    cache->opened = TRUE;

    if (MAIL_INDEX_IS_IN_MEMORY(cache->index))
        return 0;

    cache->fd = nfs_safe_open(cache->filepath,
                  cache->index->readonly ? O_RDONLY : O_RDWR);
    if (cache->fd == -1) {
        if (errno == ENOENT) {
            cache->need_compress_file_seq = 0;
            return 0;
        }

        mail_cache_set_syscall_error(cache, "open()");
        return -1;
    }

    mail_cache_init_file_cache(cache);

    if (mail_cache_map(cache, 0, sizeof(struct mail_cache_header)) < 0)
        return -1;

    return 1;
}

int mail_cache_open_and_verify(struct mail_cache *cache)
{
    int ret;

    ret = mail_cache_try_open(cache);
    if (ret > 0)
        ret = mail_cache_header_fields_read(cache);
    if (ret < 0) {
        /* failed for some reason - doesn't really matter,
           it's disabled for now. */
        mail_cache_file_close(cache);
    }
    return ret;
}

static struct mail_cache *mail_cache_alloc(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = i_new(struct mail_cache, 1);
    cache->index = index;
    cache->fd = -1;
    cache->filepath =
        i_strconcat(index->filepath, MAIL_CACHE_FILE_SUFFIX, NULL);
    cache->field_pool = pool_alloconly_create("Cache fields", 1024);
    cache->field_name_hash =
        hash_create(default_pool, cache->field_pool, 0,
                strcase_hash, (hash_cmp_callback_t *)strcasecmp);

    cache->dotlock_settings.use_excl_lock = index->use_excl_dotlocks;
    cache->dotlock_settings.nfs_flush = index->nfs_flush;
    cache->dotlock_settings.timeout = MAIL_CACHE_LOCK_TIMEOUT;
    cache->dotlock_settings.stale_timeout = MAIL_CACHE_LOCK_CHANGE_TIMEOUT;

    if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
        if (index->mmap_disable)
            cache->file_cache = file_cache_new(-1);
    }

    cache->ext_id =
        mail_index_ext_register(index, "cache", 0,
                    sizeof(uint32_t), sizeof(uint32_t));
    mail_index_register_expunge_handler(index, cache->ext_id, FALSE,
                        mail_cache_expunge_handler, cache);
    mail_index_register_sync_handler(index, cache->ext_id,
                     mail_cache_sync_handler,
                                         MAIL_INDEX_SYNC_HANDLER_FILE |
                                         MAIL_INDEX_SYNC_HANDLER_HEAD |
                     (cache->file_cache == NULL ? 0 :
                      MAIL_INDEX_SYNC_HANDLER_VIEW));

    if (cache->file_cache != NULL) {
        mail_index_register_sync_lost_handler(index,
            mail_cache_sync_lost_handler);
    }
    return cache;
}

struct mail_cache *mail_cache_open_or_create(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = mail_cache_alloc(index);
    return cache;
}

struct mail_cache *mail_cache_create(struct mail_index *index)
{
    struct mail_cache *cache;

    cache = mail_cache_alloc(index);
    if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
        if (unlink(cache->filepath) < 0 && errno != ENOENT)
            mail_cache_set_syscall_error(cache, "unlink()");
    }
    return cache;
}

void mail_cache_free(struct mail_cache **_cache)
{
    struct mail_cache *cache = *_cache;

    *_cache = NULL;
    if (cache->file_cache != NULL) {
        mail_index_unregister_sync_lost_handler(cache->index,
            mail_cache_sync_lost_handler);

        file_cache_free(&cache->file_cache);
    }

    mail_index_unregister_expunge_handler(cache->index, cache->ext_id);
    mail_index_unregister_sync_handler(cache->index, cache->ext_id);

    mail_cache_file_close(cache);

    hash_destroy(&cache->field_name_hash);
    pool_unref(&cache->field_pool);
    i_free(cache->field_file_map);
    i_free(cache->file_field_map);
    i_free(cache->fields);
    i_free(cache->filepath);
    i_free(cache);
}

void mail_cache_flush_read_cache(struct mail_cache *cache, bool just_locked)
{
    if (!cache->index->nfs_flush)
        return;

    /* Assume flock() is independent of fcntl() locks. This isn't true
       with Linux 2.6 NFS, but with it there's no point in using flock() */
    if (cache->locked &&
        cache->index->lock_method == FILE_LOCK_METHOD_FCNTL) {
        nfs_flush_read_cache(cache->filepath, cache->fd,
                     F_WRLCK, just_locked);
    } else {
        nfs_flush_read_cache(cache->filepath, cache->fd,
                     F_UNLCK, FALSE);
    }
}

static int mail_cache_lock_file(struct mail_cache *cache)
{
    int ret;

    if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
        i_assert(cache->file_lock == NULL);
        ret = mail_index_lock_fd(cache->index, cache->filepath,
                     cache->fd, F_WRLCK,
                     MAIL_CACHE_LOCK_TIMEOUT,
                     &cache->file_lock);
    } else {
        i_assert(cache->dotlock == NULL);
        ret = file_dotlock_create(&cache->dotlock_settings,
                      cache->filepath, 0, &cache->dotlock);
        if (ret < 0) {
            mail_cache_set_syscall_error(cache,
                             "file_dotlock_create()");
        }
    }

    if (ret <= 0)
        return ret;

    if (cache->index->nfs_flush)
        nfs_flush_attr_cache_fd(cache->filepath, cache->fd);
    mail_cache_flush_read_cache(cache, TRUE);
    return 1;
}

static void mail_cache_unlock_file(struct mail_cache *cache)
{
    if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK)
        file_unlock(&cache->file_lock);
    else
        (void)file_dotlock_delete(&cache->dotlock);
}

int mail_cache_lock(struct mail_cache *cache, bool require_same_reset_id)
{
    const struct mail_index_ext *ext;
    struct mail_index_view *iview;
    uint32_t reset_id;
    int i, ret;

    i_assert(!cache->locked);

    if (!cache->opened)
        (void)mail_cache_open_and_verify(cache);

    if (MAIL_CACHE_IS_UNUSABLE(cache) ||
        MAIL_INDEX_IS_IN_MEMORY(cache->index))
        return 0;

    iview = mail_index_view_open(cache->index);
    ext = mail_index_view_get_ext(iview, cache->ext_id);
    reset_id = ext == NULL ? 0 : ext->reset_id;
    mail_index_view_close(&iview);

    if (ext == NULL && require_same_reset_id) {
        /* cache not used */
        return 0;
    }

    for (i = 0; i < 3; i++) {
        if (cache->hdr->file_seq != reset_id &&
            (require_same_reset_id || i == 0)) {
            /* we want the latest cache file */
            ret = mail_cache_reopen(cache);
            if (ret < 0 || (ret == 0 && require_same_reset_id))
                break;
        }

        ret = mail_cache_lock_file(cache);
        if (ret <= 0)
            break;
        cache->locked = TRUE;

        if (cache->hdr->file_seq == reset_id ||
            !require_same_reset_id) {
            /* got it */
            break;
        }

        /* okay, so it was just compressed. try again. */
        (void)mail_cache_unlock(cache);
        ret = 0;
    }

    if (ret > 0) {
        /* make sure our header is up to date */
        if (cache->file_cache != NULL) {
            file_cache_invalidate(cache->file_cache, 0,
                          sizeof(struct mail_cache_header));
        }
        if (mail_cache_map(cache, 0, 0) == 0)
            cache->hdr_copy = *cache->hdr;
        else {
            (void)mail_cache_unlock(cache);
            ret = -1;
        }
    }

    i_assert((ret <= 0 && !cache->locked) || (ret > 0 && cache->locked));
    return ret;
}

static void mail_cache_update_need_compress(struct mail_cache *cache)
{
    const struct mail_cache_header *hdr = cache->hdr;
    unsigned int cont_percentage;
    uoff_t max_del_space;

        cont_percentage = hdr->continued_record_count * 100 /
        (cache->index->map->rec_map->records_count == 0 ? 1 :
         cache->index->map->rec_map->records_count);
    if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE &&
        hdr->used_file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) {
        /* too many continued rows, compress */
        cache->need_compress_file_seq = hdr->file_seq;
    }

    /* see if we've reached the max. deleted space in file */
    max_del_space = hdr->used_file_size / 100 *
        MAIL_CACHE_COMPRESS_PERCENTAGE;
    if (hdr->deleted_space >= max_del_space &&
        hdr->used_file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE)
        cache->need_compress_file_seq = hdr->file_seq;
}

int mail_cache_unlock(struct mail_cache *cache)
{
    int ret = 0;

    i_assert(cache->locked);

    if (cache->field_header_write_pending)
                ret = mail_cache_header_fields_update(cache);

    cache->locked = FALSE;

    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
        /* we found it to be broken during the lock. just clean up. */
        cache->hdr_modified = FALSE;
        return -1;
    }

    if (cache->hdr_modified) {
        cache->hdr_modified = FALSE;
        if (mail_cache_write(cache, &cache->hdr_copy,
                     sizeof(cache->hdr_copy), 0) < 0)
            ret = -1;
        cache->hdr_ro_copy = cache->hdr_copy;
        mail_cache_update_need_compress(cache);
    }

    if (cache->index->nfs_flush) {
        if (fdatasync(cache->fd) < 0)
            mail_cache_set_syscall_error(cache, "fdatasync()");
    }

    mail_cache_unlock_file(cache);
    return ret;
}

int mail_cache_write(struct mail_cache *cache, const void *data, size_t size,
             uoff_t offset)
{
    if (pwrite_full(cache->fd, data, size, offset) < 0) {
        mail_cache_set_syscall_error(cache, "pwrite_full()");
        return -1;
    }

    if (cache->file_cache != NULL) {
        file_cache_write(cache->file_cache, data, size, offset);

        /* data pointer may change if file cache was grown */
        cache->data = file_cache_get_map(cache->file_cache,
                         &cache->mmap_length);
    }
    return 0;
}

struct mail_cache_view *
mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview)
{
    struct mail_cache_view *view;

    view = i_new(struct mail_cache_view, 1);
    view->cache = cache;
    view->view = iview;
    i_array_init(&view->looping_offsets, 32);
    view->cached_exists_buf =
        buffer_create_dynamic(default_pool,
                      cache->file_fields_count + 10);
    return view;
}

void mail_cache_view_close(struct mail_cache_view *view)
{
    i_assert(view->trans_view == NULL);

    if (view->cache->field_header_write_pending)
                (void)mail_cache_header_fields_update(view->cache);

    array_free(&view->looping_offsets);
    buffer_free(&view->cached_exists_buf);
    i_free(view);
}