src/lib-index/mail-cache.c

	mail-cache.c revision bba52ecbb0cfb6585f1a4ff29695dd2d27af98d2
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2003-2012 Dovecot authors, see the included COPYING file */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
16f816d3f3c32ae3351834253f52ddd0212bcbf3Timo Sirainen#include "array.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "buffer.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "hash.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "nfs-workarounds.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "file-cache.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mmap-util.h"
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen#include "read-full.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "write-full.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-cache-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <unistd.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen#define MAIL_CACHE_MIN_HEADER_READ_SIZE 4096
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenvoid mail_cache_set_syscall_error(struct mail_cache *cache,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                  const char *function)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
e8fd7988ec183fb6c104aed19a61f1a096c51d34Timo Sirainen    mail_index_file_set_syscall_error(cache->index, cache->filepath,
e8fd7988ec183fb6c104aed19a61f1a096c51d34Timo Sirainen                      function);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainenstatic void mail_cache_unlink(struct mail_cache *cache)
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen{
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen    if (!cache->index->readonly)
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen        (void)unlink(cache->filepath);
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen}
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainenvoid mail_cache_reset(struct mail_cache *cache)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen    mail_cache_unlink(cache);
ad850190d946d34966a56838cfdb216e021b5b5fTimo Sirainen    /* mark the cache as unusable */
ad850190d946d34966a56838cfdb216e021b5b5fTimo Sirainen    cache->hdr = NULL;
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen}
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainenvoid mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen{
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen    va_list va;
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen
d35fee8d1e5e31614dba5e64d45ed23c7d6bfa53Timo Sirainen    mail_cache_reset(cache);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    va_start(va, fmt);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen    T_BEGIN {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen        mail_index_set_error(cache->index,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen                     "Corrupted index cache file %s: %s",
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen                     cache->filepath,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen                     t_strdup_vprintf(fmt, va));
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen    } T_END;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    va_end(va);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainenvoid mail_cache_file_close(struct mail_cache *cache)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (cache->mmap_base != NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        if (munmap(cache->mmap_base, cache->mmap_length) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            mail_cache_set_syscall_error(cache, "munmap()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    if (cache->file_cache != NULL)
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        file_cache_set_fd(cache->file_cache, -1);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache->mmap_base = NULL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache->hdr = NULL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache->mmap_length = 0;
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen    cache->last_field_header_offset = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    if (cache->file_lock != NULL)
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        file_lock_free(&cache->file_lock);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    cache->locked = FALSE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (cache->fd != -1) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        if (close(cache->fd) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            mail_cache_set_syscall_error(cache, "close()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        cache->fd = -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainenstatic void mail_cache_init_file_cache(struct mail_cache *cache)
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen{
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen    struct stat st;
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen    if (cache->file_cache == NULL)
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen        return;
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen    file_cache_set_fd(cache->file_cache, cache->fd);
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen    if (fstat(cache->fd, &st) == 0)
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen        file_cache_set_size(cache->file_cache, st.st_size);
85da8c055280cd45553b6b335e9fb226d6e2801eTimo Sirainen    else if (!ESTALE_FSTAT(errno))
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen        mail_cache_set_syscall_error(cache, "fstat()");
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen    cache->st_ino = st.st_ino;
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen    cache->st_dev = st.st_dev;
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen}
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainenstatic bool mail_cache_need_reopen(struct mail_cache *cache)
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen{
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen    struct stat st;
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen        if (cache->need_compress_file_seq != 0) {
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen            /* we're waiting for compression */
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen            return FALSE;
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen        }
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen        if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) {
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen            /* disabled */
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen            return FALSE;
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen        }
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen    }
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen    if (cache->fd == -1)
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen        return TRUE;
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen    /* see if the file has changed */
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen    if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen        i_assert(!cache->locked);
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen        nfs_flush_file_handle_cache(cache->filepath);
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen    }
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen    if (nfs_safe_stat(cache->filepath, &st) < 0) {
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen        mail_cache_set_syscall_error(cache, "stat()");
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen        return TRUE;
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen    }
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen    if (st.st_ino != cache->st_ino ||
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen        !CMP_DEV_T(st.st_dev, cache->st_dev)) {
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen        /* file changed */
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen        return TRUE;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen    }
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen    if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen        /* if the old file has been deleted, the new file may have
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen           the same inode as the old one. we'll catch this here by
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen           checking if fstat() fails with ESTALE */
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen        if (fstat(cache->fd, &st) < 0) {
85da8c055280cd45553b6b335e9fb226d6e2801eTimo Sirainen            if (ESTALE_FSTAT(errno))
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen                return TRUE;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen            mail_cache_set_syscall_error(cache, "fstat()");
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen            return FALSE;
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen        }
2131ef7a3390f15ea6a958256ea54908f1096350Timo Sirainen    }
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen    return FALSE;
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen}
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainenint mail_cache_reopen(struct mail_cache *cache)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    struct mail_index_view *view;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    const struct mail_index_ext *ext;
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    const void *data;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    i_assert(!cache->locked);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
69e03a846f6980144aa75bff0590c04852bffbbcTimo Sirainen    if (!mail_cache_need_reopen(cache)) {
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen        /* reopening does no good */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return 0;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    mail_cache_file_close(cache);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen    cache->fd = nfs_safe_open(cache->filepath,
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen                  cache->index->readonly ? O_RDONLY : O_RDWR);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    if (cache->fd == -1) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        if (errno == ENOENT)
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen            cache->need_compress_file_seq = 0;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        else
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            mail_cache_set_syscall_error(cache, "open()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen    mail_cache_init_file_cache(cache);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    if (mail_cache_map(cache, 0, 0, &data) < 0)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (mail_cache_header_fields_read(cache) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    view = mail_index_view_open(cache->index);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    ext = mail_index_view_get_ext(view, cache->ext_id);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    if (ext == NULL || cache->hdr->file_seq != ext->reset_id) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        /* still different - maybe a race condition or maybe the
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen           file_seq really is corrupted. either way, this shouldn't
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen           happen often so we'll just mark cache to be compressed
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen           later which fixes this. */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        cache->need_compress_file_seq = cache->hdr->file_seq;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen        mail_index_view_close(&view);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return 0;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    mail_index_view_close(&view);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainenstatic void mail_cache_update_need_compress(struct mail_cache *cache)
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen{
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen    const struct mail_cache_header *hdr = cache->hdr;
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen    unsigned int cont_percentage;
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen    uoff_t max_del_space;
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen        cont_percentage = hdr->continued_record_count * 100 /
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen        (cache->index->map->rec_map->records_count == 0 ? 1 :
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen         cache->index->map->rec_map->records_count);
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen    if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE &&
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen        hdr->used_file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) {
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen        /* too many continued rows, compress */
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen        cache->need_compress_file_seq = hdr->file_seq;
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen    }
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen    /* see if we've reached the max. deleted space in file */
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen    max_del_space = hdr->used_file_size / 100 *
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen        MAIL_CACHE_COMPRESS_PERCENTAGE;
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen    if (hdr->deleted_space >= max_del_space &&
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen        hdr->used_file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE)
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen        cache->need_compress_file_seq = hdr->file_seq;
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen}
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainenstatic bool mail_cache_verify_header(struct mail_cache *cache,
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen                     const struct mail_cache_header *hdr)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    /* check that the header is still ok */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (cache->mmap_length < sizeof(struct mail_cache_header)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_corrupted(cache, "File too small");
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f501ad38c51cf1d8f4f84313922c785e6ae6e81fTimo Sirainen    if (hdr->version != MAIL_CACHE_VERSION) {
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen        /* version changed - upgrade silently */
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen        mail_cache_unlink(cache);
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen        return FALSE;
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen    }
e192a3b1ca8ae857e7d87298ea507d32977ba570Timo Sirainen    if (hdr->compat_sizeof_uoff_t != sizeof(uoff_t)) {
e192a3b1ca8ae857e7d87298ea507d32977ba570Timo Sirainen        /* architecture change - handle silently(?) */
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen        mail_cache_unlink(cache);
e192a3b1ca8ae857e7d87298ea507d32977ba570Timo Sirainen        return FALSE;
811f2e26d9782d9cb99fdf82e18ffa0a77564fe2Timo Sirainen    }
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen
f501ad38c51cf1d8f4f84313922c785e6ae6e81fTimo Sirainen    if (hdr->indexid != cache->index->indexid) {
c4877db8b6559846f4b58be8e42422dc734c193fTimo Sirainen        /* index id changed - handle silently */
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen        mail_cache_unlink(cache);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
f501ad38c51cf1d8f4f84313922c785e6ae6e81fTimo Sirainen    if (hdr->file_seq == 0) {
8b9342aa96b2f297e23afb261f9f7dd859800952Timo Sirainen        mail_cache_set_corrupted(cache, "file_seq is 0");
8b9342aa96b2f297e23afb261f9f7dd859800952Timo Sirainen        return FALSE;
8b9342aa96b2f297e23afb261f9f7dd859800952Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    /* only check the header if we're locked */
37e6cf44d61a81c6839e3ab76234b54309d8d292Timo Sirainen    if (!cache->locked)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen    if (hdr->used_file_size < sizeof(struct mail_cache_header)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_corrupted(cache, "used_file_size too small");
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen    if ((hdr->used_file_size % sizeof(uint32_t)) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_corrupted(cache, "used_file_size not aligned");
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    if (cache->mmap_base != NULL &&
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        hdr->used_file_size > cache->mmap_length) {
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen        mail_cache_set_corrupted(cache, "used_file_size too large");
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    return TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainenstatic int
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainenmail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size,
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen              const void *hdr_data, bool copy_hdr)
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen{
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    const struct mail_cache_header *hdr = hdr_data;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    if (offset == 0) {
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen        /* verify the header validity only with offset=0. this way
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen           we won't waste time re-verifying it all the time */
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        if (!mail_cache_verify_header(cache, hdr)) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen            cache->need_compress_file_seq =
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen                !MAIL_CACHE_IS_UNUSABLE(cache) &&
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen                cache->hdr->file_seq != 0 ?
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen                cache->hdr->file_seq : 0;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen            return -1;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        }
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    }
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    if (hdr_data != NULL) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        if (!copy_hdr)
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen            cache->hdr = hdr;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        else {
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen            memcpy(&cache->hdr_ro_copy, hdr,
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen                   sizeof(cache->hdr_ro_copy));
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen            cache->hdr = &cache->hdr_ro_copy;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        mail_cache_update_need_compress(cache);
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    } else {
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen        i_assert(cache->hdr != NULL);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    if (offset + size > cache->mmap_length)
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        return 0;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    return 1;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen}
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainenstatic int
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainenmail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size,
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen             const void **data_r)
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen{
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    const void *hdr_data;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    void *data;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    ssize_t ret;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    if (cache->read_buf == NULL) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        cache->read_buf =
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen            buffer_create_dynamic(default_pool, size);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    } else if (cache->read_offset <= offset &&
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen           cache->read_offset + cache->read_buf->used >= offset+size) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        /* already mapped */
bb86f8f22f2561438ce710d2113f04a4d0082b50Timo Sirainen        *data_r = CONST_PTR_OFFSET(cache->read_buf->data,
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen                       offset - cache->read_offset);
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen        hdr_data = offset == 0 ? *data_r : NULL;
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen        return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    } else {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        buffer_set_used_size(cache->read_buf, 0);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    }
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen    if (offset == 0 && size < MAIL_CACHE_MIN_HEADER_READ_SIZE) {
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen        /* we can usually read the fields header after the cache
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen           header. we need them both, so try to read them all with one
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen           pread() call. */
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen        size = MAIL_CACHE_MIN_HEADER_READ_SIZE;
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen    }
ab1e5b156d1b5480d36ed6e8e06197339d803038Timo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    data = buffer_append_space_unsafe(cache->read_buf, size);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    ret = pread(cache->fd, data, size, offset);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    if (ret < 0) {
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        if (errno != ESTALE)
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen            mail_cache_set_syscall_error(cache, "read()");
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        buffer_set_used_size(cache->read_buf, 0);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        cache->hdr = NULL;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        cache->mmap_length = 0;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        return -1;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    buffer_set_used_size(cache->read_buf, ret);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    cache->read_offset = offset;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    cache->mmap_length = offset + size;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    *data_r = data;
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    hdr_data = offset == 0 ? *data_r : NULL;
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen}
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainenint mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen           const void **data_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    const void *data;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    ssize_t ret;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    if (size == 0)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        size = sizeof(struct mail_cache_header);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    cache->remap_counter++;
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    if (cache->map_with_read)
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        return mail_cache_map_with_read(cache, offset, size, data_r);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    if (cache->file_cache != NULL) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        cache->hdr = NULL;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        ret = file_cache_read(cache->file_cache, offset, size);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        if (ret < 0) {
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen                        /* In case of ESTALE we'll simply fail without error
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen                           messages. The caller will then just have to
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen                           fallback to generating the value itself.
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen                           We can't simply reopen the cache flie, because
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen                           using it requires also having updated file
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen                           offsets. */
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen                        if (errno != ESTALE)
1e47cfede3a0b62654105daab00e97b5d660bc6bTimo Sirainen                                mail_cache_set_syscall_error(cache, "read()");
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen            return -1;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen        data = file_cache_get_map(cache->file_cache,
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen                      &cache->mmap_length);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        *data_r = offset > cache->mmap_length ? NULL :
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen            CONST_PTR_OFFSET(data, offset);
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen        return mail_cache_map_finish(cache, offset, size, data, TRUE);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (offset < cache->mmap_length &&
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        size <= cache->mmap_length - offset) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        /* already mapped */
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen        *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset);
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen        return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (cache->mmap_base != NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        if (munmap(cache->mmap_base, cache->mmap_length) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            mail_cache_set_syscall_error(cache, "munmap()");
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    } else {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        if (cache->fd == -1) {
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen            /* unusable, waiting for compression or
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen               index is in memory */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            i_assert(cache->need_compress_file_seq != 0 ||
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen                 MAIL_INDEX_IS_IN_MEMORY(cache->index));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    /* map the whole file */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache->hdr = NULL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache->mmap_length = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen    cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (cache->mmap_base == MAP_FAILED) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        cache->mmap_base = NULL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_syscall_error(cache, "mmap()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    *data_r = offset > cache->mmap_length ? NULL :
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        CONST_PTR_OFFSET(cache->mmap_base, offset);
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen    return mail_cache_map_finish(cache, offset, size,
f81801789c71f64a2fc3c44d09f9864bbc68cd45Timo Sirainen                     cache->mmap_base, FALSE);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainenstatic int mail_cache_try_open(struct mail_cache *cache)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    const void *data;
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    cache->opened = TRUE;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen    if (MAIL_INDEX_IS_IN_MEMORY(cache->index))
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen        return 0;
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen    cache->fd = nfs_safe_open(cache->filepath,
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen                  cache->index->readonly ? O_RDONLY : O_RDWR);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (cache->fd == -1) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        if (errno == ENOENT) {
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen            cache->need_compress_file_seq = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            return 0;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_syscall_error(cache, "open()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
abe7afb8f1766fbcef1b9df513109e43d7d16e49Timo Sirainen    mail_cache_init_file_cache(cache);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    if (mail_cache_map(cache, 0, sizeof(struct mail_cache_header),
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen               &data) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        return -1;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    return 1;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen}
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainenint mail_cache_open_and_verify(struct mail_cache *cache)
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen{
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    int ret;
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    ret = mail_cache_try_open(cache);
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    if (ret > 0)
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen        ret = mail_cache_header_fields_read(cache);
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    if (ret < 0) {
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen        /* failed for some reason - doesn't really matter,
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen           it's disabled for now. */
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen        mail_cache_file_close(cache);
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    }
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainenstatic struct mail_cache *mail_cache_alloc(struct mail_index *index)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    struct mail_cache *cache;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache = i_new(struct mail_cache, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache->index = index;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache->fd = -1;
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen    cache->filepath =
a835194f9a9dae88528367a791cbc282589f6c01Timo Sirainen        i_strconcat(index->filepath, MAIL_CACHE_FILE_SUFFIX, NULL);
0b878c6a17c608fcd8b52a5762ed2c6a5cf4700aTimo Sirainen    cache->field_pool = pool_alloconly_create("Cache fields", 2048);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    cache->field_name_hash =
3ccab0bac68040f179a7de45c516cec258e28fdbTimo Sirainen        hash_table_create(default_pool, cache->field_pool, 0,
3ccab0bac68040f179a7de45c516cec258e28fdbTimo Sirainen                  strcase_hash, (hash_cmp_callback_t *)strcasecmp);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen    cache->dotlock_settings.use_excl_lock =
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen        (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen    cache->dotlock_settings.nfs_flush =
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen        (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen    cache->dotlock_settings.timeout =
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen        I_MIN(MAIL_CACHE_LOCK_TIMEOUT, index->max_lock_timeout_secs);
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen    cache->dotlock_settings.stale_timeout = MAIL_CACHE_LOCK_CHANGE_TIMEOUT;
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen    if (!MAIL_INDEX_IS_IN_MEMORY(index) &&
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen        (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0)
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen        cache->file_cache = file_cache_new(-1);
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen    cache->map_with_read =
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen        (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    cache->ext_id =
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        mail_index_ext_register(index, "cache", 0,
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen                    sizeof(uint32_t), sizeof(uint32_t));
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen    mail_index_register_expunge_handler(index, cache->ext_id, FALSE,
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen                        mail_cache_expunge_handler, cache);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    mail_index_register_sync_handler(index, cache->ext_id,
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen                     mail_cache_sync_handler,
9315dd69233d554452df0c12bc57002d2042a8f4Timo Sirainen                                         MAIL_INDEX_SYNC_HANDLER_FILE |
9315dd69233d554452df0c12bc57002d2042a8f4Timo Sirainen                                         MAIL_INDEX_SYNC_HANDLER_HEAD |
69bd816e46fdee6182d0cb2e4c6be32399a555c8Timo Sirainen                     (cache->file_cache == NULL ? 0 :
69bd816e46fdee6182d0cb2e4c6be32399a555c8Timo Sirainen                      MAIL_INDEX_SYNC_HANDLER_VIEW));
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen
f23298fea47eecbeded985ee2537a34c4c4ef56bTimo Sirainen    if (cache->file_cache != NULL) {
f23298fea47eecbeded985ee2537a34c4c4ef56bTimo Sirainen        mail_index_register_sync_lost_handler(index,
f23298fea47eecbeded985ee2537a34c4c4ef56bTimo Sirainen            mail_cache_sync_lost_handler);
f23298fea47eecbeded985ee2537a34c4c4ef56bTimo Sirainen    }
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen    return cache;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainenstruct mail_cache *mail_cache_open_or_create(struct mail_index *index)
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen{
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen    struct mail_cache *cache;
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen    cache = mail_cache_alloc(index);
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen    return cache;
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen}
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainenstruct mail_cache *mail_cache_create(struct mail_index *index)
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen{
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen    struct mail_cache *cache;
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen    cache = mail_cache_alloc(index);
5196f9ea42d02000f9c3d22f20aa816140af4422Timo Sirainen    if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
5196f9ea42d02000f9c3d22f20aa816140af4422Timo Sirainen        if (unlink(cache->filepath) < 0 && errno != ENOENT)
5196f9ea42d02000f9c3d22f20aa816140af4422Timo Sirainen            mail_cache_set_syscall_error(cache, "unlink()");
5196f9ea42d02000f9c3d22f20aa816140af4422Timo Sirainen    }
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen    return cache;
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen}
f239eb76f77afcbc0bfc97c9b52b4407d1bc3fe6Timo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainenvoid mail_cache_free(struct mail_cache **_cache)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    struct mail_cache *cache = *_cache;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    *_cache = NULL;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    if (cache->file_cache != NULL) {
519e0a461271843833a2b42626ad93f6e7ddc497Timo Sirainen        mail_index_unregister_sync_lost_handler(cache->index,
519e0a461271843833a2b42626ad93f6e7ddc497Timo Sirainen            mail_cache_sync_lost_handler);
519e0a461271843833a2b42626ad93f6e7ddc497Timo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen        file_cache_free(&cache->file_cache);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
519e0a461271843833a2b42626ad93f6e7ddc497Timo Sirainen    mail_index_unregister_expunge_handler(cache->index, cache->ext_id);
519e0a461271843833a2b42626ad93f6e7ddc497Timo Sirainen    mail_index_unregister_sync_handler(cache->index, cache->ext_id);
519e0a461271843833a2b42626ad93f6e7ddc497Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    mail_cache_file_close(cache);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
3ccab0bac68040f179a7de45c516cec258e28fdbTimo Sirainen    hash_table_destroy(&cache->field_name_hash);
648d24583c1574441c4fa0331a90bd4d6e7996c5Timo Sirainen    pool_unref(&cache->field_pool);
0cea9b1f4fa0495a48f5f097e40492517d67e1baTimo Sirainen    i_free(cache->field_file_map);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    i_free(cache->file_field_map);
0cea9b1f4fa0495a48f5f097e40492517d67e1baTimo Sirainen    i_free(cache->fields);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    i_free(cache->filepath);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    i_free(cache);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainenstatic int mail_cache_lock_file(struct mail_cache *cache, bool nonblock)
2af769daebd83719ac696a440e06f6020471cec0Timo Sirainen{
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen    unsigned int timeout_secs;
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen    int ret;
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen    if (cache->last_lock_failed) {
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen        /* previous locking failed. don't waste time waiting on it
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen           again, just try once to see if it's available now. */
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen        nonblock = TRUE;
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen    }
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        i_assert(cache->file_lock == NULL);
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen        timeout_secs = I_MIN(MAIL_CACHE_LOCK_TIMEOUT,
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen                     cache->index->max_lock_timeout_secs);
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen        ret = mail_index_lock_fd(cache->index, cache->filepath,
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen                     cache->fd, F_WRLCK,
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen                     nonblock ? 0 : timeout_secs,
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen                     &cache->file_lock);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    } else {
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen        enum dotlock_create_flags flags =
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen            nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0;
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        i_assert(cache->dotlock == NULL);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen        ret = file_dotlock_create(&cache->dotlock_settings,
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen                      cache->filepath, flags,
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen                      &cache->dotlock);
4aae8acbcfa9cac96b4af39bfabcbe569e804827Timo Sirainen        if (ret < 0) {
5bdad39213d28ab35e615a7f4ea1712ab25b6a80Timo Sirainen            mail_cache_set_syscall_error(cache,
5bdad39213d28ab35e615a7f4ea1712ab25b6a80Timo Sirainen                             "file_dotlock_create()");
5bdad39213d28ab35e615a7f4ea1712ab25b6a80Timo Sirainen        }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    }
763f83d3cc47bce05cbc396419c4db2b71dd8e68Timo Sirainen    cache->last_lock_failed = ret <= 0;
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
4aae8acbcfa9cac96b4af39bfabcbe569e804827Timo Sirainen    /* don't bother warning if locking failed due to a timeout. since cache
4aae8acbcfa9cac96b4af39bfabcbe569e804827Timo Sirainen       updating isn't all that important we're using a very short timeout
4aae8acbcfa9cac96b4af39bfabcbe569e804827Timo Sirainen       so it can be triggered sometimes on heavy load */
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen    if (ret <= 0)
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen        return ret;
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen    mail_index_flush_read_cache(cache->index, cache->filepath, cache->fd,
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen                    TRUE);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen    return 1;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen}
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainenstatic void mail_cache_unlock_file(struct mail_cache *cache)
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen{
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    if (cache->index->lock_method != FILE_LOCK_METHOD_DOTLOCK)
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        file_unlock(&cache->file_lock);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    else
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        (void)file_dotlock_delete(&cache->dotlock);
2af769daebd83719ac696a440e06f6020471cec0Timo Sirainen}
2af769daebd83719ac696a440e06f6020471cec0Timo Sirainen
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainenstatic int
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainenmail_cache_lock_full(struct mail_cache *cache, bool require_same_reset_id,
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen             bool nonblock)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    const struct mail_index_ext *ext;
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    const void *data;
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen    struct mail_index_view *iview;
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen    uint32_t reset_id;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    int i, ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen    i_assert(!cache->locked);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    if (!cache->opened)
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen        (void)mail_cache_open_and_verify(cache);
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
0d0451206a91e9f96e522075dce28a89adc2325dTimo Sirainen    if (MAIL_CACHE_IS_UNUSABLE(cache) ||
1c0590b2729567ad60dafde4d2c5f19635755a3dTimo Sirainen        MAIL_INDEX_IS_IN_MEMORY(cache->index) ||
1c0590b2729567ad60dafde4d2c5f19635755a3dTimo Sirainen        cache->index->readonly)
ba482d3624ca4f1b3d638e6e8470ba5134f21493Timo Sirainen        return 0;
ba482d3624ca4f1b3d638e6e8470ba5134f21493Timo Sirainen
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen    iview = mail_index_view_open(cache->index);
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen    ext = mail_index_view_get_ext(iview, cache->ext_id);
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen    reset_id = ext == NULL ? 0 : ext->reset_id;
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen    mail_index_view_close(&iview);
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen    if (ext == NULL && require_same_reset_id) {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        /* cache not used */
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        return 0;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    }
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    for (i = 0; i < 3; i++) {
cf3bea6d9b57f8608bec22d98ad547a507b05f66Timo Sirainen        if (cache->hdr->file_seq != reset_id &&
cf3bea6d9b57f8608bec22d98ad547a507b05f66Timo Sirainen            (require_same_reset_id || i == 0)) {
cf3bea6d9b57f8608bec22d98ad547a507b05f66Timo Sirainen            /* we want the latest cache file */
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen            if (reset_id < cache->hdr->file_seq) {
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen                /* either we're still waiting for index to
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen                   catch up with a cache compression, or
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen                   that catching up is never going to happen */
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen                ret = 0;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen                break;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen            }
cf3bea6d9b57f8608bec22d98ad547a507b05f66Timo Sirainen            ret = mail_cache_reopen(cache);
e6e43b396799aa5704c679a3017d6c7195f9347dTimo Sirainen            if (ret < 0 || (ret == 0 && require_same_reset_id))
cf3bea6d9b57f8608bec22d98ad547a507b05f66Timo Sirainen                break;
cf3bea6d9b57f8608bec22d98ad547a507b05f66Timo Sirainen        }
cf3bea6d9b57f8608bec22d98ad547a507b05f66Timo Sirainen
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen        if ((ret = mail_cache_lock_file(cache, nonblock)) <= 0) {
84da9c6d6e162b064608cbfa9a47e0d60553c593Timo Sirainen            ret = -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            break;
84da9c6d6e162b064608cbfa9a47e0d60553c593Timo Sirainen        }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen        cache->locked = TRUE;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen        if (cache->hdr->file_seq == reset_id ||
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen            !require_same_reset_id) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            /* got it */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        /* okay, so it was just compressed. try again. */
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen        (void)mail_cache_unlock(cache);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        ret = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    if (ret > 0) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        /* make sure our header is up to date */
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen        if (cache->file_cache != NULL) {
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen            file_cache_invalidate(cache->file_cache, 0,
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen                          sizeof(struct mail_cache_header));
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen        }
600848916ea123b89235bb51d660f662f2670538Timo Sirainen        if (mail_cache_map(cache, 0, 0, &data) > 0)
7981779f9aebd25728d3c26555d598ff842cf2e2Timo Sirainen            cache->hdr_copy = *cache->hdr;
7981779f9aebd25728d3c26555d598ff842cf2e2Timo Sirainen        else {
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen            (void)mail_cache_unlock(cache);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen            ret = -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    i_assert((ret <= 0 && !cache->locked) || (ret > 0 && cache->locked));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainenint mail_cache_lock(struct mail_cache *cache, bool require_same_reset_id)
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen{
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen    return mail_cache_lock_full(cache, require_same_reset_id, FALSE);
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen}
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainenint mail_cache_try_lock(struct mail_cache *cache)
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen{
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen    return mail_cache_lock_full(cache, FALSE, TRUE);
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen}
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainenint mail_cache_unlock(struct mail_cache *cache)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen    int ret = 0;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen    i_assert(cache->locked);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen    if (cache->field_header_write_pending)
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen                ret = mail_cache_header_fields_update(cache);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen    cache->locked = FALSE;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen        /* we found it to be broken during the lock. just clean up. */
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen        cache->hdr_modified = FALSE;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen        return -1;
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen    }
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen    if (cache->hdr_modified) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen        cache->hdr_modified = FALSE;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen        if (mail_cache_write(cache, &cache->hdr_copy,
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen                     sizeof(cache->hdr_copy), 0) < 0)
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen            ret = -1;
f501ad38c51cf1d8f4f84313922c785e6ae6e81fTimo Sirainen        cache->hdr_ro_copy = cache->hdr_copy;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        mail_cache_update_need_compress(cache);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen    }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen    if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen        if (fdatasync(cache->fd) < 0)
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen            mail_cache_set_syscall_error(cache, "fdatasync()");
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen    }
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    mail_cache_unlock_file(cache);
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen    return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainenint mail_cache_write(struct mail_cache *cache, const void *data, size_t size,
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen             uoff_t offset)
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen{
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen    if (pwrite_full(cache->fd, data, size, offset) < 0) {
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen        mail_cache_set_syscall_error(cache, "pwrite_full()");
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen        return -1;
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen    }
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen    if (cache->file_cache != NULL)
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen        file_cache_write(cache->file_cache, data, size, offset);
bba52ecbb0cfb6585f1a4ff29695dd2d27af98d2Timo Sirainen    if (cache->read_buf != NULL)
bba52ecbb0cfb6585f1a4ff29695dd2d27af98d2Timo Sirainen        buffer_set_used_size(cache->read_buf, 0);
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen    return 0;
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen}
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen
f3976df875193529127d584cb713983e8160bdcfTimo Sirainenbool mail_cache_exists(struct mail_cache *cache)
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen{
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen    return !MAIL_CACHE_IS_UNUSABLE(cache);
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen}
f3976df875193529127d584cb713983e8160bdcfTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstruct mail_cache_view *
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    struct mail_cache_view *view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    view = i_new(struct mail_cache_view, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    view->cache = cache;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    view->view = iview;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    view->cached_exists_buf =
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        buffer_create_dynamic(default_pool,
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen                      cache->file_fields_count + 10);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    return view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainenvoid mail_cache_view_close(struct mail_cache_view **_view)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen    struct mail_cache_view *view = *_view;
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen    i_assert(view->trans_view == NULL);
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen    *_view = NULL;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen    if (view->cache->field_header_write_pending &&
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen        !view->cache->compressing)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen                (void)mail_cache_header_fields_update(view->cache);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen    buffer_free(&view->cached_exists_buf);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    i_free(view);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainenvoid mail_cache_view_update_cache_decisions(struct mail_cache_view *view,
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen                        bool update)
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen{
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen    view->no_decision_updates = !update;
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen}
8fa86f7ef06aa6cf0239c7ca2eb98889691d40d4Timo Sirainen
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainenuint32_t mail_cache_get_first_new_seq(struct mail_index_view *view)
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen{
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen    const struct mail_index_header *idx_hdr;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen    uint32_t first_new_seq, message_count;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen    idx_hdr = mail_index_get_header(view);
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen    if (idx_hdr->day_first_uid[7] == 0)
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen        return 1;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen    if (!mail_index_lookup_seq_range(view, idx_hdr->day_first_uid[7],
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen                     (uint32_t)-1, &first_new_seq,
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen                     &message_count)) {
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen        /* all messages are too old */
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen        return message_count+1;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen    }
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen    return first_new_seq;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen}