mail-cache.c revision 5024402a628e9a6fc21eb8cdf2778486c6f2f92c
e59faf65ce864fe95dc00f5d52b8323cdbd0608aTimo Sirainen/* Copyright (C) 2003 Timo Sirainen */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#define _XOPEN_SOURCE 500 /* for pwrite() / Linux */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen#include "lib.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "buffer.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "byteorder.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "file-lock.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "file-set-size.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "ioloop.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mmap-util.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "write-full.h"
2cd2518bab14292a67cf8a490b58ab9ef89879daTimo Sirainen#include "mail-index.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-index-util.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-cache.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include <stddef.h>
6ec925f52d04ec8700e47cb005bd7ddc65ac5614Timo Sirainen#include <unistd.h>
c60178b2610a9b193ff72aa18448398ef72529a1Timo Sirainen#include <sys/stat.h>
4d4d6d4745682790c20d759ba93dbea46b812c5dTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen/* Never compress the file if it's smaller than this */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#define COMPRESS_MIN_SIZE (1024*50)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
cf2e6953d03a1c22f272ec19432fc03c136ac1bbTimo Sirainen/* Compress the file when deleted space reaches n% of total size */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#define COMPRESS_PERCENTAGE 20
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen/* Compress the file when n% of rows contain continued rows.
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen 200% means that there's 2 continued rows per record. */
5323bec1c24184863a13bc14a2dc9487093eea3dTimo Sirainen#define COMPRESS_CONTINUED_PERCENTAGE 200
5323bec1c24184863a13bc14a2dc9487093eea3dTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen/* Initial size for the file */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#define MAIL_CACHE_INITIAL_SIZE (sizeof(struct mail_cache_header) + 10240)
617e13833c798435e2be425b99c27ecaad1b8393Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen/* When more space is needed, grow the file n% larger than the previous size */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#define MAIL_CACHE_GROW_PERCENTAGE 10
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#define MAIL_CACHE_LOCK_TIMEOUT 120
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen#define MAIL_CACHE_LOCK_STALE_TIMEOUT 60
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#define CACHE_RECORD(cache, data_pos) \
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ((struct mail_cache_record *) \
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ((char *) (cache)->mmap_base + ((data_pos) & ~3)))
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstruct mail_cache_header {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t indexid;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t continued_record_count;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t used_file_size;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t deleted_space;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t used_fields; /* enum mail_cache_field */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t field_usage_start; /* time_t */
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen uint32_t field_usage_counts[32];
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT];
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen};
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainenstruct mail_cache_record {
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen uint32_t fields; /* enum mail_cache_field */
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen uint32_t next_offset;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen uint32_t size; /* full record size, including this header */
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen};
230ef558135f16a66b86cbe3762524eaa9ae9d81Timo Sirainen
230ef558135f16a66b86cbe3762524eaa9ae9d81Timo Sirainenstruct mail_cache {
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen struct mail_index *index;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen char *filepath;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen int fd;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen void *mmap_base;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen size_t mmap_length;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen uint32_t used_file_size;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen struct mail_cache_header *header;
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen pool_t split_header_pool;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen uint32_t split_offsets[MAIL_CACHE_HEADERS_COUNT];
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen const char *const *split_headers[MAIL_CACHE_HEADERS_COUNT];
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen
617e13833c798435e2be425b99c27ecaad1b8393Timo Sirainen enum mail_cache_field default_cache_fields;
617e13833c798435e2be425b99c27ecaad1b8393Timo Sirainen enum mail_cache_field never_cache_fields;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen struct mail_cache_transaction_ctx *trans_ctx;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen unsigned int locks;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen unsigned int anon_mmap:1;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen unsigned int mmap_refresh:1;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen unsigned int silent:1;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen};
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen
617e13833c798435e2be425b99c27ecaad1b8393Timo Sirainenstruct mail_cache_transaction_ctx {
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen struct mail_cache *cache;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen unsigned int next_unused_header_lowwater;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen unsigned int last_idx;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen struct mail_cache_record cache_rec;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen buffer_t *cache_data;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen unsigned int first_uid, last_uid;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen buffer_t *index_marks, *cache_marks;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen};
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainenunsigned int mail_cache_field_sizes[32] = {
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen sizeof(enum mail_index_record_flag),
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen sizeof(uoff_t),
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen 16,
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen sizeof(struct mail_sent_date),
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen sizeof(time_t),
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen sizeof(uoff_t),
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen sizeof(uoff_t),
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen sizeof(uoff_t),
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen sizeof(uoff_t),
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen 0, 0, 0,
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* variable sized */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen};
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenenum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT] = {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_CACHE_HEADERS1,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_CACHE_HEADERS2,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_CACHE_HEADERS3,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_CACHE_HEADERS4
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen};
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic const unsigned char *null4[] = { 0, 0, 0, 0 };
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic const char *
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenmail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int mail_cache_write(struct mail_cache_transaction_ctx *ctx);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic struct mail_cache_record *
601b455f4d5e780044b9e4fac5f687c1b07ae145Timo Sirainenmail_cache_lookup(struct mail_cache *cache,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_index_record *rec);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainenstatic int mail_cache_set_syscall_error(struct mail_cache *cache,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const char *function)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
fb7ac3e31c92627efe076318319976ac1c27ae2aTimo Sirainen i_assert(function != NULL);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (ENOSPACE(errno)) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->index->nodiskspace = TRUE;
72cc352b25ad401b923436c6ed0f1f3adaffa737Timo Sirainen return FALSE;
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
614529ee060755c0b282102b70daf56bcd64222dTimo Sirainen index_set_error(cache->index, "%s failed with index cache file %s: %m",
61618d4c58080570f689614fec204ae14e90cef2Timo Sirainen function, cache->filepath);
6ec925f52d04ec8700e47cb005bd7ddc65ac5614Timo Sirainen return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int mail_cache_create_memory(struct mail_cache *cache,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_cache_header *hdr)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->mmap_length = MAIL_CACHE_INITIAL_SIZE;
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen cache->mmap_base = mmap_anon(cache->mmap_length);
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen if (cache->mmap_base == MAP_FAILED) {
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen index_set_error(cache->index, "mmap_anon(%"PRIuSIZE_T")",
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen cache->mmap_length);
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen return FALSE;
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->header = cache->mmap_base;
4b2a1030f9b51e90f6ff4cdbc115871a398c1e0fTimo Sirainen *cache->header = *hdr;
72cc352b25ad401b923436c6ed0f1f3adaffa737Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen cache->anon_mmap = TRUE;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen cache->filepath = i_strdup_printf("(in-memory index cache for %s)",
72cc352b25ad401b923436c6ed0f1f3adaffa737Timo Sirainen cache->index->mailbox_path);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic void mail_cache_file_close(struct mail_cache *cache)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen if (cache->anon_mmap) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (munmap_anon(cache->mmap_base, cache->mmap_length) < 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_set_syscall_error(cache, "munmap_anon()");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen } else if (cache->mmap_base != NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (munmap(cache->mmap_base, cache->mmap_length) < 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_set_syscall_error(cache, "munmap()");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->mmap_base = NULL;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->header = NULL;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->mmap_length = 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->fd != -1) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (close(cache->fd) < 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_set_syscall_error(cache, "close()");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->fd = -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
f5e1d3d6b34ec152aa1ff15c7bd3d3552e9227eaTimo Sirainenstatic int mail_cache_file_reopen(struct mail_cache *cache)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen int fd;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->anon_mmap) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* cache was set corrupted, we'll have to quit */
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen fd = open(cache->filepath, O_RDWR);
3a0f9aa9504497e4e47f32df54fbf47fdc7423b6Timo Sirainen if (fd == -1)
3a0f9aa9504497e4e47f32df54fbf47fdc7423b6Timo Sirainen return mail_cache_set_syscall_error(cache, "open()");
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen mail_cache_file_close(cache);
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen cache->fd = fd;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int mmap_verify_header(struct mail_cache *cache)
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen{
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen struct mail_cache_header *hdr;
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen /* check that the header is still ok */
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen if (cache->mmap_length < sizeof(struct mail_cache_header))
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen return mail_cache_set_corrupted(cache, "File too small");
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen cache->header = hdr = cache->mmap_base;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->header->indexid != cache->index->indexid) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* index id changed */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->header->indexid != 0)
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen mail_cache_set_corrupted(cache, "indexid changed");
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen cache->index->inconsistent = TRUE; /* easiest way to rebuild */
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen return FALSE;
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen if (cache->trans_ctx != NULL) {
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen /* we've updated used_file_size, do nothing */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->used_file_size = nbo_to_uint32(hdr->used_file_size);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* only check the header if we're locked */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->locks == 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->used_file_size < sizeof(struct mail_cache_header)) {
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen mail_cache_set_corrupted(cache, "used_file_size too small");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
601b455f4d5e780044b9e4fac5f687c1b07ae145Timo Sirainen }
601b455f4d5e780044b9e4fac5f687c1b07ae145Timo Sirainen if ((cache->used_file_size % sizeof(uint32_t)) != 0) {
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen mail_cache_set_corrupted(cache, "used_file_size not aligned");
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen return FALSE;
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen }
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->used_file_size > cache->mmap_length) {
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen /* maybe a crash truncated the file - just fix it */
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen hdr->used_file_size = uint32_to_nbo(cache->mmap_length & ~3);
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (msync(cache->mmap_base, sizeof(*hdr), MS_SYNC) < 0)
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen return mail_cache_set_syscall_error(cache, "msync()");
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen }
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen return TRUE;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen}
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainenstatic int mmap_update_nocheck(struct mail_cache *cache,
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen size_t offset, size_t size)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen if (cache->header != NULL &&
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->header->indexid != cache->index->indexid) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* indexid changed, most likely it was rebuilt.
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen try reopening. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!mail_cache_file_reopen(cache))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen /* force mmap refresh */
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen size = 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (size != 0 && offset < cache->mmap_length &&
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen size <= cache->mmap_length - offset) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* already mapped */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!cache->mmap_refresh)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen cache->mmap_refresh = FALSE;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen }
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (cache->anon_mmap)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
3a0f9aa9504497e4e47f32df54fbf47fdc7423b6Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->mmap_base != NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->locks != 0) {
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen /* in the middle of transaction - write the changes */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (msync(cache->mmap_base, cache->mmap_length,
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen MS_SYNC) < 0) {
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen mail_cache_set_syscall_error(cache, "msync()");
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen return FALSE;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen }
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (munmap(cache->mmap_base, cache->mmap_length) < 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_set_syscall_error(cache, "munmap()");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(cache->fd != -1);
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* map the whole file */
2cd2518bab14292a67cf8a490b58ab9ef89879daTimo Sirainen cache->header = NULL;
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen cache->mmap_length = 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->mmap_base = mmap_rw_file(cache->fd, &cache->mmap_length);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->mmap_base == MAP_FAILED) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->mmap_base = NULL;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return mail_cache_set_syscall_error(cache, "mmap()");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen}
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainenstatic int mmap_update(struct mail_cache *cache, size_t offset, size_t size)
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return mmap_update_nocheck(cache, offset, size) &&
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mmap_verify_header(cache);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainenstatic int mail_cache_open_and_verify(struct mail_cache *cache, int silent)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen{
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen struct stat st;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen mail_cache_file_close(cache);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->fd = open(cache->filepath, O_RDWR);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (cache->fd == -1) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (errno == ENOENT)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_set_syscall_error(cache, "open()");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (fstat(cache->fd, &st) < 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_set_syscall_error(cache, "fstat()");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (st.st_size < sizeof(struct mail_cache_header))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->mmap_refresh = TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!mmap_update_nocheck(cache, 0, sizeof(struct mail_cache_header)))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* verify that this really is the cache for wanted index */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->silent = silent;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!mmap_verify_header(cache)) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->silent = FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->silent = FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
614529ee060755c0b282102b70daf56bcd64222dTimo Sirainenstatic void mail_index_clear_cache_offsets(struct mail_index *index)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
a3dcda4b01461c7690c655a013ec12851cdb78d4Timo Sirainen struct mail_index_record *rec;
61618d4c58080570f689614fec204ae14e90cef2Timo Sirainen
61618d4c58080570f689614fec204ae14e90cef2Timo Sirainen index->sync_stamp = 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rec = index->lookup(index, 1);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen while (rec != NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rec->cache_offset = 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rec = index->next(index, rec);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int mail_cache_open_or_create_file(struct mail_cache *cache,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_cache_header *hdr)
129db31de1780783a175633eba5811e44c361a81Timo Sirainen{
129db31de1780783a175633eba5811e44c361a81Timo Sirainen int ret, fd;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->filepath = i_strconcat(cache->index->filepath,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_CACHE_FILE_PREFIX, NULL);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ret = mail_cache_open_and_verify(cache, FALSE);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (ret != 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return ret > 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* we'll have to clear cache_offsets which requires exclusive lock */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->index->inconsistent = FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!mail_index_set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* maybe a rebuild.. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_CACHE_LOCK_STALE_TIMEOUT, NULL, NULL);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (fd == -1) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_set_syscall_error(cache, "file_dotlock_open()");
63849db64682675a2fd3e1aea05c10ecbc6d473aTimo Sirainen return FALSE;
114162093a3eb36c23a4ce4d4f2a43541dc18cc2Timo Sirainen }
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen /* see if someone else just created the cache file */
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen ret = mail_cache_open_and_verify(cache, TRUE);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (ret != 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (void)file_dotlock_delete(cache->filepath, fd);
114162093a3eb36c23a4ce4d4f2a43541dc18cc2Timo Sirainen return ret > 0;
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen }
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen /* rebuild then */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (write_full(fd, hdr, sizeof(*hdr)) < 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_set_syscall_error(cache, "write_full()");
63849db64682675a2fd3e1aea05c10ecbc6d473aTimo Sirainen (void)file_dotlock_delete(cache->filepath, fd);
129db31de1780783a175633eba5811e44c361a81Timo Sirainen return FALSE;
129db31de1780783a175633eba5811e44c361a81Timo Sirainen }
dfcc56dc6c485fe45a3fa18325f047bfaae65019Timo Sirainen if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) {
dfcc56dc6c485fe45a3fa18325f047bfaae65019Timo Sirainen mail_cache_set_syscall_error(cache, "file_set_size()");
63849db64682675a2fd3e1aea05c10ecbc6d473aTimo Sirainen (void)file_dotlock_delete(cache->filepath, fd);
63849db64682675a2fd3e1aea05c10ecbc6d473aTimo Sirainen return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_clear_cache_offsets(cache->index);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_file_close(cache);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->fd = dup(fd);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen mail_cache_set_syscall_error(cache, "file_dotlock_replace()");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->mmap_refresh = TRUE;
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen if (!mmap_update(cache, 0, sizeof(struct mail_cache_header)))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenint mail_cache_open_or_create(struct mail_index *index)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_cache_header hdr;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_cache *cache;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen memset(&hdr, 0, sizeof(hdr));
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen hdr.indexid = index->indexid;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache = i_new(struct mail_cache, 1);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->index = index;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->fd = -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->split_header_pool = pool_alloconly_create("Headers", 512);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen index->cache = cache;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* we'll do anon-mmaping only if initially requested. if we fail
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen because of out of disk space, we'll just let the main index code
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen know it and fail. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (INDEX_IS_IN_MEMORY(index)) {
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen if (!mail_cache_create_memory(cache, &hdr)) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_free(cache);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen } else {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!mail_cache_open_or_create_file(cache, &hdr)) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_free(cache);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen }
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* unset inconsistency - we already rebuilt the cache file */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen index->inconsistent = FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen}
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainenvoid mail_cache_free(struct mail_cache *cache)
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen{
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen i_assert(cache->trans_ctx == NULL);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen cache->index->cache = NULL;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_cache_file_close(cache);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen pool_unref(cache->split_header_pool);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_free(cache->filepath);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_free(cache);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenvoid mail_cache_set_defaults(struct mail_cache *cache,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen enum mail_cache_field default_cache_fields,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen enum mail_cache_field never_cache_fields)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen cache->default_cache_fields = default_cache_fields;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen cache->never_cache_fields = never_cache_fields;
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen}
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic const struct mail_cache_record *
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainenmail_cache_compress_record(struct mail_cache *cache,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_index_record *rec,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen enum mail_cache_field remove_fields,
uint32_t *size_r)
{
enum mail_cache_field cached_fields, field;
struct mail_cache_record cache_rec;
buffer_t *buffer;
const void *data;
size_t size;
uint32_t nb_size;
int i;
memset(&cache_rec, 0, sizeof(cache_rec));
buffer = buffer_create_dynamic(data_stack_pool, 4096, (size_t)-1);
cached_fields = mail_cache_get_fields(cache, rec) & ~remove_fields;
buffer_append(buffer, &cache_rec, sizeof(cache_rec));
for (i = 0; i < 32; i++) {
field = nbo32_bitmasks[i];
if ((cached_fields & field) == 0)
continue;
if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) {
cached_fields &= ~field;
continue;
}
nb_size = uint32_to_nbo((uint32_t)size);
if ((field & MAIL_CACHE_FIXED_MASK) == 0)
buffer_append(buffer, &nb_size, sizeof(nb_size));
buffer_append(buffer, data, size);
if ((size & 3) != 0)
buffer_append(buffer, null4, 4 - (size & 3));
}
cache_rec.fields = cached_fields;
cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer));
buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec));
data = buffer_get_data(buffer, &size);
*size_r = size;
return data;
}
static int mail_cache_copy(struct mail_cache *cache, int fd)
{
struct mail_cache_header *hdr;
const struct mail_cache_record *cache_rec;
struct mail_index_record *rec;
enum mail_cache_field used_fields, remove_fields;
unsigned char *mmap_base;
const char *str;
uint32_t new_file_size, offset, size, nb_size;
int i;
/* pick some reasonably good file size */
new_file_size = cache->used_file_size -
nbo_to_uint32(cache->header->deleted_space);
new_file_size = (new_file_size + 1023) & ~1023;
if (new_file_size < MAIL_CACHE_INITIAL_SIZE)
new_file_size = MAIL_CACHE_INITIAL_SIZE;
if (file_set_size(fd, new_file_size) < 0)
return mail_cache_set_syscall_error(cache, "file_set_size()");
mmap_base = mmap(NULL, new_file_size, PROT_READ | PROT_WRITE,
MAP_SHARED, fd, 0);
if (mmap_base == MAP_FAILED)
return mail_cache_set_syscall_error(cache, "mmap()");
/* skip file's header */
hdr = (struct mail_cache_header *) mmap_base;
offset = sizeof(*hdr);
/* get the newest message header list */
remove_fields = 0;
for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) {
str = mail_cache_get_header_fields_str(cache, i);
if (str != NULL) {
hdr->header_offsets[0] = uint32_to_nbo(offset | 2);
size = strlen(str) + 1;
nb_size = uint32_to_nbo(size);
memcpy(mmap_base + offset, &nb_size, sizeof(nb_size));
offset += sizeof(nb_size);
memcpy(mmap_base + offset, str, size);
offset += (size + 3) & ~3;
break;
}
}
/* remove other headers */
for (i--; i >= 0; i--)
remove_fields |= mail_cache_header_fields[i];
used_fields = 0;
rec = cache->index->lookup(cache->index, 1);
while (rec != NULL) {
cache_rec = mail_cache_lookup(cache, rec);
if (cache_rec == NULL)
rec->cache_offset = 0;
else if ((nbo_to_uint32(cache_rec->next_offset) & 2) == 0 &&
remove_fields == 0) {
/* just one unmodified block, copy it */
size = nbo_to_uint32(cache_rec->size);
i_assert(offset + size <= new_file_size);
memcpy(mmap_base + offset, cache_rec, size);
rec->cache_offset = uint32_to_nbo(offset | 2);
size = (size + 3) & ~3;
offset += size;
} else {
/* multiple blocks, sort them into buffer */
t_push();
cache_rec = mail_cache_compress_record(cache, rec,
remove_fields,
&size);
i_assert(offset + size <= new_file_size);
memcpy(mmap_base + offset, cache_rec, size);
used_fields |= cache_rec->fields;
t_pop();
rec->cache_offset = uint32_to_nbo(offset | 2);
offset += size;
}
rec = cache->index->next(cache->index, rec);
}
/* update header */
hdr->indexid = cache->index->indexid;
hdr->used_file_size = uint32_to_nbo(offset);
hdr->used_fields = used_fields;
hdr->field_usage_start = uint32_to_nbo(ioloop_time);
/* write everything to disk */
if (msync(mmap_base, offset, MS_SYNC) < 0)
return mail_cache_set_syscall_error(cache, "msync()");
if (fdatasync(fd) < 0)
return mail_cache_set_syscall_error(cache, "fdatasync()");
return TRUE;
}
int mail_cache_compress(struct mail_cache *cache)
{
int fd, ret = TRUE;
i_assert(cache->trans_ctx == NULL);
if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
return FALSE;
if (mail_cache_lock(cache, TRUE) <= 0)
return FALSE;
fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
MAIL_CACHE_LOCK_STALE_TIMEOUT, NULL, NULL);
if (fd == -1) {
mail_cache_set_syscall_error(cache, "file_dotlock_open()");
return FALSE;
}
/* now we'll begin the actual moving. keep rebuild-flag on
while doing it. */
cache->index->header->flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
if (!mail_index_fmdatasync(cache->index, cache->index->header_size))
return FALSE;
if (!mail_cache_copy(cache, fd)) {
(void)file_dotlock_delete(cache->filepath, fd);
ret = FALSE;
} else {
mail_cache_file_close(cache);
cache->fd = dup(fd);
if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
mail_cache_set_syscall_error(cache,
"file_dotlock_replace()");
ret = FALSE;
}
if (!mmap_update(cache, 0, 0))
ret = FALSE;
}
cache->index->header->flags &= ~(MAIL_INDEX_HDR_FLAG_REBUILD |
MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE);
if (!mail_cache_unlock(cache))
ret = FALSE;
return ret;
}
int mail_cache_truncate(struct mail_cache *cache)
{
struct mail_cache_header hdr;
int ret, fd;
memset(&hdr, 0, sizeof(hdr));
hdr.indexid = cache->index->indexid;
hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
cache->used_file_size = sizeof(hdr);
if (cache->anon_mmap) {
*cache->header = hdr;
return TRUE;
}
ret = mail_cache_open_and_verify(cache, FALSE);
if (ret != 0)
return ret > 0;
fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
MAIL_CACHE_LOCK_STALE_TIMEOUT, NULL, NULL);
if (fd == -1) {
mail_cache_set_syscall_error(cache, "file_dotlock_open()");
return FALSE;
}
if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
mail_cache_set_syscall_error(cache, "write_full()");
(void)file_dotlock_delete(cache->filepath, fd);
return FALSE;
}
if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) {
mail_cache_set_syscall_error(cache, "file_set_size()");
(void)file_dotlock_delete(cache->filepath, fd);
return FALSE;
}
mail_cache_file_close(cache);
cache->fd = dup(fd);
if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
mail_cache_set_syscall_error(cache, "file_dotlock_replace()");
return FALSE;
}
cache->mmap_refresh = TRUE;
if (!mmap_update(cache, 0, sizeof(struct mail_cache_header)))
return FALSE;
return TRUE;
}
int mail_cache_mark_file_deleted(struct mail_cache *cache)
{
uint32_t indexid = 0;
if (cache->anon_mmap)
cache->header->indexid = 0;
else {
if (pwrite(cache->fd, &indexid, sizeof(indexid), 0) < 0)
return mail_cache_set_syscall_error(cache, "pwrite()");
}
return TRUE;
}
int mail_cache_lock(struct mail_cache *cache, int nonblock)
{
int ret;
if (cache->locks++ != 0)
return TRUE;
if (cache->anon_mmap)
return TRUE;
if (nonblock) {
ret = file_try_lock(cache->fd, F_WRLCK);
if (ret < 0)
mail_cache_set_syscall_error(cache, "file_try_lock()");
} else {
ret = file_wait_lock(cache->fd, F_WRLCK);
if (ret <= 0)
mail_cache_set_syscall_error(cache, "file_wait_lock()");
}
if (ret > 0) {
if (!mmap_verify_header(cache)) {
(void)mail_cache_unlock(cache);
ret = -1;
}
}
return ret;
}
int mail_cache_unlock(struct mail_cache *cache)
{
if (--cache->locks > 0)
return TRUE;
if (cache->anon_mmap)
return TRUE;
if (file_wait_lock(cache->fd, F_UNLCK) <= 0) {
mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)");
return FALSE;
}
return TRUE;
}
void mail_cache_unlock_later(struct mail_cache *cache)
{
cache->index->cache_later_locks++;
}
int mail_cache_is_locked(struct mail_cache *cache)
{
return cache->locks > 0;
}
int mail_cache_transaction_begin(struct mail_cache *cache, int nonblock,
struct mail_cache_transaction_ctx **ctx_r)
{
int ret;
i_assert(cache->trans_ctx == NULL);
ret = mail_cache_lock(cache, nonblock);
if (ret <= 0)
return ret;
*ctx_r = i_new(struct mail_cache_transaction_ctx, 1);
(*ctx_r)->cache = cache;
(*ctx_r)->cache_data =
buffer_create_dynamic(system_pool, 8192, (size_t)-1);
(*ctx_r)->last_idx = (unsigned int)-1;
cache->trans_ctx = *ctx_r;
return 1;
}
int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
{
int ret = TRUE;
i_assert(ctx->cache->trans_ctx != NULL);
(void)mail_cache_transaction_rollback(ctx);
if (!mail_cache_unlock(ctx->cache))
ret = FALSE;
ctx->cache->trans_ctx = NULL;
if (ctx->cache_marks != NULL)
buffer_free(ctx->cache_marks);
if (ctx->index_marks != NULL)
buffer_free(ctx->index_marks);
buffer_free(ctx->cache_data);
i_free(ctx);
return ret;
}
static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
{
memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
ctx->last_idx = (unsigned int)-1;
ctx->next_unused_header_lowwater = 0;
ctx->first_uid = ctx->last_uid = 0;
if (ctx->cache_marks != NULL)
buffer_set_used_size(ctx->cache_marks, 0);
if (ctx->index_marks != NULL)
buffer_set_used_size(ctx->index_marks, 0);
buffer_set_used_size(ctx->cache_data, 0);
}
static void mark_update(buffer_t **buf, uint32_t offset, uint32_t data)
{
unsigned char lower_data;
if (*buf == NULL)
*buf = buffer_create_dynamic(system_pool, 1024, (size_t)-1);
/* data is in big endian, we want to update only the lowest byte */
offset += sizeof(uint32_t) - 1;
buffer_append(*buf, &offset, sizeof(offset));
lower_data = data & 0xff;
buffer_append(*buf, &lower_data, 1);
}
static int write_mark_updates(struct mail_index *index, buffer_t *marks,
const char *path, int fd)
{
const unsigned char *data, *end;
uint32_t offset;
size_t size;
data = buffer_get_data(marks, &size);
end = data + size;
while (data < end) {
memcpy(&offset, data, sizeof(offset));
data += sizeof(offset);
if (pwrite(fd, data, sizeof(*data), offset) < 0) {
index_file_set_syscall_error(index, path, "pwrite()");
return FALSE;
}
data++;
}
return TRUE;
}
static void write_mark_updates_in_memory(buffer_t *marks, void *mmap_base,
size_t mmap_length)
{
const unsigned char *data, *end;
uint32_t offset;
size_t size;
data = buffer_get_data(marks, &size);
end = data + size;
while (data < end) {
memcpy(&offset, data, sizeof(offset));
data += sizeof(offset);
i_assert(offset < mmap_length);
((char *) mmap_base)[offset] = *data;
data++;
}
}
static void commit_all_changes_in_memory(struct mail_cache_transaction_ctx *ctx)
{
struct mail_cache *cache = ctx->cache;
if (ctx->cache_marks != NULL) {
write_mark_updates_in_memory(ctx->cache_marks,
cache->mmap_base,
cache->mmap_length);
}
if (ctx->index_marks != NULL) {
write_mark_updates_in_memory(ctx->index_marks,
cache->index->mmap_base,
cache->index->mmap_used_length);
}
}
static int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
{
struct mail_cache *cache = ctx->cache;
uint32_t cont;
if (ctx->cache->anon_mmap) {
commit_all_changes_in_memory(ctx);
return TRUE;
}
/* write everything to disk */
if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0)
return mail_cache_set_syscall_error(cache, "msync()");
if (fdatasync(cache->fd) < 0)
return mail_cache_set_syscall_error(cache, "fdatasync()");
if (ctx->cache_marks != NULL &&
buffer_get_used_size(ctx->cache_marks) != 0) {
/* now that we're sure it's there, set on all the used-bits */
if (!write_mark_updates(cache->index, ctx->cache_marks,
cache->filepath, cache->fd))
return FALSE;
/* update continued records count */
cont = nbo_to_uint32(cache->header->continued_record_count);
cont += buffer_get_used_size(ctx->cache_marks) /
(sizeof(uint32_t)+1);
if (cont * 100 / cache->index->header->messages_count >=
COMPRESS_CONTINUED_PERCENTAGE &&
cache->used_file_size >= COMPRESS_MIN_SIZE) {
/* too many continued rows, compress */
cache->index->set_flags |=
MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
}
cache->header->continued_record_count = uint32_to_nbo(cont);
}
/* write index last */
if (ctx->index_marks != NULL &&
buffer_get_used_size(ctx->index_marks) != 0) {
if (!mail_index_fmdatasync(cache->index,
cache->index->mmap_used_length))
return FALSE;
if (!write_mark_updates(cache->index, ctx->index_marks,
cache->index->filepath,
cache->index->fd))
return FALSE;
}
return TRUE;
}
int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
{
int ret = TRUE;
if (ctx->last_idx != (unsigned int)-1) {
if (!mail_cache_write(ctx))
return FALSE;
}
ctx->cache->header->used_file_size =
uint32_to_nbo(ctx->cache->used_file_size);
if (!commit_all_changes(ctx))
ret = FALSE;
mail_cache_transaction_flush(ctx);
return ret;
}
int mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx)
{
/* no need to actually modify the file - we just didn't update
used_file_size */
ctx->cache->used_file_size =
nbo_to_uint32(ctx->cache->header->used_file_size);
mail_cache_transaction_flush(ctx);
return TRUE;
}
static int mail_cache_grow(struct mail_cache *cache, uint32_t size)
{
struct stat st;
void *base;
uoff_t grow_size, new_fsize;
new_fsize = cache->used_file_size + size;
grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE;
if (grow_size < 16384)
grow_size = 16384;
new_fsize += grow_size;
new_fsize &= ~1023;
if (cache->anon_mmap) {
i_assert(new_fsize < SSIZE_T_MAX);
base = mremap_anon(cache->mmap_base, cache->mmap_length,
(size_t)new_fsize, MREMAP_MAYMOVE);
if (base == MAP_FAILED) {
mail_cache_set_syscall_error(cache, "mremap_anon()");
return FALSE;
}
cache->mmap_base = base;
cache->mmap_length = (size_t)new_fsize;
cache->header = cache->mmap_base;
return TRUE;
}
if (fstat(cache->fd, &st) < 0)
return mail_cache_set_syscall_error(cache, "fstat()");
if (cache->used_file_size + size <= (uoff_t)st.st_size) {
/* no need to grow, just update mmap */
if (!mmap_update(cache, 0, 0))
return FALSE;
i_assert(cache->mmap_length >= (uoff_t)st.st_size);
return TRUE;
}
if (st.st_size < (off_t)sizeof(struct mail_cache_header))
return mail_cache_set_corrupted(cache, "Header is missing");
if (file_set_size(cache->fd, (off_t)new_fsize) < 0)
return mail_cache_set_syscall_error(cache, "file_set_size()");
return mmap_update(cache, 0, 0);
}
static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx,
uint32_t size)
{
/* NOTE: must be done within transaction or rollback would break it */
uint32_t offset;
i_assert((size % sizeof(uint32_t)) == 0);
offset = ctx->cache->used_file_size;
if (offset + size > ctx->cache->mmap_length) {
if (!mail_cache_grow(ctx->cache, size))
return 0;
}
ctx->cache->used_file_size += size;
return offset;
}
static const char *
mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx)
{
uint32_t offset, data_size;
unsigned char *buf;
offset = nbo_to_uint32(cache->header->header_offsets[idx]);
if ((offset & 2) == 0 &&
(cache->trans_ctx == NULL ||
cache->trans_ctx->next_unused_header_lowwater <= idx))
return NULL;
offset &= ~2;
if (!mmap_update(cache, offset, 1024))
return NULL;
if (offset + sizeof(data_size) > cache->mmap_length) {
mail_cache_set_corrupted(cache, "Header %u points outside file",
idx);
return NULL;
}
buf = cache->mmap_base;
memcpy(&data_size, buf + offset, sizeof(data_size));
data_size = nbo_to_uint32(data_size);
offset += sizeof(data_size);
if (data_size == 0) {
mail_cache_set_corrupted(cache,
"Header %u points to empty string", idx);
return NULL;
}
if (!mmap_update(cache, offset, data_size))
return NULL;
if (offset + data_size > cache->mmap_length) {
mail_cache_set_corrupted(cache, "Header %u points outside file",
idx);
return NULL;
}
buf = cache->mmap_base;
if (buf[offset + data_size - 1] != '\0') {
mail_cache_set_corrupted(cache,
"Header %u points to invalid string", idx);
return NULL;
}
return buf + offset;
}
static const char *const *
split_header(struct mail_cache *cache, const char *header)
{
const char *const *arr, *const *tmp;
const char *null = NULL;
char *str;
buffer_t *buf;
if (header == NULL)
return NULL;
arr = t_strsplit(header, "\n");
buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1);
for (tmp = arr; *tmp != NULL; tmp++) {
str = p_strdup(cache->split_header_pool, *tmp);
buffer_append(buf, &str, sizeof(str));
}
buffer_append(buf, &null, sizeof(null));
return buffer_get_data(buf, NULL);
}
const char *const *mail_cache_get_header_fields(struct mail_cache *cache,
unsigned int idx)
{
const char *str;
int i;
i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
/* t_strsplit() is a bit slow, so we cache it */
if (cache->header->header_offsets[idx] != cache->split_offsets[idx]) {
p_clear(cache->split_header_pool);
t_push();
for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
cache->split_offsets[i] =
cache->header->header_offsets[i];
str = mail_cache_get_header_fields_str(cache, idx);
cache->split_headers[i] = split_header(cache, str);
}
t_pop();
}
return cache->split_headers[idx];
}
int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
unsigned int idx, const char *const headers[])
{
struct mail_cache *cache = ctx->cache;
uint32_t offset, update_offset, size;
buffer_t *buffer;
i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
i_assert(idx >= ctx->next_unused_header_lowwater);
i_assert((nbo_to_uint32(cache->header->header_offsets[idx]) & 2) == 0);
t_push();
buffer = buffer_create_dynamic(data_stack_pool, 512, (size_t)-1);
while (*headers != NULL) {
if (buffer_get_used_size(buffer) != 0)
buffer_append(buffer, "\n", 1);
buffer_append(buffer, *headers, strlen(*headers));
headers++;
}
buffer_append(buffer, null4, 1);
size = buffer_get_used_size(buffer);
if ((size & 3) != 0) {
buffer_append(buffer, null4, 4 - (size & 3));
size += 4 - (size & 3);
}
offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
if (offset != 0) {
memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
buffer_get_data(buffer, NULL), size);
size = uint32_to_nbo(size);
memcpy((char *) cache->mmap_base + offset,
&size, sizeof(uint32_t));
cache->header->header_offsets[idx] = uint32_to_nbo(offset);
/* mark used-bit to be updated later. not really needed for
read-safety, but if transaction get rolled back we can't let
this point to invalid location. */
update_offset = (char *) &cache->header->header_offsets[idx] -
(char *) cache->mmap_base;
mark_update(&ctx->cache_marks, update_offset, offset | 2);
/* make sure get_header_fields() still works for this header
while the transaction isn't yet committed. */
ctx->next_unused_header_lowwater = idx + 1;
}
t_pop();
return offset > 0;
}
static struct mail_cache_record *
cache_get_record(struct mail_cache *cache, uint32_t offset)
{
struct mail_cache_record *cache_rec;
size_t size;
offset = nbo_to_uint32(offset);
if ((offset & 1) != 0) {
mail_cache_set_corrupted(cache, "bit 0 set in data offset");
return NULL;
}
if ((offset & 2) == 0)
return NULL;
offset &= ~2;
if (!mmap_update(cache, offset, sizeof(*cache_rec) + 1024))
return NULL;
if (offset + sizeof(*cache_rec) > cache->mmap_length) {
mail_cache_set_corrupted(cache, "record points outside file");
return NULL;
}
cache_rec = CACHE_RECORD(cache, offset);
size = nbo_to_uint32(cache_rec->size);
if (!mmap_update(cache, offset, sizeof(*cache_rec) + size))
return NULL;
if (offset + sizeof(*cache_rec) + size > cache->mmap_length) {
mail_cache_set_corrupted(cache, "record points outside file");
return NULL;
}
return cache_rec;
}
static struct mail_cache_record *
cache_get_next_record(struct mail_cache *cache, struct mail_cache_record *rec)
{
struct mail_cache_record *next;
next = cache_get_record(cache, rec->next_offset);
if (next != NULL && next <= rec) {
mail_cache_set_corrupted(cache, "next_offset points backwards");
return NULL;
}
return next;
}
static int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
{
struct mail_cache *cache = ctx->cache;
struct mail_cache_record *cache_rec, *next;
struct mail_index_record *rec;
uint32_t write_offset, update_offset;
const void *buf;
size_t size, buf_size;
buf = buffer_get_data(ctx->cache_data, &buf_size);
size = sizeof(*cache_rec) + buf_size;
ctx->cache_rec.size = uint32_to_nbo(size);
write_offset = mail_cache_append_space(ctx, size);
if (write_offset == 0)
return FALSE;
rec = INDEX_RECORD_AT(ctx->cache->index, ctx->last_idx);
ctx->last_idx = (unsigned int)-1;
cache_rec = cache_get_record(cache, rec->cache_offset);
if (cache_rec == NULL) {
/* first cache record - update offset in index file */
i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE);
rec->cache_offset = uint32_to_nbo(write_offset);
/* mark used-bit to be updated later */
update_offset = (char *) &rec->cache_offset -
(char *) cache->index->mmap_base;
mark_update(&ctx->index_marks, update_offset, write_offset | 2);
} else {
/* find the last cache record */
while ((next = cache_get_next_record(cache, cache_rec)) != NULL)
cache_rec = next;
/* set our offset, keep the used-bit still unset */
cache_rec->next_offset = uint32_to_nbo(write_offset);
/* mark used-bit to be updated later */
update_offset = (char *) &cache_rec->next_offset -
(char *) cache->mmap_base;
mark_update(&ctx->cache_marks, update_offset, write_offset | 2);
}
memcpy((char *) cache->mmap_base + write_offset,
&ctx->cache_rec, sizeof(ctx->cache_rec));
memcpy((char *) cache->mmap_base + write_offset +
sizeof(ctx->cache_rec), buf, buf_size);
/* reset the write context */
memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
buffer_set_used_size(ctx->cache_data, 0);
return TRUE;
}
static struct mail_cache_record *
mail_cache_lookup(struct mail_cache *cache, const struct mail_index_record *rec)
{
struct mail_cache_record *cache_rec;
unsigned int idx;
if (cache->trans_ctx != NULL &&
cache->trans_ctx->first_uid <= rec->uid &&
cache->trans_ctx->last_uid >= rec->uid) {
/* we have to auto-commit since we're not capable of looking
into uncommitted records. it would be possible by checking
index_marks and cache_marks, but it's just more trouble
than worth. */
idx = INDEX_RECORD_INDEX(cache->index, rec);
if (cache->trans_ctx->last_idx == idx) {
if (!mail_cache_write(cache->trans_ctx))
return NULL;
}
if (!mail_cache_transaction_commit(cache->trans_ctx))
return NULL;
}
cache_rec = cache_get_record(cache, rec->cache_offset);
if (cache_rec == NULL)
return NULL;
return cache_rec;
}
static int get_field_num(enum mail_cache_field field)
{
int i;
for (i = 0; i < 32; i++) {
if ((field & nbo32_bitmasks[i]) != 0)
return i;
}
return -1;
}
static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx,
enum mail_cache_field field)
{
const unsigned char *buf;
unsigned int mask;
uint32_t data_size;
size_t offset = 0;
int i;
buf = buffer_get_data(ctx->cache_data, NULL);
for (i = 0; i < 32; i++) {
mask = nbo32_bitmasks[i];
if ((field & mask) != 0)
return offset;
if ((ctx->cache_rec.fields & mask) != 0) {
if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
data_size = mail_cache_field_sizes[i];
else {
memcpy(&data_size, buf + offset,
sizeof(data_size));
data_size = nbo_to_uint32(data_size);
offset += sizeof(data_size);
}
offset += (data_size + 3) & ~3;
}
}
i_unreached();
return offset;
}
int mail_cache_add(struct mail_cache_transaction_ctx *ctx,
struct mail_index_record *rec, enum mail_cache_field field,
const void *data, size_t data_size)
{
uint32_t nb_data_size;
size_t full_size, offset;
unsigned char *buf;
unsigned int idx;
int field_num;
i_assert(data_size > 0);
i_assert(data_size < (uint32_t)-1);
nb_data_size = uint32_to_nbo((uint32_t)data_size);
if ((field & MAIL_CACHE_FIXED_MASK) != 0) {
field_num = get_field_num(field);
i_assert(field_num != -1);
i_assert(mail_cache_field_sizes[field_num] == data_size);
} else if ((field & MAIL_CACHE_STRING_MASK) != 0) {
i_assert(((char *) data)[data_size-1] == '\0');
}
/* NOTE: we use index because the record pointer might not last. */
idx = INDEX_RECORD_INDEX(ctx->cache->index, rec);
if (ctx->last_idx != idx && ctx->last_idx != (unsigned int)-1) {
if (!mail_cache_write(ctx))
return FALSE;
}
ctx->last_idx = idx;
i_assert((ctx->cache_rec.fields & field) == 0);
full_size = (data_size + 3) & ~3;
if ((field & MAIL_CACHE_FIXED_MASK) == 0)
full_size += sizeof(nb_data_size);
/* fields must be ordered. find where to insert it. */
if (field > ctx->cache_rec.fields)
buf = buffer_append_space_unsafe(ctx->cache_data, full_size);
else {
offset = get_insert_offset(ctx, field);
buffer_copy(ctx->cache_data, offset + full_size,
ctx->cache_data, offset, (size_t)-1);
buf = buffer_get_space_unsafe(ctx->cache_data,
offset, full_size);
}
ctx->cache_rec.fields |= field;
/* @UNSAFE */
if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
memcpy(buf, &nb_data_size, sizeof(nb_data_size));
buf += sizeof(nb_data_size);
}
memcpy(buf, data, data_size); buf += data_size;
if ((data_size & 3) != 0)
memset(buf, 0, 4 - (data_size & 3));
/* remember the transaction uid range */
if (rec->uid < ctx->first_uid || ctx->first_uid == 0)
ctx->first_uid = rec->uid;
if (rec->uid > ctx->last_uid)
ctx->last_uid = rec->uid;
return TRUE;
}
int mail_cache_delete(struct mail_cache_transaction_ctx *ctx,
struct mail_index_record *rec)
{
struct mail_cache *cache = ctx->cache;
struct mail_cache_record *cache_rec;
uint32_t deleted_space;
uoff_t max_del_space;
cache_rec = mail_cache_lookup(cache, rec);
if (cache_rec == NULL)
return TRUE;
/* NOTE: it would be nice to erase the cached data for the record,
but some other processes might still be using them. So, we just
update the deleted_space in header */
deleted_space = nbo_to_uint32(cache->header->deleted_space);
do {
deleted_space -= nbo_to_uint32(cache_rec->size);
cache_rec = cache_get_next_record(cache, cache_rec);
} while (cache_rec != NULL);
/* see if we've reached the max. deleted space in file */
max_del_space = cache->used_file_size / 100 * COMPRESS_PERCENTAGE;
if (deleted_space >= max_del_space &&
cache->used_file_size >= COMPRESS_MIN_SIZE)
cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
cache->header->deleted_space = uint32_to_nbo(deleted_space);
return TRUE;
}
enum mail_cache_field
mail_cache_get_fields(struct mail_cache *cache,
const struct mail_index_record *rec)
{
struct mail_cache_record *cache_rec;
enum mail_cache_field fields = 0;
cache_rec = mail_cache_lookup(cache, rec);
while (cache_rec != NULL) {
fields |= cache_rec->fields;
cache_rec = cache_get_next_record(cache, cache_rec);
}
return fields;
}
static int cache_get_field(struct mail_cache *cache,
struct mail_cache_record *cache_rec,
enum mail_cache_field field,
void **data_r, size_t *size_r)
{
unsigned char *buf;
unsigned int mask;
uint32_t rec_size, data_size;
size_t offset = 0;
int i;
rec_size = nbo_to_uint32(cache_rec->size);
buf = (unsigned char *) cache_rec + sizeof(*cache_rec);
for (i = 0; i < 32; i++) {
mask = nbo32_bitmasks[i];
if ((cache_rec->fields & mask) == 0)
continue;
if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
data_size = mail_cache_field_sizes[i];
else {
memcpy(&data_size, buf + offset, sizeof(data_size));
data_size = nbo_to_uint32(data_size);
offset += sizeof(data_size);
}
if (field == mask) {
if (data_size == 0) {
mail_cache_set_corrupted(cache,
"Field size is 0");
return FALSE;
}
*data_r = buf + offset;
*size_r = data_size;
return TRUE;
}
offset += (data_size + 3) & ~3;
if (offset >= rec_size) {
mail_cache_set_corrupted(cache,
"Record continues outside it's allocated size");
return FALSE;
}
}
i_unreached();
return FALSE;
}
static int cache_lookup_field(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field field,
void **data_r, size_t *size_r)
{
struct mail_cache_record *cache_rec;
cache_rec = mail_cache_lookup(cache, rec);
while (cache_rec != NULL) {
if ((cache_rec->fields & field) != 0) {
return cache_get_field(cache, cache_rec, field,
data_r, size_r);
}
cache_rec = cache_get_next_record(cache, cache_rec);
}
return FALSE;
}
int mail_cache_lookup_field(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field field,
const void **data_r, size_t *size_r)
{
void *data;
if (!cache_lookup_field(cache, rec, field, &data, size_r))
return FALSE;
*data_r = data;
return TRUE;
}
const char *mail_cache_lookup_string_field(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field field)
{
const void *data;
size_t size;
i_assert((field & MAIL_CACHE_STRING_MASK) != 0);
if (!mail_cache_lookup_field(cache, rec, field, &data, &size))
return NULL;
if (((const char *) data)[size-1] != '\0') {
mail_cache_set_corrupted(cache,
"String field %x doesn't end with NUL", field);
return NULL;
}
return data;
}
int mail_cache_copy_fixed_field(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field field,
void *buffer, size_t buffer_size)
{
const void *data;
size_t size;
i_assert((field & MAIL_CACHE_FIXED_MASK) != 0);
if (!mail_cache_lookup_field(cache, rec, field, &data, &size))
return FALSE;
if (buffer_size != size) {
i_panic("cache: fixed field %x wrong size "
"(%"PRIuSIZE_T" vs %"PRIuSIZE_T")",
field, size, buffer_size);
}
memcpy(buffer, data, buffer_size);
return TRUE;
}
void mail_cache_mark_missing(struct mail_cache *cache,
enum mail_cache_field fields)
{
// FIXME: count these
}
enum mail_index_record_flag
mail_cache_get_index_flags(struct mail_cache *cache,
const struct mail_index_record *rec)
{
enum mail_index_record_flag flags;
if (!mail_cache_copy_fixed_field(cache, rec, MAIL_CACHE_INDEX_FLAGS,
&flags, sizeof(flags)))
return 0;
return flags;
}
int mail_cache_update_index_flags(struct mail_cache *cache,
struct mail_index_record *rec,
enum mail_index_record_flag flags)
{
void *data;
size_t size;
i_assert(cache->locks > 0);
if (!cache_lookup_field(cache, rec, MAIL_CACHE_INDEX_FLAGS,
&data, &size)) {
mail_cache_set_corrupted(cache,
"Missing index flags for record %u", rec->uid);
return FALSE;
}
memcpy(data, &flags, sizeof(flags));
return TRUE;
}
void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size)
{
if (!mmap_update(cache, 0, 0))
return NULL;
*size = cache->mmap_length;
return cache->mmap_base;
}
int mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
{
va_list va;
mail_cache_mark_file_deleted(cache);
cache->index->inconsistent = TRUE; /* easiest way to rebuild */
if (cache->silent)
return FALSE;
va_start(va, fmt);
t_push();
index_set_error(cache->index, "Corrupted index cache file %s: %s",
cache->filepath, t_strdup_vprintf(fmt, va));
t_pop();
va_end(va);
return FALSE;
}