mail-index-data.c revision 67950e1843bd879bab73d8752ee54f60bbd21987
89a126810703c666309310d0f3189e9834d70b5bTimo Sirainen/* Copyright (C) 2002 Timo Sirainen */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "lib.h"
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen#include "file-set-size.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "mmap-util.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "write-full.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "mail-index.h"
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen#include "mail-index-data.h"
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen#include "mail-index-util.h"
c4cfee078c4a185b5ba8f0c55f51275b7e885b2cTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include <stdio.h>
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include <fcntl.h>
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#define DATA_FILE_POSITION(data, rec) \
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen ((uoff_t) ((char *) (rec) - (char *) ((data)->mmap_base)))
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen/* Never compress the file if it's smaller than this */
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen#define COMPRESS_MIN_SIZE (1024*50)
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen/* Compress the file when deleted space reaches n% of total size */
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen#define COMPRESS_PERCENTAGE 20
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen/* Initial size for the file */
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen#define INDEX_DATA_INITIAL_SIZE (sizeof(MailIndexDataHeader) + 10240)
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen/* When more space is needed, grow the file n% larger than the previous size */
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen#define INDEX_DATA_GROW_PERCENTAGE 10
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainenstruct _MailIndexData {
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen MailIndex *index;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen int fd;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen char *filepath;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen void *mmap_base;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen size_t mmap_full_length;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen size_t mmap_used_length;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen MailIndexDataHeader *header;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen unsigned int anon_mmap:1;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen unsigned int dirty_mmap:1;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen unsigned int modified:1;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen unsigned int fsynced:1;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen};
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainenint index_data_set_corrupted(MailIndexData *data, const char *fmt, ...)
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen{
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen va_list va;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen INDEX_MARK_CORRUPTED(data->index);
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen data->index->inconsistent = TRUE;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen va_start(va, fmt);
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen t_push();
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen index_set_error(data->index, "Corrupted index data file %s: %s",
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen data->filepath, t_strdup_vprintf(fmt, va));
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen t_pop();
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen va_end(va);
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen return FALSE;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen}
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainenstatic int index_data_set_syscall_error(MailIndexData *data,
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen const char *function)
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen{
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen i_assert(function != NULL);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_set_error(data->index, "%s failed with index data file %s: %m",
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen function, data->filepath);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return FALSE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
8d6cb44a0161d88743756733f83c4fb278485987Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic void mail_index_data_file_close(MailIndexData *data)
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->anon_mmap) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (munmap_anon(data->mmap_base, data->mmap_full_length) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_data_set_syscall_error(data, "munmap_anon()");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen } else if (data->mmap_base != NULL) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (munmap(data->mmap_base, data->mmap_full_length) < 0)
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen index_data_set_syscall_error(data, "munmap()");
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->mmap_base = NULL;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->mmap_used_length = 0;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen data->mmap_full_length = 0;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen if (data->fd != -1) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (close(data->fd) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_data_set_syscall_error(data, "close()");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->fd = -1;
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen }
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen}
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainenstatic int data_file_reopen(MailIndexData *data)
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen{
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen int fd;
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen i_assert(!data->anon_mmap);
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen fd = open(data->filepath, O_RDWR);
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen if (fd == -1)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return index_data_set_syscall_error(data, "open()");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_index_data_file_close(data);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->fd = fd;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return TRUE;
0cea9b1f4fa0495a48f5f097e40492517d67e1baTimo Sirainen}
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic int mmap_update(MailIndexData *data, uoff_t pos, size_t size)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
0cea9b1f4fa0495a48f5f097e40492517d67e1baTimo Sirainen MailIndexDataHeader *hdr;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->header != NULL &&
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->header->indexid != data->index->indexid) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->header->indexid != 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* index was just rebuilt. we should have noticed
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen this before at index->set_lock() though. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_set_error(data->index,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "Warning: Inconsistency - Index "
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "%s was rebuilt while we had it open",
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->filepath);
8d6cb44a0161d88743756733f83c4fb278485987Timo Sirainen data->index->inconsistent = TRUE;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen return FALSE;
8d6cb44a0161d88743756733f83c4fb278485987Timo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* data file was deleted, reopen it */
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen if (!data_file_reopen(data))
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen return FALSE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
8d6cb44a0161d88743756733f83c4fb278485987Timo Sirainen /* force mmap refresh */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen size = 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (size != 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (pos + size <= data->mmap_used_length)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (pos + size <= data->mmap_full_length) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->mmap_used_length = data->header->used_file_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->mmap_used_length <= data->mmap_full_length)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* file size changed, re-mmap() */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen i_assert(!data->anon_mmap);
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen if (data->mmap_base != NULL) {
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen if (data->modified &&
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen msync(data->mmap_base, data->mmap_used_length, MS_SYNC) < 0)
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen return index_data_set_syscall_error(data, "msync()");
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen data->modified = FALSE;
bc793bfcee945ce8871edfa298fe7235744425b6Timo Sirainen data->fsynced = FALSE;
bc793bfcee945ce8871edfa298fe7235744425b6Timo Sirainen
bc793bfcee945ce8871edfa298fe7235744425b6Timo Sirainen if (munmap(data->mmap_base, data->mmap_full_length) < 0)
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen index_data_set_syscall_error(data, "munmap()");
ea4ca37ec14913354f3a0deebc0df96097eb9468Timo Sirainen }
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen
ea4ca37ec14913354f3a0deebc0df96097eb9468Timo Sirainen data->header = NULL;
ea4ca37ec14913354f3a0deebc0df96097eb9468Timo Sirainen data->mmap_used_length = 0;
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen data->mmap_base = mmap_rw_file(data->fd, &data->mmap_full_length);
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen if (data->mmap_base == MAP_FAILED) {
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen data->mmap_base = NULL;
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen return index_data_set_syscall_error(data, "mmap()");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->mmap_full_length < sizeof(MailIndexDataHeader))
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return index_data_set_corrupted(data, "File too small");
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen hdr = data->mmap_base;
abb83d133dd082527d500916fca66a72fbbbaa8dTimo Sirainen
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen if (hdr->used_file_size < sizeof(MailIndexDataHeader)) {
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen index_data_set_corrupted(data, "used_file_size too small ("
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "%"PRIuUOFF_T")", hdr->used_file_size);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return FALSE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (hdr->used_file_size > data->mmap_full_length) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_data_set_corrupted(data, "used_file_size larger than "
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "real file size (%"PRIuUOFF_T
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen " vs %"PRIuSIZE_T")",
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen hdr->used_file_size,
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen data->mmap_full_length);
44ff75ca53188056ff5a3e50428e3f2078800b3cTimo Sirainen return FALSE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data->mmap_used_length = hdr->used_file_size;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data->header = hdr;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen debug_mprotect(data->mmap_base, data->mmap_full_length, data->index);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen return TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenint mail_index_data_open(MailIndex *index)
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen{
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen MailIndexData *data;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen const char *path;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen int fd;
c4cfee078c4a185b5ba8f0c55f51275b7e885b2cTimo Sirainen
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen path = t_strconcat(index->filepath, DATA_FILE_PREFIX, NULL);
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen fd = open(path, O_RDWR);
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen if (fd == -1) {
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen if (errno == ENOENT) {
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen /* doesn't exist, rebuild the index */
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen INDEX_MARK_CORRUPTED(index);
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen }
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen return index_file_set_syscall_error(index, path, "open()");
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen }
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen data = i_new(MailIndexData, 1);
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen data->index = index;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen data->fd = fd;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen data->filepath = i_strdup(path);
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen index->data = data;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen if (!mmap_update(data, 0, sizeof(MailIndexDataHeader))) {
c4cfee078c4a185b5ba8f0c55f51275b7e885b2cTimo Sirainen mail_index_data_free(data);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return FALSE;
44ff75ca53188056ff5a3e50428e3f2078800b3cTimo Sirainen }
abb83d133dd082527d500916fca66a72fbbbaa8dTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* verify that this really is the data file for wanted index */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->header->indexid != index->indexid) {
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen INDEX_MARK_CORRUPTED(index);
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen index_set_error(index, "IndexID mismatch for data file %s",
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen path);
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen mail_index_data_free(data);
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen return FALSE;
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen }
abb83d133dd082527d500916fca66a72fbbbaa8dTimo Sirainen
abb83d133dd082527d500916fca66a72fbbbaa8dTimo Sirainen return TRUE;
abb83d133dd082527d500916fca66a72fbbbaa8dTimo Sirainen}
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainenstatic const char *init_data_file(MailIndex *index, MailIndexDataHeader *hdr,
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen int fd, const char *temppath)
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen{
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen const char *realpath;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen if (write_full(fd, hdr, sizeof(MailIndexDataHeader)) < 0) {
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen index_file_set_syscall_error(index, temppath, "write_full()");
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen return NULL;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen }
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen if (file_set_size(fd, INDEX_DATA_INITIAL_SIZE) < 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_file_set_syscall_error(index, temppath,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "file_set_size()");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return NULL;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* move temp file into .data file, deleting old one
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if it already exists */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen realpath = t_strconcat(index->filepath, DATA_FILE_PREFIX, NULL);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (rename(temppath, realpath) < 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_set_error(index, "rename(%s, %s) failed: %m",
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen temppath, realpath);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen return NULL;
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return realpath;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenint mail_index_data_create(MailIndex *index)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen MailIndexDataHeader hdr;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen MailIndexData *data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen const char *temppath, *realpath;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen int fd;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen memset(&hdr, 0, sizeof(MailIndexDataHeader));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen hdr.indexid = index->indexid;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen hdr.used_file_size = sizeof(MailIndexDataHeader);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen realpath = NULL;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* we'll do anon-mmaping only if initially requested. if we fail
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen because of out of disk space, we'll just let the main index code
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen know it and fail. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (index->nodiskspace) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen fd = -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen fd = mail_index_create_temp_file(index, &temppath);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (fd == -1) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (errno == ENOSPC)
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen index->nodiskspace = TRUE;
11768c622a8e5aaf0e5d9b3c9a872867b62b4830Timo Sirainen return FALSE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
11768c622a8e5aaf0e5d9b3c9a872867b62b4830Timo Sirainen
265e9d8457aa5ce01b7aad954632aed8c36deeb8Timo Sirainen realpath = init_data_file(index, &hdr, fd, temppath);
265e9d8457aa5ce01b7aad954632aed8c36deeb8Timo Sirainen if (realpath == NULL) {
265e9d8457aa5ce01b7aad954632aed8c36deeb8Timo Sirainen if (errno == ENOSPC)
265e9d8457aa5ce01b7aad954632aed8c36deeb8Timo Sirainen index->nodiskspace = TRUE;
11768c622a8e5aaf0e5d9b3c9a872867b62b4830Timo Sirainen
265e9d8457aa5ce01b7aad954632aed8c36deeb8Timo Sirainen (void)close(fd);
265e9d8457aa5ce01b7aad954632aed8c36deeb8Timo Sirainen (void)unlink(temppath);
265e9d8457aa5ce01b7aad954632aed8c36deeb8Timo Sirainen return FALSE;
11768c622a8e5aaf0e5d9b3c9a872867b62b4830Timo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data = i_new(MailIndexData, 1);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (fd == -1) {
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data->mmap_full_length = INDEX_DATA_INITIAL_SIZE;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data->mmap_base = mmap_anon(data->mmap_full_length);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen memcpy(data->mmap_base, &hdr, sizeof(MailIndexDataHeader));
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data->header = data->mmap_base;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->mmap_used_length = data->header->used_file_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->anon_mmap = TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->filepath = i_strdup("(in-memory index data)");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->filepath = i_strdup(realpath);
6ebcdea168735ee76e32b871c1f50f3526690447Timo Sirainen }
6ebcdea168735ee76e32b871c1f50f3526690447Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen data->index = index;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->fd = fd;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (!mmap_update(data, 0, sizeof(MailIndexDataHeader))) {
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen mail_index_data_free(data);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return FALSE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index->data = data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return TRUE;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen}
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainenvoid mail_index_data_free(MailIndexData *data)
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen{
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen data->index->data = NULL;
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen mail_index_data_file_close(data);
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen i_free(data->filepath);
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen i_free(data);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen}
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainenint mail_index_data_reset(MailIndexData *data)
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen{
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen MailIndexDataHeader hdr;
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen memset(&hdr, 0, sizeof(MailIndexDataHeader));
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen hdr.indexid = data->index->indexid;
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen hdr.used_file_size = sizeof(MailIndexDataHeader);
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen
7bb939ef70752f2731d27b18c944ea94e5b23eb5Timo Sirainen if (data->anon_mmap) {
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen memcpy(data->mmap_base, &hdr, sizeof(MailIndexDataHeader));
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen return TRUE;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen }
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (file_set_size(data->fd, INDEX_DATA_INITIAL_SIZE) < 0) {
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (errno == ENOSPC)
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen data->index->nodiskspace = TRUE;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen return index_data_set_syscall_error(data, "file_set_size()");
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen }
8e574a603c2c02105b073906a7ca91904271b80eTimo Sirainen
8e574a603c2c02105b073906a7ca91904271b80eTimo Sirainen if (lseek(data->fd, 0, SEEK_SET) < 0)
8e574a603c2c02105b073906a7ca91904271b80eTimo Sirainen return index_data_set_syscall_error(data, "lseek()");
8e574a603c2c02105b073906a7ca91904271b80eTimo Sirainen
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen if (write_full(data->fd, &hdr, sizeof(MailIndexDataHeader)) < 0) {
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen if (errno == ENOSPC)
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen data->index->nodiskspace = TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return index_data_set_syscall_error(data, "write_full()");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen data->modified = FALSE;
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen data->fsynced = FALSE;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen return mmap_update(data, 0, 0);
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen}
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen
9c4c535b86e9473ad97c6e9242ed84f3d9d69d0dTimo Sirainenint mail_index_data_mark_file_deleted(MailIndexData *data)
9c4c535b86e9473ad97c6e9242ed84f3d9d69d0dTimo Sirainen{
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen if (data->anon_mmap)
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen return TRUE;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data->header->indexid = 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (msync(data->mmap_base, sizeof(MailIndexDataHeader), MS_SYNC) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return index_data_set_syscall_error(data, "msync()");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->fsynced = FALSE;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen return TRUE;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen}
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainenvoid mail_index_data_mark_modified(MailIndexData *data)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen{
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data->modified = TRUE;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen}
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainenstatic int mail_index_data_grow(MailIndexData *data, size_t size)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen{
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen void *base;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen uoff_t new_fsize;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen off_t pos;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen new_fsize = data->header->used_file_size + size;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen new_fsize += new_fsize / 100 * INDEX_DATA_GROW_PERCENTAGE;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen if (data->anon_mmap) {
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen i_assert(new_fsize < SSIZE_T_MAX);
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen base = mremap_anon(data->mmap_base, data->mmap_full_length,
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen (size_t)new_fsize, MREMAP_MAYMOVE);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (base == MAP_FAILED) {
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen index_data_set_syscall_error(data, "mremap_anon()");
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen return FALSE;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen }
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen data->mmap_base = base;
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen data->mmap_full_length = (size_t)new_fsize;
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen return TRUE;
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen }
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen pos = lseek(data->fd, 0, SEEK_END);
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen if (pos < 0)
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen return index_data_set_syscall_error(data, "lseek()");
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen if (data->header->used_file_size + size <= (uoff_t)pos) {
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen /* no need to grow, just update mmap */
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen if (!mmap_update(data, 0, 0))
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen return FALSE;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen i_assert(data->mmap_full_length >= (uoff_t)pos);
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen return TRUE;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen }
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen if (pos < (int)sizeof(MailIndexDataHeader))
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen return index_data_set_corrupted(data, "Header is missing");
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen if (file_set_size(data->fd, new_fsize) < 0) {
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen if (errno == ENOSPC)
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen data->index->nodiskspace = TRUE;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return index_data_set_syscall_error(data, "file_set_size()");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen return mmap_update(data, 0, 0);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen}
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainenuoff_t mail_index_data_append(MailIndexData *data, const void *buffer,
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen size_t size)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen{
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen uoff_t offset;
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen i_assert((size & (MEM_ALIGN_SIZE-1)) == 0);
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen i_assert(data->index->lock_type == MAIL_LOCK_EXCLUSIVE);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (size > data->mmap_full_length ||
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen data->mmap_full_length - size < data->header->used_file_size) {
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen if (!mail_index_data_grow(data, size))
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen return 0;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen }
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen offset = data->header->used_file_size;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen i_assert(offset + size <= data->mmap_full_length);
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen memcpy((char *) data->mmap_base + offset, buffer, size);
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen data->header->used_file_size += size;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data->modified = TRUE;
2c20ffcb5bb1ccdfdcd0b0ff0c7296f65b990362Timo Sirainen return offset;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen}
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainenint mail_index_data_delete(MailIndexData *data, MailIndexRecord *index_rec)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen{
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen MailIndexDataRecordHeader *rec_hdr;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen uoff_t max_del_space;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen i_assert(data->index->lock_type == MAIL_LOCK_EXCLUSIVE);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen rec_hdr = mail_index_data_lookup_header(data, index_rec);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (rec_hdr == NULL)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return FALSE;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen /* just mark it deleted. */
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen data->header->deleted_space += rec_hdr->data_size;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen /* see if we've reached the max. deleted space in file */
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen if (data->header->used_file_size >= COMPRESS_MIN_SIZE &&
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen (data->index->header->flags & MAIL_INDEX_FLAG_COMPRESS_DATA) == 0) {
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen max_del_space = data->header->used_file_size /
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen 100 * COMPRESS_PERCENTAGE;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (data->header->deleted_space >= max_del_space)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen data->index->set_flags |= MAIL_INDEX_FLAG_COMPRESS_DATA;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen }
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen data->modified = TRUE;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen return TRUE;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen}
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenint mail_index_data_sync_file(MailIndexData *data, int *fsync_fd)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen *fsync_fd = -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->anon_mmap)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (data->modified) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (msync(data->mmap_base, data->mmap_used_length, MS_SYNC) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return index_data_set_syscall_error(data, "msync()");
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen data->fsynced = FALSE;
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen }
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen if (!data->fsynced) {
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen data->fsynced = TRUE;
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen *fsync_fd = data->fd;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen return TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo SirainenMailIndexDataRecordHeader *
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainenmail_index_data_lookup_header(MailIndexData *data, MailIndexRecord *index_rec)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen{
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen uoff_t pos;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen pos = index_rec->data_position;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (pos == 0) {
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen /* data not yet written to record */
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen return NULL;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen }
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (!mmap_update(data, pos, sizeof(MailIndexDataRecordHeader)))
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen return NULL;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (pos + sizeof(MailIndexDataRecordHeader) > data->mmap_used_length) {
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen index_data_set_corrupted(data,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "Data position of record %u points outside file "
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "(%"PRIuUOFF_T" + %"PRIuSIZE_T" > %"PRIuSIZE_T")",
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen index_rec->uid, pos, sizeof(MailIndexDataRecordHeader),
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->mmap_used_length);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return NULL;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen if ((pos % MEM_ALIGN_SIZE) != 0) {
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen index_data_set_corrupted(data,
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen "Data position (%"PRIuUOFF_T") is not memory aligned "
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen "for record %u", pos, index_rec->uid);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return NULL;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return (MailIndexDataRecordHeader *) ((char *) data->mmap_base + pos);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
d5ac54ef50db16b50689b5c8b7bb64d344190832Timo SirainenMailIndexDataRecord *
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenmail_index_data_lookup(MailIndexData *data, MailIndexRecord *index_rec,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen MailDataField field)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen MailIndexDataRecordHeader *rec_hdr;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen MailIndexDataRecord *rec;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen uoff_t pos, max_pos;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_reset_error(data->index);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (index_rec->data_position == 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* data not yet written to record */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return NULL;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen rec_hdr = mail_index_data_lookup_header(data, index_rec);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (rec_hdr == NULL)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return NULL;
if (!mmap_update(data, index_rec->data_position, rec_hdr->data_size))
return NULL;
pos = index_rec->data_position;
max_pos = index_rec->data_position + rec_hdr->data_size;
if (pos > data->mmap_used_length ||
(data->mmap_used_length - pos < rec_hdr->data_size)) {
index_data_set_corrupted(data,
"Given data size larger than file size "
"(%"PRIuUOFF_T" + %u > %"PRIuSIZE_T") for record %u",
index_rec->data_position, rec_hdr->data_size,
data->mmap_used_length, index_rec->uid);
return NULL;
}
pos += sizeof(MailIndexDataRecordHeader);
do {
rec = (MailIndexDataRecord *) ((char *) data->mmap_base + pos);
if (rec->full_field_size > max_pos ||
pos + sizeof(MailIndexDataRecord) > max_pos ||
pos + DATA_RECORD_SIZE(rec) > max_pos) {
index_data_set_corrupted(data,
"Field %d size points outside file "
"(%"PRIuUOFF_T" / %"PRIuUOFF_T") for record %u",
(int)field, pos, max_pos, index_rec->uid);
break;
}
if ((rec->full_field_size % MEM_ALIGN_SIZE) != 0) {
index_data_set_corrupted(data,
"Field %d size %u is not memory aligned "
"for record %u", (int)field,
rec->full_field_size, index_rec->uid);
break;
}
if (rec->field == field || field == 0) {
/* match */
return rec;
} else if (rec->field < field) {
/* jump to next record */
pos += DATA_RECORD_SIZE(rec);
} else {
/* the fields are sorted by field type, so it's not
possible the wanted field could come after this. */
break;
}
} while (pos < max_pos);
return NULL;
}
MailIndexDataRecord *
mail_index_data_next(MailIndexData *data, MailIndexRecord *index_rec,
MailIndexDataRecord *rec)
{
MailIndexDataRecordHeader *rec_hdr;
uoff_t pos, end_pos, max_pos;
index_reset_error(data->index);
if (rec == NULL)
return NULL;
rec_hdr = (MailIndexDataRecordHeader *) ((char *) data->mmap_base +
index_rec->data_position);
/* get position to next record */
pos = DATA_FILE_POSITION(data, rec) + DATA_RECORD_SIZE(rec);
max_pos = index_rec->data_position + rec_hdr->data_size;
/* make sure it's within range */
if (pos >= max_pos)
return NULL;
rec = (MailIndexDataRecord *) ((char *) data->mmap_base + pos);
end_pos = pos + DATA_RECORD_SIZE(rec);
if (end_pos < pos || end_pos > max_pos) {
index_data_set_corrupted(data, "Field size points outside file "
"(%"PRIuUOFF_T" + %u > %"PRIuUOFF_T")",
pos, rec->full_field_size, max_pos);
return NULL;
}
return rec;
}
int mail_index_data_record_verify(MailIndexData *data, MailIndexDataRecord *rec)
{
int i;
if (rec->full_field_size > INT_MAX) {
/* we already checked that the full_field_size is within file,
so this can happen only if the file really is huge.. */
index_data_set_corrupted(data, "full_field_size (%u) > INT_MAX",
rec->full_field_size);
return FALSE;
}
/* make sure the data actually contains \0 */
for (i = (int)rec->full_field_size-1; i >= 0; i--) {
if (rec->data[i] == '\0') {
/* yes, everything ok */
return TRUE;
}
}
index_data_set_corrupted(data, "Missing \\0 with field %u "
"(%"PRIuUOFF_T")", rec->field,
DATA_FILE_POSITION(data, rec));
return FALSE;
}
void *mail_index_data_get_mmaped(MailIndexData *data, size_t *size)
{
if (!mmap_update(data, 0, 0))
return NULL;
*size = data->mmap_used_length;
return data->mmap_base;
}