mail-transaction-log.c revision 2ca4cb08680aebb1474d762738cf436871f095fb
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase/* Copyright (C) 2003-2004 Timo Sirainen */
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "lib.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "ioloop.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "buffer.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "file-lock.h"
87009975356ac69021bebf6a09e6c3b19dd2008bTim Reddehase#include "file-dotlock.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "read-full.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "write-full.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "mmap-util.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "mail-index-private.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "mail-index-view-private.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "mail-transaction-log-private.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#include "mail-transaction-util.h"
511ba3e3d7deb4a13a03ce522a99a0395acc61b7Tim Reddehase#include "mail-index-transaction-private.h"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase
a1133e03bf198594728c913fdfa6f1a8f825e4d3Tim Reddehase#include <stddef.h>
a1133e03bf198594728c913fdfa6f1a8f825e4d3Tim Reddehase#include <sys/stat.h>
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase/* this lock should never exist for a long time.. */
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#define LOG_DOTLOCK_TIMEOUT 30
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#define LOG_DOTLOCK_STALE_TIMEOUT 0
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#define LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT 300
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase#define LOG_NEW_DOTLOCK_SUFFIX ".newlock"
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehasestruct mail_transaction_add_ctx {
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase struct mail_transaction_log *log;
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase struct mail_index_view *view;
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase buffer_t *appends, *expunges;
5d6db2efe44fab2f3616d0876191ce52e0600984Tim Reddehase buffer_t *flag_updates, *cache_updates;
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase};
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehase
6cf3ba1dcf972672069f0826427619587d32d9e6Tim Reddehasestatic struct mail_transaction_log_file *
mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
const char *path);
static int mail_transaction_log_rotate(struct mail_transaction_log *log,
int lock_type);
static int
mail_transaction_log_file_lock(struct mail_transaction_log_file *file,
int lock_type);
static int mail_transaction_log_lock_head(struct mail_transaction_log *log);
void
mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
const char *fmt, ...)
{
va_list va;
file->hdr.indexid = 0;
if (pwrite_full(file->fd, &file->hdr.indexid,
sizeof(file->hdr.indexid), 0) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "pwrite()");
}
va_start(va, fmt);
t_push();
mail_index_set_error(file->log->index,
"Corrupted transaction log file %s: %s",
file->filepath, t_strdup_vprintf(fmt, va));
t_pop();
va_end(va);
}
#define INDEX_HAS_MISSING_LOGS(index, file) \
!(((file)->hdr.file_seq == (index)->hdr->log_file_seq && \
(index)->hdr->log_file_offset >= \
sizeof(struct mail_transaction_log_header)) || \
((file)->hdr.prev_file_seq == (index)->hdr->log_file_seq && \
(file)->hdr.prev_file_offset == (index)->hdr->log_file_offset))
static int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)
{
struct mail_index *index = log->index;
struct mail_transaction_log_file *file;
unsigned int lock_id;
int ret;
if (mail_transaction_log_lock_head(log) < 0)
return -1;
file = log->head;
file->refcount++;
ret = mail_index_lock_shared(index, TRUE, &lock_id);
if (ret == 0) {
ret = mail_index_map(index, FALSE);
if (ret <= 0)
ret = -1;
else if (INDEX_HAS_MISSING_LOGS(index, file)) {
/* broken - fix it by creating a new log file */
ret = mail_transaction_log_rotate(log, F_UNLCK);
}
}
if (--file->refcount == 0)
mail_transaction_logs_clean(log);
else
(void)mail_transaction_log_file_lock(file, F_UNLCK);
return ret;
}
struct mail_transaction_log *
mail_transaction_log_open_or_create(struct mail_index *index)
{
struct mail_transaction_log *log;
const char *path;
log = i_new(struct mail_transaction_log, 1);
log->index = index;
path = t_strconcat(log->index->filepath,
MAIL_TRANSACTION_LOG_PREFIX, NULL);
log->head = mail_transaction_log_file_open_or_create(log, path);
if (log->head == NULL) {
i_free(log);
return NULL;
}
if (index->fd != -1 &&
INDEX_HAS_MISSING_LOGS(index, log->head)) {
/* head log file isn't same as head index file -
shouldn't happen except in race conditions. lock them and
check again - FIXME: missing error handling. */
(void)mail_transaction_log_check_file_seq(log);
}
return log;
}
void mail_transaction_log_close(struct mail_transaction_log *log)
{
mail_transaction_log_views_close(log);
log->head->refcount--;
mail_transaction_logs_clean(log);
log->index->log = NULL;
i_free(log);
}
static int
mail_transaction_log_file_dotlock(struct mail_transaction_log_file *file,
int lock_type)
{
int ret;
if (lock_type == F_UNLCK) {
file->lock_type = F_UNLCK;
if (--file->log->dotlock_count > 0)
return 0;
ret = file_unlock_dotlock(file->filepath, &file->log->dotlock);
if (ret < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "file_unlock_dotlock()");
return -1;
}
if (ret == 0) {
mail_index_set_error(file->log->index,
"Dotlock was lost for transaction log file %s",
file->filepath);
return -1;
}
return 0;
}
if (file->log->dotlock_count > 0)
ret = 1;
else {
ret = file_lock_dotlock(file->filepath, NULL, FALSE,
LOG_DOTLOCK_TIMEOUT,
LOG_DOTLOCK_STALE_TIMEOUT,
LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT,
NULL, NULL, &file->log->dotlock);
}
if (ret > 0) {
file->log->dotlock_count++;
file->lock_type = F_WRLCK;
return 0;
}
if (ret < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"file_lock_dotlock()");
return -1;
}
mail_index_set_error(file->log->index,
"Timeout while waiting for release of "
"dotlock for transaction log file %s",
file->filepath);
file->log->index->index_lock_timeout = TRUE;
return -1;
}
static int
mail_transaction_log_file_lock(struct mail_transaction_log_file *file,
int lock_type)
{
int ret;
if (lock_type == file->lock_type)
return 0;
if (lock_type == F_UNLCK) {
i_assert(file->lock_type != F_UNLCK);
} else {
i_assert(file->lock_type == F_UNLCK);
}
if (file->log->index->fcntl_locks_disable)
return mail_transaction_log_file_dotlock(file, lock_type);
ret = file_wait_lock_full(file->fd, lock_type, DEFAULT_LOCK_TIMEOUT,
NULL, NULL);
if (ret > 0) {
file->lock_type = lock_type;
return 0;
}
if (ret < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"file_wait_lock()");
return -1;
}
mail_index_set_error(file->log->index,
"Timeout while waiting for release of "
"%s fcntl() lock for transaction log file %s",
lock_type == F_WRLCK ? "exclusive" : "shared",
file->filepath);
file->log->index->index_lock_timeout = TRUE;
return -1;
}
static void
mail_transaction_log_file_close(struct mail_transaction_log_file *file)
{
if (file->lock_type != F_UNLCK)
(void)mail_transaction_log_file_lock(file, F_UNLCK);
if (file->buffer != NULL)
buffer_free(file->buffer);
if (file->mmap_base != NULL) {
if (munmap(file->mmap_base, file->mmap_size) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"munmap()");
}
}
if (close(file->fd) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "close()");
}
i_free(file->filepath);
i_free(file);
}
static int
mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
struct stat *st)
{
int ret;
uint32_t old_size = file->hdr.used_size;
if (file->lock_type != F_UNLCK)
ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);
else {
if (mail_transaction_log_file_lock(file, F_RDLCK) < 0)
return -1;
/* we have to fstat() again since it may have changed after
locking. */
if (fstat(file->fd, st) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"fstat()");
(void)mail_transaction_log_file_lock(file, F_UNLCK);
return -1;
}
ret = pread_full(file->fd, &file->hdr,
sizeof(file->hdr), 0);
(void)mail_transaction_log_file_lock(file, F_UNLCK);
}
file->last_mtime = st->st_mtime;
if (ret < 0) {
// FIXME: handle ESTALE
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"pread_full()");
return -1;
}
if (ret == 0) {
mail_transaction_log_file_set_corrupted(file,
"unexpected end of file while reading header");
return 0;
}
if (file->hdr.indexid == 0) {
/* corrupted */
mail_index_set_error(file->log->index,
"Transaction log file %s: marked corrupted",
file->filepath);
return 0;
}
if (file->hdr.indexid != file->log->index->indexid) {
if (file->log->index->fd == -1) {
/* creating index file, silently rebuild
transaction log as well */
return 0;
}
/* index file was probably just rebuilt and we don't know
about it yet */
mail_index_set_error(file->log->index,
"Transaction log file %s: invalid indexid (%u != %u)",
file->filepath, file->hdr.indexid,
file->log->index->indexid);
return 0;
}
if (file->hdr.used_size > st->st_size) {
mail_transaction_log_file_set_corrupted(file,
"used_size (%u) > file size (%"PRIuUOFF_T")",
file->hdr.used_size, (uoff_t)st->st_size);
return 0;
}
if (file->hdr.used_size < old_size) {
mail_transaction_log_file_set_corrupted(file,
"used_size (%u) < old_size (%u)",
file->hdr.used_size, old_size);
return 0;
}
return 1;
}
static int
mail_transaction_log_file_create2(struct mail_transaction_log *log,
const char *path, int fd,
dev_t dev, ino_t ino)
{
struct mail_index *index = log->index;
struct mail_transaction_log_header hdr;
struct stat st;
unsigned int lock_id;
int fd2, ret;
/* log creation is locked now - see if someone already created it */
fd2 = open(path, O_RDWR);
if (fd2 != -1) {
if ((ret = fstat(fd2, &st)) < 0) {
mail_index_file_set_syscall_error(index, path,
"fstat()");
} else if (st.st_dev == dev && st.st_ino == ino) {
/* same file, still broken */
} else {
(void)file_dotlock_delete(path, LOG_NEW_DOTLOCK_SUFFIX,
fd);
return fd2;
}
(void)close(fd2);
fd2 = -1;
if (ret < 0)
return -1;
} else if (errno != ENOENT) {
mail_index_file_set_syscall_error(index, path, "open()");
return -1;
}
memset(&hdr, 0, sizeof(hdr));
hdr.indexid = index->indexid;
hdr.used_size = sizeof(hdr);
if (index->fd != -1) {
if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
return -1;
hdr.prev_file_seq = index->hdr->log_file_seq;
hdr.prev_file_offset = index->hdr->log_file_offset;
}
hdr.file_seq = index->hdr->log_file_seq+1;
if (index->fd != -1)
mail_index_unlock(index, lock_id);
if (log->head != NULL && hdr.file_seq <= log->head->hdr.file_seq) {
/* make sure the sequence grows */
hdr.file_seq = log->head->hdr.file_seq+1;
}
if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
mail_index_file_set_syscall_error(index, path,
"write_full()");
return -1;
}
fd2 = dup(fd);
if (fd2 < 0) {
mail_index_file_set_syscall_error(index, path, "dup()");
return -1;
}
if (file_dotlock_replace(path, LOG_NEW_DOTLOCK_SUFFIX, fd, FALSE) <= 0)
return -1;
/* success */
return fd2;
}
static int
mail_transaction_log_file_create(struct mail_transaction_log *log,
const char *path, dev_t dev, ino_t ino)
{
int fd, fd2;
/* With dotlocking we might already have path.lock created, so this
filename has to be different. */
fd = file_dotlock_open(path, NULL, LOG_NEW_DOTLOCK_SUFFIX,
LOG_DOTLOCK_TIMEOUT,
LOG_DOTLOCK_STALE_TIMEOUT,
LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT, NULL, NULL);
if (fd == -1) {
mail_index_file_set_syscall_error(log->index, path,
"file_dotlock_open()");
return -1;
}
fd2 = mail_transaction_log_file_create2(log, path, fd, dev, ino);
if (fd2 < 0) {
(void)file_dotlock_delete(path, LOG_NEW_DOTLOCK_SUFFIX, fd);
return -1;
}
return fd2;
}
static struct mail_transaction_log_file *
mail_transaction_log_file_fd_open(struct mail_transaction_log *log,
const char *path, int fd)
{
struct mail_transaction_log_file **p;
struct mail_transaction_log_file *file;
struct stat st;
int ret;
if (fstat(fd, &st) < 0) {
mail_index_file_set_syscall_error(log->index, path, "fstat()");
(void)close(fd);
return NULL;
}
file = i_new(struct mail_transaction_log_file, 1);
file->refcount = 1;
file->log = log;
file->filepath = i_strdup(path);
file->fd = fd;
file->lock_type = F_UNLCK;
file->st_dev = st.st_dev;
file->st_ino = st.st_ino;
ret = mail_transaction_log_file_read_hdr(file, &st);
if (ret == 0) {
/* corrupted header */
fd = mail_transaction_log_file_create(log, path,
st.st_dev, st.st_ino);
if (fd == -1)
ret = -1;
else if (fstat(fd, &st) < 0) {
mail_index_file_set_syscall_error(log->index, path,
"fstat()");
(void)close(fd);
fd = -1;
ret = -1;
}
if (fd != -1) {
(void)close(file->fd);
file->fd = fd;
file->st_dev = st.st_dev;
file->st_ino = st.st_ino;
memset(&file->hdr, 0, sizeof(file->hdr));
ret = mail_transaction_log_file_read_hdr(file, &st);
}
}
if (ret <= 0) {
mail_transaction_log_file_close(file);
return NULL;
}
for (p = &log->tail; *p != NULL; p = &(*p)->next) {
if ((*p)->hdr.file_seq >= file->hdr.file_seq) {
/* log replaced with file having same sequence as
previous one. shouldn't happen unless previous
log file was corrupted.. */
break;
}
}
*p = file;
return file;
}
static struct mail_transaction_log_file *
mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
const char *path)
{
int fd;
fd = open(path, O_RDWR);
if (fd == -1) {
if (errno != ENOENT) {
mail_index_file_set_syscall_error(log->index, path,
"open()");
return NULL;
}
fd = mail_transaction_log_file_create(log, path, 0, 0);
if (fd == -1)
return NULL;
}
return mail_transaction_log_file_fd_open(log, path, fd);
}
void mail_transaction_logs_clean(struct mail_transaction_log *log)
{
struct mail_transaction_log_file **p, *next;
for (p = &log->tail; *p != NULL; ) {
if ((*p)->refcount != 0)
p = &(*p)->next;
else {
next = (*p)->next;
mail_transaction_log_file_close(*p);
*p = next;
}
}
if (log->tail == NULL)
log->head = NULL;
}
static int mail_transaction_log_rotate(struct mail_transaction_log *log,
int lock_type)
{
struct mail_transaction_log_file *file;
struct stat st;
int fd;
if (fstat(log->head->fd, &st) < 0) {
mail_index_file_set_syscall_error(log->index,
log->head->filepath,
"fstat()");
return -1;
}
fd = mail_transaction_log_file_create(log, log->head->filepath,
st.st_dev, st.st_ino);
if (fd == -1)
return -1;
file = mail_transaction_log_file_fd_open(log, log->head->filepath, fd);
if (file == NULL)
return -1;
if (lock_type != F_UNLCK) {
if (mail_transaction_log_file_lock(file, lock_type) < 0) {
file->refcount--;
mail_transaction_logs_clean(log);
return -1;
}
}
i_assert(file->lock_type == lock_type);
if (--log->head->refcount == 0)
mail_transaction_logs_clean(log);
else
(void)mail_transaction_log_file_lock(log->head, F_UNLCK);
i_assert(log->head != file);
log->head = file;
return 0;
}
static int mail_transaction_log_recreate(struct mail_transaction_log *log)
{
unsigned int lock_id;
int ret;
if (mail_index_lock_shared(log->index, TRUE, &lock_id) < 0)
return -1;
ret = mail_transaction_log_rotate(log, F_UNLCK);
mail_index_unlock(log->index, lock_id);
return ret;
}
static int mail_transaction_log_refresh(struct mail_transaction_log *log)
{
struct mail_transaction_log_file *file;
struct stat st;
const char *path;
int ret;
path = t_strconcat(log->index->filepath,
MAIL_TRANSACTION_LOG_PREFIX, NULL);
if (stat(path, &st) < 0) {
mail_index_file_set_syscall_error(log->index, path, "stat()");
if (errno == ENOENT && log->head->lock_type == F_WRLCK) {
/* lost? */
return mail_transaction_log_recreate(log);
}
return -1;
}
if (log->head != NULL &&
log->head->st_ino == st.st_ino &&
log->head->st_dev == st.st_dev) {
/* same file */
ret = mail_transaction_log_file_read_hdr(log->head, &st);
if (ret == 0 && log->head->lock_type == F_WRLCK) {
/* corrupted, recreate */
return mail_transaction_log_recreate(log);
}
return ret <= 0 ? -1 : 0;
}
file = mail_transaction_log_file_open_or_create(log, path);
if (file == NULL)
return -1;
i_assert(file->lock_type == F_UNLCK);
if (log->head != NULL) {
if (--log->head->refcount == 0)
mail_transaction_logs_clean(log);
}
log->head = file;
return 0;
}
int mail_transaction_log_file_find(struct mail_transaction_log *log,
uint32_t file_seq,
struct mail_transaction_log_file **file_r)
{
struct mail_transaction_log_file *file;
if (file_seq > log->head->hdr.file_seq) {
if (mail_transaction_log_refresh(log) < 0)
return -1;
}
for (file = log->tail; file != NULL; file = file->next) {
if (file->hdr.file_seq == file_seq) {
*file_r = file;
return 1;
}
}
return 0;
}
static int
mail_transaction_log_file_read(struct mail_transaction_log_file *file,
uoff_t offset)
{
void *data;
size_t size;
int ret;
i_assert(file->mmap_base == NULL);
i_assert(offset <= file->hdr.used_size);
if (file->buffer != NULL && file->buffer_offset > offset) {
/* we have to insert missing data to beginning of buffer */
size = file->buffer_offset - offset;
buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
file->buffer_offset = offset;
data = buffer_get_space_unsafe(file->buffer, 0, size);
ret = pread_full(file->fd, data, size, offset);
if (ret < 0 && errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
ret = 0;
}
if (ret <= 0)
return ret;
}
if (file->buffer == NULL) {
size = file->hdr.used_size - offset;
file->buffer = buffer_create_dynamic(default_pool,
size, (size_t)-1);
file->buffer_offset = offset;
size = 0;
} else {
size = buffer_get_used_size(file->buffer);
if (file->buffer_offset + size >= file->hdr.used_size) {
/* caller should have checked this.. */
return 1;
}
}
offset = file->buffer_offset + size;
size = file->hdr.used_size - file->buffer_offset - size;
if (size == 0)
return 1;
data = buffer_append_space_unsafe(file->buffer, size);
ret = pread_full(file->fd, data, size, offset);
if (ret < 0 && errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
ret = 0;
}
return ret;
}
int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
uoff_t start_offset, uoff_t end_offset)
{
struct mail_index *index = file->log->index;
size_t size;
struct stat st;
int ret, use_mmap;
i_assert(start_offset <= end_offset);
if (file->hdr.indexid == 0) {
/* corrupted */
return 0;
}
/* with mmap_no_write we could alternatively just write to log with
msync() rather than pwrite(). that'd cause slightly more disk I/O,
so rather use more memory. */
use_mmap = !index->mmap_disable && !index->mmap_no_write;
if (file->buffer != NULL && file->buffer_offset <= start_offset) {
/* see if we already have it */
size = buffer_get_used_size(file->buffer);
if (file->buffer_offset + size >= end_offset)
return 1;
}
if (fstat(file->fd, &st) < 0) {
mail_index_file_set_syscall_error(index, file->filepath,
"fstat()");
return -1;
}
if (st.st_size == file->hdr.used_size &&
file->buffer_offset <= start_offset && end_offset == (uoff_t)-1) {
/* we've seen the whole file.. do we have all of it mapped? */
size = file->buffer == NULL ? 0 :
buffer_get_used_size(file->buffer);
if (file->buffer_offset + size == file->hdr.used_size)
return 1;
}
if (file->buffer != NULL &&
(file->mmap_base != NULL || use_mmap)) {
buffer_free(file->buffer);
file->buffer = NULL;
}
if (file->mmap_base != NULL) {
if (munmap(file->mmap_base, file->mmap_size) < 0) {
mail_index_file_set_syscall_error(index, file->filepath,
"munmap()");
}
file->mmap_base = NULL;
}
if (mail_transaction_log_file_read_hdr(file, &st) <= 0)
return -1;
if (end_offset == (uoff_t)-1)
end_offset = file->hdr.used_size;
if (start_offset < sizeof(file->hdr)) {
mail_transaction_log_file_set_corrupted(file,
"offset (%"PRIuUOFF_T") < header size (%"PRIuSIZE_T")",
start_offset, sizeof(file->hdr));
return -1;
}
if (end_offset > file->hdr.used_size) {
mail_transaction_log_file_set_corrupted(file,
"offset (%"PRIuUOFF_T") > used_size (%u)",
end_offset, file->hdr.used_size);
return -1;
}
if (!use_mmap) {
ret = mail_transaction_log_file_read(file, start_offset);
if (ret <= 0) {
if (ret < 0) {
mail_index_file_set_syscall_error(index,
file->filepath, "pread_full()");
} else {
mail_transaction_log_file_set_corrupted(file,
"Unexpected EOF");
}
/* make sure we don't leave ourself in
inconsistent state */
if (file->buffer != NULL) {
buffer_free(file->buffer);
file->buffer = NULL;
}
}
return ret;
}
file->mmap_size = file->hdr.used_size;
file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ,
MAP_SHARED, file->fd, 0);
if (file->mmap_base == MAP_FAILED) {
file->mmap_base = NULL;
mail_index_file_set_syscall_error(index, file->filepath,
"mmap()");
return -1;
}
file->buffer = buffer_create_const_data(default_pool, file->mmap_base,
file->mmap_size);
file->buffer_offset = 0;
return 1;
}
static int mail_transaction_log_lock_head(struct mail_transaction_log *log)
{
struct mail_transaction_log_file *file;
int ret = 0;
/* we want to get the head file locked. this is a bit racy,
since by the time we have it locked a new log file may have been
created.
creating new log file requires locking the head file, so if we
can lock it and don't see another file, we can be sure no-one is
creating a new log at the moment */
for (;;) {
file = log->head;
if (mail_transaction_log_file_lock(file, F_WRLCK) < 0)
return -1;
file->refcount++;
ret = mail_transaction_log_refresh(log);
if (--file->refcount == 0) {
mail_transaction_logs_clean(log);
file = NULL;
}
if (ret == 0 && log->head == file) {
/* success */
break;
}
i_assert(log->head->lock_type == F_UNLCK);
if (file != NULL) {
if (mail_transaction_log_file_lock(file, F_UNLCK) < 0)
return -1;
}
if (ret < 0)
break;
/* try again */
}
return ret;
}
static void
mail_transaction_log_append_fix(struct mail_index_transaction *t,
const struct mail_transaction_header *hdr,
const void *data)
{
const struct mail_transaction_append_header *append_hdr = data;
const struct mail_index_record *old, *old_end;
struct mail_index_record *appends, *end, *rec, *dest;
uint32_t record_size = t->append_record_size;
size_t size;
int deleted = FALSE;
if (t->appends == NULL)
return;
appends = buffer_get_modifyable_data(t->appends, &size);
end = PTR_OFFSET(appends, size);
if (appends == end)
return;
/* we'll just check that none of the appends are already in
transaction log. this could happen if we crashed before we had
a chance to update index file */
old = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
old_end = CONST_PTR_OFFSET(data, hdr->size);
while (old != old_end) {
/* appends are sorted */
for (rec = appends; rec != end; ) {
if (rec->uid >= old->uid) {
if (rec->uid == old->uid) {
rec->uid = 0;
deleted = TRUE;
}
break;
}
rec = PTR_OFFSET(rec, record_size);
}
old = CONST_PTR_OFFSET(old, append_hdr->record_size);
}
if (deleted) {
/* compress deleted appends away */
for (rec = dest = appends; rec != end; ) {
if (rec->uid != 0)
dest++;
else if (rec != dest)
*rec = *dest;
rec = PTR_OFFSET(rec, record_size);
}
buffer_set_used_size(t->appends,
(char *)dest - (char *)appends);
}
}
static int mail_transaction_log_scan_pending(struct mail_transaction_log *log,
struct mail_index_transaction *t)
{
struct mail_transaction_log_view *sync_view;
const struct mail_transaction_header *hdr;
const void *data;
uint32_t max_cache_file_seq = 0;
int ret;
sync_view = mail_transaction_log_view_open(log);
ret = mail_transaction_log_view_set(sync_view, t->view->log_file_seq,
t->view->log_file_offset,
log->head->hdr.file_seq,
log->head->hdr.used_size,
MAIL_TRANSACTION_TYPE_MASK);
while ((ret = mail_transaction_log_view_next(sync_view,
&hdr, &data, NULL)) == 1) {
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_APPEND:
mail_transaction_log_append_fix(t, hdr, data);
break;
case MAIL_TRANSACTION_CACHE_RESET: {
const struct mail_transaction_cache_reset *reset = data;
max_cache_file_seq = reset->new_file_seq;
break;
}
}
}
/* make sure we're not writing cache_offsets to old cache file */
if (t->new_cache_file_seq == 0 && max_cache_file_seq != 0 &&
max_cache_file_seq != t->last_cache_file_seq) {
buffer_free(t->cache_updates);
t->cache_updates = NULL;
}
mail_transaction_log_view_close(sync_view);
return ret;
}
static int log_append_buffer(struct mail_transaction_log_file *file,
const buffer_t *buf, const buffer_t *hdr_buf,
enum mail_transaction_type type, int external)
{
struct mail_transaction_header hdr;
const void *data, *hdr_data;
size_t size, hdr_size;
i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
data = buffer_get_data(buf, &size);
if (size == 0)
return 0;
if (hdr_buf != NULL)
hdr_data = buffer_get_data(hdr_buf, &hdr_size);
else {
hdr_data = NULL;
hdr_size = 0;
}
hdr.type = type;
if (type == MAIL_TRANSACTION_EXPUNGE)
hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
if (external)
hdr.type |= MAIL_TRANSACTION_EXTERNAL;
hdr.size = size + hdr_size;
if (pwrite_full(file->fd, &hdr, sizeof(hdr), file->hdr.used_size) < 0)
return -1;
file->hdr.used_size += sizeof(hdr);
if (hdr_size > 0) {
if (pwrite_full(file->fd, hdr_data, hdr_size,
file->hdr.used_size) < 0)
return -1;
file->hdr.used_size += hdr_size;
}
if (pwrite_full(file->fd, data, size, file->hdr.used_size) < 0)
return -1;
file->hdr.used_size += size;
return 0;
}
static const buffer_t *get_cache_reset_buf(struct mail_index_transaction *t)
{
struct mail_transaction_cache_reset u;
buffer_t *buf;
memset(&u, 0, sizeof(u));
u.new_file_seq = t->new_cache_file_seq;
buf = buffer_create_static(pool_datastack_create(), sizeof(u));
buffer_append(buf, &u, sizeof(u));
return buf;
}
static const buffer_t *
log_get_hdr_update_buffer(struct mail_index_transaction *t)
{
buffer_t *buf;
struct mail_transaction_header_update u;
uint16_t offset;
int state = 0;
memset(&u, 0, sizeof(u));
buf = buffer_create_dynamic(pool_datastack_create(), 256, (size_t)-1);
for (offset = 0; offset <= sizeof(t->hdr_change); offset++) {
if (offset < sizeof(t->hdr_change) && t->hdr_mask[offset]) {
if (state == 0) {
u.offset = offset;
state++;
}
} else {
if (state > 0) {
u.size = offset - u.offset;
buffer_append(buf, &u, sizeof(uint16_t)*2);
buffer_append(buf, t->hdr_change + u.offset,
u.size);
state = 0;
}
}
}
return buf;
}
int mail_transaction_log_append(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r)
{
struct mail_transaction_extra_rec_header extra_rec_hdr;
struct mail_index_view *view = t->view;
struct mail_index *index;
struct mail_transaction_log *log;
struct mail_transaction_log_file *file;
size_t offset;
uoff_t append_offset;
buffer_t *hdr_buf;
unsigned int i;
int ret;
index = mail_index_view_get_index(view);
log = index->log;
if (t->updates == NULL && t->new_cache_file_seq == 0 &&
t->cache_updates == NULL && t->expunges == NULL &&
t->appends == NULL && !t->hdr_changed) {
/* nothing to append */
*log_file_seq_r = 0;
*log_file_offset_r = 0;
return 0;
}
if (log->index->log_locked) {
i_assert(view->external);
} else {
if (mail_transaction_log_lock_head(log) < 0)
return -1;
}
if (log->head->hdr.used_size > MAIL_TRANSACTION_LOG_ROTATE_SIZE &&
log->head->last_mtime <
ioloop_time - MAIL_TRANSACTION_LOG_ROTATE_MIN_TIME) {
/* we might want to rotate, but check first that head file
sequence matches the one in index header, ie. we have
everything synced in index. */
unsigned int lock_id;
uint32_t seq;
if (mail_index_lock_shared(log->index, TRUE, &lock_id) == 0) {
seq = index->hdr->log_file_seq;
mail_index_unlock(log->index, lock_id);
} else {
seq = 0;
}
if (log->head->hdr.file_seq == seq) {
if (mail_transaction_log_rotate(log, F_WRLCK) < 0) {
/* that didn't work. well, try to continue
anyway */
}
}
}
file = log->head;
append_offset = file->hdr.used_size;
if (t->appends != NULL ||
(t->cache_updates != NULL && t->new_cache_file_seq == 0)) {
if (mail_transaction_log_scan_pending(log, t) < 0) {
if (!log->index->log_locked) {
(void)mail_transaction_log_file_lock(file,
F_UNLCK);
}
return -1;
}
}
ret = 0;
if (t->appends != NULL) {
struct mail_transaction_append_header hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.record_size = t->append_record_size;
hdr_buf = buffer_create_data(pool_datastack_create(),
&hdr, sizeof(hdr));
buffer_set_used_size(hdr_buf, sizeof(hdr));
ret = log_append_buffer(file, t->appends, hdr_buf,
MAIL_TRANSACTION_APPEND,
view->external);
}
if (t->updates != NULL && ret == 0) {
ret = log_append_buffer(file, t->updates, NULL,
MAIL_TRANSACTION_FLAG_UPDATE,
view->external);
}
if (t->new_cache_file_seq != 0) {
ret = log_append_buffer(file, get_cache_reset_buf(t), NULL,
MAIL_TRANSACTION_CACHE_RESET,
view->external);
}
if (t->cache_updates != NULL && ret == 0) {
ret = log_append_buffer(file, t->cache_updates, NULL,
MAIL_TRANSACTION_CACHE_UPDATE,
view->external);
}
hdr_buf = buffer_create_data(pool_datastack_create(),
&extra_rec_hdr, sizeof(extra_rec_hdr));
buffer_set_used_size(hdr_buf, sizeof(extra_rec_hdr));
for (i = 0; i < view->index->extra_records_count; i++) {
if (t->extra_rec_updates[i] == NULL || ret != 0)
continue;
/* FIXME: do data_id mapping conversion */
extra_rec_hdr.idx = i;
ret = log_append_buffer(file, t->extra_rec_updates[i], hdr_buf,
MAIL_TRANSACTION_EXTRA_REC_UPDATE,
view->external);
}
if (t->expunges != NULL && ret == 0) {
ret = log_append_buffer(file, t->expunges, NULL,
MAIL_TRANSACTION_EXPUNGE,
view->external);
}
if (t->hdr_changed && ret == 0) {
ret = log_append_buffer(file, log_get_hdr_update_buffer(t),
NULL, MAIL_TRANSACTION_HEADER_UPDATE,
view->external);
}
if (ret == 0) {
/* rewrite used_size */
offset = offsetof(struct mail_transaction_log_header,
used_size);
ret = pwrite_full(file->fd, &file->hdr.used_size,
sizeof(file->hdr.used_size), offset);
}
if (ret == 0 && (t->updates != NULL || t->appends != NULL) &&
t->hide_transaction) {
mail_index_view_add_synced_transaction(view, file->hdr.file_seq,
append_offset);
}
if (ret < 0) {
file->hdr.used_size = append_offset;
mail_index_file_set_syscall_error(log->index, file->filepath,
"pwrite()");
} else if (fsync(file->fd) < 0) {
/* we don't know how much of it got written,
it may be corrupted now.. */
mail_index_file_set_syscall_error(log->index, file->filepath,
"fsync()");
ret = -1;
}
*log_file_seq_r = file->hdr.file_seq;
*log_file_offset_r = file->hdr.used_size;
if (!log->index->log_locked)
(void)mail_transaction_log_file_lock(file, F_UNLCK);
return ret;
}
int mail_transaction_log_sync_lock(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r)
{
i_assert(!log->index->log_locked);
if (mail_transaction_log_lock_head(log) < 0)
return -1;
log->index->log_locked = TRUE;
*file_seq_r = log->head->hdr.file_seq;
*file_offset_r = log->head->hdr.used_size;
return 0;
}
void mail_transaction_log_sync_unlock(struct mail_transaction_log *log)
{
i_assert(log->index->log_locked);
log->index->log_locked = FALSE;
(void)mail_transaction_log_file_lock(log->head, F_UNLCK);
}
void mail_transaction_log_get_head(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r)
{
i_assert(log->index->log_locked);
*file_seq_r = log->head->hdr.file_seq;
*file_offset_r = log->head->hdr.used_size;
}