mail-transaction-log.c revision 74d48e8b8c4ac1ec7a3b2a9fc4b7b5176aec01e8
294N/A/* Copyright (C) 2003-2004 Timo Sirainen */
294N/A
787N/A#include "lib.h"
789N/A#include "ioloop.h"
789N/A#include "buffer.h"
789N/A#include "file-lock.h"
1336N/A#include "file-dotlock.h"
789N/A#include "read-full.h"
789N/A#include "write-full.h"
873N/A#include "mmap-util.h"
789N/A#include "mail-index-private.h"
877N/A#include "mail-index-view-private.h"
789N/A#include "mail-transaction-log-private.h"
294N/A#include "mail-transaction-util.h"
873N/A#include "mail-index-transaction-private.h"
873N/A
789N/A#include <stddef.h>
789N/A#include <sys/stat.h>
789N/A
873N/A/* this lock should never exist for a long time.. */
789N/A#define LOG_DOTLOCK_TIMEOUT 30
789N/A#define LOG_DOTLOCK_STALE_TIMEOUT 0
789N/A#define LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT 300
789N/A
789N/Astruct mail_transaction_add_ctx {
789N/A struct mail_transaction_log *log;
789N/A struct mail_index_view *view;
789N/A
789N/A buffer_t *appends, *expunges;
1336N/A buffer_t *flag_updates, *cache_updates;
1336N/A};
789N/A
873N/Astatic struct mail_transaction_log_file *
1336N/Amail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
1336N/A const char *path);
1336N/Astatic int mail_transaction_log_rotate(struct mail_transaction_log *log,
1336N/A int lock_type);
1336N/A
1336N/Astatic int
1336N/Amail_transaction_log_file_lock(struct mail_transaction_log_file *file,
1336N/A int lock_type);
1336N/Astatic int mail_transaction_log_lock_head(struct mail_transaction_log *log);
1336N/A
1336N/Avoid
1336N/Amail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
1336N/A const char *fmt, ...)
1336N/A{
1336N/A va_list va;
1336N/A
1336N/A file->hdr.indexid = 0;
1336N/A if (pwrite_full(file->fd, &file->hdr.indexid,
873N/A sizeof(file->hdr.indexid), 0) < 0) {
873N/A mail_index_file_set_syscall_error(file->log->index,
873N/A file->filepath, "pwrite()");
789N/A }
787N/A
294N/A va_start(va, fmt);
1336N/A t_push();
1336N/A mail_index_set_error(file->log->index,
1336N/A "Corrupted transaction log file %s: %s",
1336N/A file->filepath, t_strdup_vprintf(fmt, va));
1337N/A t_pop();
294N/A va_end(va);
294N/A}
294N/A
1336N/A#define INDEX_HAS_MISSING_LOGS(index, file) \
1336N/A !(((file)->hdr.file_seq == (index)->hdr->log_file_seq && \
1337N/A (index)->hdr->log_file_offset >= \
1337N/A sizeof(struct mail_transaction_log_header)) || \
1337N/A ((file)->hdr.prev_file_seq == (index)->hdr->log_file_seq && \
1337N/A (file)->hdr.prev_file_offset == (index)->hdr->log_file_offset))
1337N/A
1337N/Astatic int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)
1337N/A{
1337N/A struct mail_index *index = log->index;
1337N/A struct mail_transaction_log_file *file;
1336N/A unsigned int lock_id;
1337N/A int ret;
1337N/A
1337N/A if (mail_transaction_log_lock_head(log) < 0)
294N/A return -1;
294N/A
294N/A file = log->head;
294N/A file->refcount++;
789N/A
294N/A ret = mail_index_lock_shared(index, TRUE, &lock_id);
1109N/A if (ret == 0) {
1109N/A ret = mail_index_map(index, FALSE);
1337N/A if (ret <= 0)
1337N/A ret = -1;
1109N/A else if (INDEX_HAS_MISSING_LOGS(index, file)) {
1337N/A /* broken - fix it by creating a new log file */
1337N/A ret = mail_transaction_log_rotate(log, F_UNLCK);
1109N/A }
787N/A }
869N/A
869N/A if (--file->refcount == 0)
869N/A mail_transaction_logs_clean(log);
869N/A else
789N/A (void)mail_transaction_log_file_lock(file, F_UNLCK);
789N/A return ret;
789N/A}
789N/A
789N/Astruct mail_transaction_log *
789N/Amail_transaction_log_open_or_create(struct mail_index *index)
294N/A{
789N/A struct mail_transaction_log *log;
294N/A const char *path;
294N/A
789N/A log = i_new(struct mail_transaction_log, 1);
294N/A log->index = index;
294N/A
294N/A path = t_strconcat(log->index->filepath,
294N/A MAIL_TRANSACTION_LOG_PREFIX, NULL);
294N/A log->head = mail_transaction_log_file_open_or_create(log, path);
294N/A if (log->head == NULL) {
294N/A i_free(log);
789N/A return NULL;
789N/A }
789N/A
787N/A if (index->fd != -1 &&
789N/A INDEX_HAS_MISSING_LOGS(index, log->head)) {
294N/A /* head log file isn't same as head index file -
869N/A shouldn't happen except in race conditions. lock them and
868N/A check again - FIXME: missing error handling. */
789N/A (void)mail_transaction_log_check_file_seq(log);
789N/A }
869N/A return log;
869N/A}
869N/A
869N/Avoid mail_transaction_log_close(struct mail_transaction_log *log)
789N/A{
789N/A mail_transaction_log_views_close(log);
869N/A
949N/A log->head->refcount--;
789N/A mail_transaction_logs_clean(log);
789N/A
979N/A log->index->log = NULL;
1340N/A i_free(log);
1340N/A}
789N/A
787N/Astatic int
787N/Amail_transaction_log_file_dotlock(struct mail_transaction_log_file *file,
294N/A int lock_type)
294N/A{
789N/A int ret;
294N/A
294N/A if (lock_type == F_UNLCK) {
294N/A file->lock_type = F_UNLCK;
294N/A if (--file->log->dotlock_count > 0)
789N/A return 0;
294N/A
294N/A ret = file_unlock_dotlock(file->filepath, &file->log->dotlock);
294N/A if (ret < 0) {
789N/A mail_index_file_set_syscall_error(file->log->index,
294N/A file->filepath, "file_unlock_dotlock()");
789N/A return -1;
294N/A }
789N/A
789N/A if (ret == 0) {
911N/A mail_index_set_error(file->log->index,
911N/A "Dotlock was lost for transaction log file %s",
911N/A file->filepath);
915N/A return -1;
911N/A }
911N/A return 0;
911N/A }
869N/A
869N/A if (file->log->dotlock_count > 0)
869N/A ret = 1;
869N/A else {
789N/A ret = file_lock_dotlock(file->filepath, NULL, FALSE,
857N/A LOG_DOTLOCK_TIMEOUT,
789N/A LOG_DOTLOCK_STALE_TIMEOUT,
789N/A LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT,
789N/A NULL, NULL, &file->log->dotlock);
789N/A }
789N/A if (ret > 0) {
294N/A file->log->dotlock_count++;
294N/A file->lock_type = F_WRLCK;
789N/A return 0;
294N/A }
294N/A if (ret < 0) {
789N/A mail_index_file_set_syscall_error(file->log->index,
294N/A file->filepath,
294N/A "file_lock_dotlock()");
294N/A return -1;
789N/A }
294N/A
1339N/A mail_index_set_error(file->log->index,
1339N/A "Timeout while waiting for release of "
1340N/A "dotlock for transaction log file %s",
1340N/A file->filepath);
877N/A file->log->index->index_lock_timeout = TRUE;
869N/A return -1;
869N/A}
868N/A
869N/Astatic int
869N/Amail_transaction_log_file_lock(struct mail_transaction_log_file *file,
869N/A int lock_type)
868N/A{
789N/A int ret;
941N/A
1086N/A if (lock_type == F_UNLCK) {
911N/A i_assert(file->lock_type != F_UNLCK);
941N/A } else {
911N/A i_assert(file->lock_type == F_UNLCK);
294N/A }
869N/A
869N/A if (file->log->index->fcntl_locks_disable)
941N/A return mail_transaction_log_file_dotlock(file, lock_type);
1086N/A
872N/A ret = file_wait_lock_full(file->fd, lock_type, DEFAULT_LOCK_TIMEOUT,
941N/A NULL, NULL);
872N/A if (ret > 0) {
869N/A file->lock_type = lock_type;
994N/A return 0;
1086N/A }
994N/A if (ret < 0) {
994N/A mail_index_file_set_syscall_error(file->log->index,
994N/A file->filepath,
994N/A "file_wait_lock()");
869N/A return -1;
1109N/A }
1109N/A
1109N/A mail_index_set_error(file->log->index,
1109N/A "Timeout while waiting for release of "
1109N/A "%s fcntl() lock for transaction log file %s",
1109N/A lock_type == F_WRLCK ? "exclusive" : "shared",
868N/A file->filepath);
869N/A file->log->index->index_lock_timeout = TRUE;
869N/A return -1;
869N/A}
869N/A
868N/Astatic void
870N/Amail_transaction_log_file_close(struct mail_transaction_log_file *file)
870N/A{
870N/A if (file->lock_type != F_UNLCK)
870N/A (void)mail_transaction_log_file_lock(file, F_UNLCK);
870N/A
869N/A if (file->buffer != NULL)
869N/A buffer_free(file->buffer);
869N/A
869N/A if (file->mmap_base != NULL) {
1336N/A if (munmap(file->mmap_base, file->mmap_size) < 0) {
1336N/A mail_index_file_set_syscall_error(file->log->index,
1336N/A file->filepath,
1336N/A "munmap()");
1336N/A }
1336N/A }
1336N/A
1336N/A if (close(file->fd) < 0) {
1336N/A mail_index_file_set_syscall_error(file->log->index,
1336N/A file->filepath, "close()");
1336N/A }
1336N/A
1336N/A i_free(file->filepath);
869N/A i_free(file);
869N/A}
912N/A
912N/Astatic int
868N/Amail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
789N/A struct stat *st)
789N/A{
789N/A int ret;
789N/A uint32_t old_size = file->hdr.used_size;
789N/A
877N/A if (file->lock_type != F_UNLCK)
877N/A ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);
877N/A else {
877N/A if (mail_transaction_log_file_lock(file, F_RDLCK) < 0)
877N/A return -1;
787N/A
294N/A /* we have to fstat() again since it may have changed after
789N/A locking. */
789N/A if (fstat(file->fd, st) < 0) {
789N/A mail_index_file_set_syscall_error(file->log->index,
789N/A file->filepath,
789N/A "fstat()");
789N/A (void)mail_transaction_log_file_lock(file, F_UNLCK);
877N/A return -1;
877N/A }
877N/A
877N/A ret = pread_full(file->fd, &file->hdr,
877N/A sizeof(file->hdr), 0);
789N/A (void)mail_transaction_log_file_lock(file, F_UNLCK);
294N/A }
869N/A
869N/A file->last_mtime = st->st_mtime;
1091N/A
869N/A if (ret < 0) {
869N/A // FIXME: handle ESTALE
869N/A mail_index_file_set_syscall_error(file->log->index,
869N/A file->filepath,
1086N/A "pread_full()");
1086N/A return -1;
1086N/A }
1086N/A if (ret == 0) {
294N/A mail_transaction_log_file_set_corrupted(file,
294N/A "unexpected end of file while reading header");
294N/A return 0;
294N/A }
294N/A if (file->hdr.indexid == 0) {
294N/A /* corrupted */
294N/A mail_index_set_error(file->log->index,
294N/A "Transaction log file %s: marked corrupted",
294N/A file->filepath);
294N/A return 0;
869N/A }
294N/A if (file->hdr.indexid != file->log->index->indexid) {
294N/A if (file->log->index->fd == -1) {
869N/A /* creating index file, silently rebuild
873N/A transaction log as well */
1086N/A return 0;
1091N/A }
873N/A
873N/A /* index file was probably just rebuilt and we don't know
872N/A about it yet */
1155N/A mail_index_set_error(file->log->index,
1114N/A "Transaction log file %s: invalid indexid (%u != %u)",
1114N/A file->filepath, file->hdr.indexid,
1114N/A file->log->index->indexid);
1114N/A return 0;
789N/A }
789N/A if (file->hdr.used_size > st->st_size) {
789N/A mail_transaction_log_file_set_corrupted(file,
789N/A "used_size (%u) > file size (%"PRIuUOFF_T")",
789N/A file->hdr.used_size, (uoff_t)st->st_size);
789N/A return 0;
789N/A }
789N/A if (file->hdr.used_size < old_size) {
789N/A mail_transaction_log_file_set_corrupted(file,
789N/A "used_size (%u) < old_size (%u)",
789N/A file->hdr.used_size, old_size);
789N/A return 0;
789N/A }
789N/A
789N/A return 1;
789N/A}
789N/A
789N/Astatic int
789N/Amail_transaction_log_file_create(struct mail_transaction_log *log,
789N/A const char *path, dev_t dev, ino_t ino)
789N/A{
789N/A#define LOG_NEW_DOTLOCK_SUFFIX ".newlock"
789N/A struct mail_index *index = log->index;
789N/A struct mail_transaction_log_header hdr;
789N/A struct stat st;
789N/A unsigned int lock_id;
789N/A int fd, fd2, ret;
789N/A
789N/A /* With dotlocking we might already have path.lock created, so this
789N/A filename has to be different. */
789N/A fd = file_dotlock_open(path, NULL, LOG_NEW_DOTLOCK_SUFFIX,
789N/A LOG_DOTLOCK_TIMEOUT,
789N/A LOG_DOTLOCK_STALE_TIMEOUT,
789N/A LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT, NULL, NULL);
1340N/A if (fd == -1) {
1340N/A mail_index_file_set_syscall_error(index, path,
1340N/A "file_dotlock_open()");
1340N/A return -1;
1340N/A }
789N/A
789N/A /* log creation is locked now - see if someone already created it */
789N/A fd2 = open(path, O_RDWR);
789N/A if (fd2 != -1) {
789N/A if ((ret = fstat(fd2, &st)) < 0) {
789N/A mail_index_file_set_syscall_error(index, path,
789N/A "fstat()");
789N/A } else if (st.st_dev == dev && st.st_ino == ino) {
789N/A /* same file, still broken */
789N/A } else {
789N/A (void)file_dotlock_delete(path, LOG_NEW_DOTLOCK_SUFFIX,
789N/A fd);
789N/A return fd2;
789N/A }
789N/A
789N/A (void)close(fd2);
294N/A fd2 = -1;
1155N/A
1155N/A if (ret < 0)
1155N/A return -1;
1339N/A } else if (errno != ENOENT) {
1155N/A mail_index_file_set_syscall_error(index, path, "open()");
1155N/A return -1;
1155N/A }
1155N/A
1155N/A memset(&hdr, 0, sizeof(hdr));
1155N/A hdr.indexid = index->indexid;
1155N/A hdr.used_size = sizeof(hdr);
1155N/A
1155N/A if (index->fd != -1) {
1155N/A if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
1155N/A return -1;
789N/A hdr.prev_file_seq = index->hdr->log_file_seq;
789N/A hdr.prev_file_offset = index->hdr->log_file_offset;
1155N/A }
1155N/A hdr.file_seq = index->hdr->log_file_seq+1;
789N/A
789N/A if (index->fd != -1)
789N/A mail_index_unlock(index, lock_id);
789N/A
789N/A if (log->head != NULL && hdr.file_seq <= log->head->hdr.file_seq) {
789N/A /* make sure the sequence grows */
1155N/A hdr.file_seq = log->head->hdr.file_seq+1;
1155N/A }
1155N/A
1155N/A if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
1155N/A mail_index_file_set_syscall_error(index, path,
1155N/A "write_full()");
1155N/A (void)file_dotlock_delete(path, LOG_NEW_DOTLOCK_SUFFIX, fd);
1155N/A return -1;
1155N/A }
1155N/A
789N/A fd2 = dup(fd);
789N/A if (fd2 < 0) {
789N/A mail_index_file_set_syscall_error(index, path, "dup()");
789N/A (void)file_dotlock_delete(path, LOG_NEW_DOTLOCK_SUFFIX, fd);
1091N/A return -1;
789N/A }
789N/A
789N/A if (file_dotlock_replace(path, LOG_NEW_DOTLOCK_SUFFIX, fd, FALSE) <= 0)
1091N/A return -1;
1091N/A
1091N/A /* success */
1091N/A return fd2;
1091N/A}
1091N/A
1091N/Astatic struct mail_transaction_log_file *
1091N/Amail_transaction_log_file_fd_open(struct mail_transaction_log *log,
1091N/A const char *path, int fd)
1091N/A{
1091N/A struct mail_transaction_log_file **p;
1091N/A struct mail_transaction_log_file *file;
1091N/A struct stat st;
1091N/A int ret;
1091N/A
1091N/A if (fstat(fd, &st) < 0) {
1091N/A mail_index_file_set_syscall_error(log->index, path, "fstat()");
1091N/A return NULL;
1091N/A }
789N/A
1091N/A file = i_new(struct mail_transaction_log_file, 1);
789N/A file->refcount = 1;
789N/A file->log = log;
789N/A file->filepath = i_strdup(path);
789N/A file->fd = fd;
789N/A file->lock_type = F_UNLCK;
789N/A file->st_dev = st.st_dev;
789N/A file->st_ino = st.st_ino;
789N/A
789N/A ret = mail_transaction_log_file_read_hdr(file, &st);
789N/A if (ret == 0) {
789N/A /* corrupted header */
789N/A fd = mail_transaction_log_file_create(log, path,
789N/A st.st_dev, st.st_ino);
789N/A if (fd == -1)
789N/A ret = -1;
789N/A else if (fstat(fd, &st) < 0) {
789N/A mail_index_file_set_syscall_error(log->index, path,
789N/A "fstat()");
789N/A (void)close(fd);
789N/A fd = -1;
789N/A ret = -1;
789N/A }
789N/A
789N/A if (fd != -1) {
789N/A (void)close(file->fd);
789N/A file->fd = fd;
787N/A
789N/A file->st_dev = st.st_dev;
789N/A file->st_ino = st.st_ino;
789N/A
789N/A memset(&file->hdr, 0, sizeof(file->hdr));
789N/A ret = mail_transaction_log_file_read_hdr(file, &st);
789N/A }
789N/A }
789N/A if (ret <= 0) {
789N/A mail_transaction_log_file_close(file);
789N/A return NULL;
789N/A }
789N/A
789N/A for (p = &log->tail; *p != NULL; p = &(*p)->next) {
1341N/A if ((*p)->hdr.file_seq >= file->hdr.file_seq) {
1341N/A /* log replaced with file having same sequence as
1341N/A previous one. shouldn't happen unless previous
1341N/A log file was corrupted.. */
789N/A break;
789N/A }
789N/A }
789N/A *p = file;
789N/A
789N/A return file;
789N/A}
789N/A
789N/Astatic struct mail_transaction_log_file *
789N/Amail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
789N/A const char *path)
789N/A{
789N/A int fd;
789N/A
789N/A fd = open(path, O_RDWR);
789N/A if (fd == -1) {
789N/A if (errno != ENOENT) {
789N/A mail_index_file_set_syscall_error(log->index, path,
789N/A "open()");
789N/A return NULL;
789N/A }
789N/A
789N/A fd = mail_transaction_log_file_create(log, path, 0, 0);
789N/A if (fd == -1)
789N/A return NULL;
789N/A }
789N/A
789N/A return mail_transaction_log_file_fd_open(log, path, fd);
789N/A}
789N/A
789N/Avoid mail_transaction_logs_clean(struct mail_transaction_log *log)
789N/A{
1155N/A struct mail_transaction_log_file **p, *next;
1341N/A
789N/A for (p = &log->tail; *p != NULL; ) {
789N/A if ((*p)->refcount != 0)
789N/A p = &(*p)->next;
789N/A else {
789N/A next = (*p)->next;
789N/A mail_transaction_log_file_close(*p);
789N/A *p = next;
789N/A }
787N/A }
789N/A
789N/A if (log->tail == NULL)
789N/A log->head = NULL;
789N/A}
789N/A
789N/Astatic int mail_transaction_log_rotate(struct mail_transaction_log *log,
789N/A int lock_type)
789N/A{
789N/A struct mail_transaction_log_file *file;
789N/A struct stat st;
789N/A int fd;
789N/A
789N/A if (fstat(log->head->fd, &st) < 0) {
789N/A mail_index_file_set_syscall_error(log->index,
789N/A log->head->filepath,
1341N/A "fstat()");
789N/A return -1;
789N/A }
789N/A
789N/A fd = mail_transaction_log_file_create(log, log->head->filepath,
789N/A st.st_dev, st.st_ino);
789N/A if (fd == -1)
789N/A return -1;
789N/A
789N/A file = mail_transaction_log_file_fd_open(log, log->head->filepath, fd);
789N/A if (file == NULL)
789N/A return -1;
789N/A
789N/A if (lock_type != F_UNLCK) {
789N/A if (mail_transaction_log_file_lock(file, lock_type) < 0) {
789N/A file->refcount--;
789N/A mail_transaction_logs_clean(log);
789N/A return -1;
789N/A }
789N/A }
789N/A i_assert(file->lock_type == lock_type);
789N/A
789N/A if (--log->head->refcount == 0)
1339N/A mail_transaction_logs_clean(log);
1339N/A else
789N/A (void)mail_transaction_log_file_lock(log->head, F_UNLCK);
789N/A
789N/A i_assert(log->head != file);
789N/A log->head = file;
789N/A return 0;
789N/A}
789N/A
789N/Astatic int mail_transaction_log_recreate(struct mail_transaction_log *log)
789N/A{
789N/A unsigned int lock_id;
789N/A int ret;
1339N/A
1339N/A if (mail_index_lock_shared(log->index, TRUE, &lock_id) < 0)
789N/A return -1;
789N/A
789N/A ret = mail_transaction_log_rotate(log, F_UNLCK);
789N/A mail_index_unlock(log->index, lock_id);
787N/A return ret;
294N/A}
294N/A
294N/Astatic int mail_transaction_log_refresh(struct mail_transaction_log *log)
789N/A{
789N/A struct mail_transaction_log_file *file;
294N/A struct stat st;
789N/A const char *path;
789N/A int ret;
789N/A
789N/A path = t_strconcat(log->index->filepath,
1339N/A MAIL_TRANSACTION_LOG_PREFIX, NULL);
1339N/A if (stat(path, &st) < 0) {
789N/A mail_index_file_set_syscall_error(log->index, path, "stat()");
789N/A if (errno == ENOENT && log->head->lock_type == F_WRLCK) {
789N/A /* lost? */
789N/A return mail_transaction_log_recreate(log);
789N/A }
789N/A return -1;
789N/A }
789N/A
1338N/A if (log->head != NULL &&
1338N/A log->head->st_ino == st.st_ino &&
1338N/A log->head->st_dev == st.st_dev) {
789N/A /* same file */
789N/A ret = mail_transaction_log_file_read_hdr(log->head, &st);
789N/A if (ret == 0 && log->head->lock_type == F_WRLCK) {
789N/A /* corrupted, recreate */
789N/A return mail_transaction_log_recreate(log);
789N/A }
789N/A return ret <= 0 ? -1 : 0;
789N/A }
789N/A
1338N/A file = mail_transaction_log_file_open_or_create(log, path);
1338N/A if (file == NULL)
789N/A return -1;
789N/A
789N/A i_assert(file->lock_type == F_UNLCK);
789N/A
789N/A if (log->head != NULL) {
789N/A if (--log->head->refcount == 0)
789N/A mail_transaction_logs_clean(log);
789N/A }
789N/A
789N/A log->head = file;
789N/A return 0;
789N/A}
789N/A
789N/Aint mail_transaction_log_file_find(struct mail_transaction_log *log,
789N/A uint32_t file_seq,
789N/A struct mail_transaction_log_file **file_r)
789N/A{
789N/A struct mail_transaction_log_file *file;
789N/A
873N/A if (file_seq > log->head->hdr.file_seq) {
873N/A if (mail_transaction_log_refresh(log) < 0)
873N/A return -1;
873N/A }
873N/A
873N/A for (file = log->tail; file != NULL; file = file->next) {
873N/A if (file->hdr.file_seq == file_seq) {
1086N/A *file_r = file;
1086N/A return 1;
1086N/A }
1339N/A }
1339N/A
1086N/A return 0;
789N/A}
789N/A
789N/Astatic int
789N/Amail_transaction_log_file_read(struct mail_transaction_log_file *file,
789N/A uoff_t offset)
789N/A{
1086N/A void *data;
789N/A size_t size;
789N/A int ret;
789N/A
857N/A i_assert(file->mmap_base == NULL);
857N/A i_assert(offset <= file->hdr.used_size);
1086N/A
857N/A if (file->buffer != NULL && file->buffer_offset > offset) {
857N/A /* we have to insert missing data to beginning of buffer */
857N/A size = file->buffer_offset - offset;
857N/A buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
1339N/A file->buffer_offset = offset;
1339N/A
857N/A data = buffer_get_space_unsafe(file->buffer, 0, size);
857N/A ret = pread_full(file->fd, data, size, offset);
857N/A if (ret < 0 && errno == ESTALE) {
857N/A /* log file was deleted in NFS server, fail silently */
857N/A ret = 0;
1339N/A }
1339N/A if (ret <= 0)
857N/A return ret;
857N/A }
869N/A
868N/A if (file->buffer == NULL) {
869N/A size = file->hdr.used_size - offset;
869N/A file->buffer = buffer_create_dynamic(default_pool,
869N/A size, (size_t)-1);
911N/A file->buffer_offset = offset;
911N/A size = 0;
869N/A } else {
917N/A size = buffer_get_used_size(file->buffer);
1339N/A if (file->buffer_offset + size >= file->hdr.used_size) {
1339N/A /* caller should have checked this.. */
1339N/A return 1;
1339N/A }
1339N/A }
1339N/A offset = file->buffer_offset + size;
1339N/A
1339N/A size = file->hdr.used_size - file->buffer_offset - size;
1339N/A if (size == 0)
1339N/A return 1;
1339N/A
869N/A data = buffer_append_space_unsafe(file->buffer, size);
868N/A
1086N/A ret = pread_full(file->fd, data, size, offset);
1086N/A if (ret < 0 && errno == ESTALE) {
1086N/A /* log file was deleted in NFS server, fail silently */
1339N/A ret = 0;
1339N/A }
1339N/A return ret;
1339N/A}
1339N/A
1339N/Aint mail_transaction_log_file_map(struct mail_transaction_log_file *file,
1339N/A uoff_t start_offset, uoff_t end_offset)
1086N/A{
789N/A struct mail_index *index = file->log->index;
294N/A size_t size;
294N/A struct stat st;
294N/A int ret, use_mmap;
789N/A
869N/A i_assert(start_offset <= end_offset);
877N/A
789N/A if (file->hdr.indexid == 0) {
869N/A /* corrupted */
869N/A return 0;
869N/A }
994N/A
868N/A /* with mmap_no_write we could alternatively just write to log with
1114N/A msync() rather than pwrite(). that'd cause slightly more disk I/O,
789N/A so rather use more memory. */
869N/A use_mmap = !index->mmap_disable && !index->mmap_no_write;
911N/A
872N/A if (file->buffer != NULL && file->buffer_offset <= start_offset) {
1336N/A /* see if we already have it */
789N/A size = buffer_get_used_size(file->buffer);
789N/A if (file->buffer_offset + size >= end_offset)
294N/A return 1;
294N/A }
294N/A
294N/A if (fstat(file->fd, &st) < 0) {
294N/A mail_index_file_set_syscall_error(index, file->filepath,
294N/A "fstat()");
294N/A return -1;
294N/A }
294N/A
294N/A if (st.st_size == file->hdr.used_size &&
789N/A file->buffer_offset <= start_offset && end_offset == (uoff_t)-1) {
868N/A /* we've seen the whole file.. do we have all of it mapped? */
294N/A size = file->buffer == NULL ? 0 :
789N/A buffer_get_used_size(file->buffer);
789N/A if (file->buffer_offset + size == file->hdr.used_size)
294N/A return 1;
294N/A }
294N/A
294N/A if (file->buffer != NULL &&
294N/A (file->mmap_base != NULL || use_mmap)) {
869N/A buffer_free(file->buffer);
294N/A file->buffer = NULL;
294N/A }
294N/A if (file->mmap_base != NULL) {
787N/A if (munmap(file->mmap_base, file->mmap_size) < 0) {
787N/A mail_index_file_set_syscall_error(index, file->filepath,
789N/A "munmap()");
789N/A }
789N/A file->mmap_base = NULL;
789N/A }
789N/A
789N/A if (mail_transaction_log_file_read_hdr(file, &st) <= 0)
787N/A return -1;
789N/A
789N/A if (end_offset == (uoff_t)-1)
789N/A end_offset = file->hdr.used_size;
787N/A
789N/A if (start_offset < sizeof(file->hdr)) {
789N/A mail_transaction_log_file_set_corrupted(file,
1339N/A "offset (%"PRIuUOFF_T") < header size (%"PRIuSIZE_T")",
1339N/A start_offset, sizeof(file->hdr));
789N/A return -1;
787N/A }
789N/A if (end_offset > file->hdr.used_size) {
789N/A mail_transaction_log_file_set_corrupted(file,
789N/A "offset (%"PRIuUOFF_T") > used_size (%u)",
789N/A end_offset, file->hdr.used_size);
789N/A return -1;
789N/A }
789N/A
789N/A if (!use_mmap) {
789N/A ret = mail_transaction_log_file_read(file, start_offset);
787N/A if (ret <= 0) {
789N/A if (ret < 0) {
1308N/A mail_index_file_set_syscall_error(index,
789N/A file->filepath, "pread_full()");
789N/A } else {
789N/A mail_transaction_log_file_set_corrupted(file,
789N/A "Unexpected EOF");
787N/A }
789N/A
1339N/A /* make sure we don't leave ourself in
1339N/A inconsistent state */
1339N/A if (file->buffer != NULL) {
1339N/A buffer_free(file->buffer);
789N/A file->buffer = NULL;
789N/A }
789N/A }
789N/A return ret;
787N/A }
787N/A
1086N/A file->mmap_size = file->hdr.used_size;
1086N/A file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ,
1086N/A MAP_SHARED, file->fd, 0);
1086N/A if (file->mmap_base == MAP_FAILED) {
1091N/A file->mmap_base = NULL;
1278N/A mail_index_file_set_syscall_error(index, file->filepath,
1086N/A "mmap()");
1091N/A return -1;
1086N/A }
1339N/A file->buffer = buffer_create_const_data(default_pool, file->mmap_base,
1339N/A file->mmap_size);
1086N/A file->buffer_offset = 0;
1086N/A return 1;
1086N/A}
1086N/A
1086N/Astatic int mail_transaction_log_lock_head(struct mail_transaction_log *log)
294N/A{
294N/A struct mail_transaction_log_file *file;
294N/A int ret = 0;
294N/A
868N/A /* we want to get the head file locked. this is a bit racy,
294N/A since by the time we have it locked a new log file may have been
294N/A created.
294N/A
294N/A creating new log file requires locking the head file, so if we
789N/A can lock it and don't see another file, we can be sure no-one is
789N/A creating a new log at the moment */
789N/A
789N/A for (;;) {
294N/A file = log->head;
787N/A if (mail_transaction_log_file_lock(file, F_WRLCK) < 0)
787N/A return -1;
787N/A
1340N/A file->refcount++;
1086N/A ret = mail_transaction_log_refresh(log);
1086N/A if (--file->refcount == 0) {
1086N/A mail_transaction_logs_clean(log);
1086N/A file = NULL;
1086N/A }
1340N/A
294N/A if (ret == 0 && log->head == file) {
868N/A /* success */
868N/A break;
789N/A }
789N/A
789N/A i_assert(log->head->lock_type == F_UNLCK);
294N/A if (file != NULL) {
294N/A if (mail_transaction_log_file_lock(file, F_UNLCK) < 0)
868N/A return -1;
868N/A }
789N/A
789N/A if (ret < 0)
789N/A break;
789N/A
789N/A /* try again */
294N/A }
868N/A
869N/A return ret;
869N/A}
869N/A
868N/Astatic int mail_transaction_log_fix_appends(struct mail_transaction_log *log,
868N/A struct mail_index_transaction *t)
868N/A{
868N/A struct mail_transaction_log_view *sync_view;
868N/A const struct mail_index_record *old, *old_end;
868N/A struct mail_index_record *appends, *end, *rec, *dest;
868N/A const struct mail_transaction_header *hdr;
294N/A const void *data;
294N/A size_t size;
868N/A uint32_t record_size;
789N/A int ret, deleted = FALSE;
294N/A
294N/A if (t->appends == NULL)
294N/A return 0;
789N/A
294N/A record_size = log->index->record_size;
appends = buffer_get_modifyable_data(t->appends, &size);
end = PTR_OFFSET(appends, size);
if (appends == end)
return 0;
/* we'll just check that none of the appends are already in
transaction log. this could happen if we crashed before we had
a chance to update index file */
sync_view = mail_transaction_log_view_open(log);
ret = mail_transaction_log_view_set(sync_view, t->view->log_file_seq,
t->view->log_file_offset,
log->head->hdr.file_seq,
log->head->hdr.used_size,
MAIL_TRANSACTION_TYPE_MASK);
while ((ret = mail_transaction_log_view_next(sync_view,
&hdr, &data, NULL)) == 1) {
if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) !=
MAIL_TRANSACTION_APPEND)
continue;
old = data;
old_end = CONST_PTR_OFFSET(old, hdr->size);
while (old != old_end) {
/* appends are sorted */
for (rec = appends; rec != end; ) {
if (rec->uid >= old->uid) {
if (rec->uid == old->uid) {
rec->uid = 0;
deleted = TRUE;
}
break;
}
rec = PTR_OFFSET(rec, record_size);
}
old = CONST_PTR_OFFSET(old, record_size);
}
}
if (deleted) {
/* compress deleted appends away */
for (rec = dest = appends; rec != end; ) {
if (rec->uid != 0)
dest++;
else if (rec != dest)
*rec = *dest;
rec = PTR_OFFSET(rec, record_size);
}
buffer_set_used_size(t->appends,
(char *)dest - (char *)appends);
}
mail_transaction_log_view_close(sync_view);
return ret;
}
static int log_append_buffer(struct mail_transaction_log_file *file,
const buffer_t *buf, const buffer_t *hdr_buf,
enum mail_transaction_type type, int external)
{
struct mail_transaction_header hdr;
const void *data, *hdr_data;
size_t size, hdr_size;
i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
data = buffer_get_data(buf, &size);
if (size == 0)
return 0;
if (hdr_buf != NULL)
hdr_data = buffer_get_data(hdr_buf, &hdr_size);
else {
hdr_data = NULL;
hdr_size = 0;
}
hdr.type = type;
if (type == MAIL_TRANSACTION_EXPUNGE)
hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
if (external)
hdr.type |= MAIL_TRANSACTION_EXTERNAL;
hdr.size = size + hdr_size;
if (pwrite_full(file->fd, &hdr, sizeof(hdr), file->hdr.used_size) < 0)
return -1;
file->hdr.used_size += sizeof(hdr);
if (hdr_size > 0) {
if (pwrite_full(file->fd, hdr_data, hdr_size,
file->hdr.used_size) < 0)
return -1;
file->hdr.used_size += hdr_size;
}
if (pwrite_full(file->fd, data, size, file->hdr.used_size) < 0)
return -1;
file->hdr.used_size += size;
return 0;
}
static const buffer_t *get_cache_reset_buf(struct mail_index_transaction *t)
{
struct mail_transaction_cache_reset u;
buffer_t *buf;
memset(&u, 0, sizeof(u));
u.new_file_seq = t->new_cache_file_seq;
buf = buffer_create_static(pool_datastack_create(), sizeof(u));
buffer_append(buf, &u, sizeof(u));
return buf;
}
static const buffer_t *
log_get_hdr_update_buffer(struct mail_index_transaction *t)
{
buffer_t *buf;
struct mail_transaction_header_update u;
uint16_t offset;
int state = 0;
memset(&u, 0, sizeof(u));
buf = buffer_create_dynamic(pool_datastack_create(), 256, (size_t)-1);
for (offset = 0; offset <= sizeof(t->hdr_change); offset++) {
if (offset < sizeof(t->hdr_change) && t->hdr_mask[offset]) {
if (state == 0) {
u.offset = offset;
state++;
}
} else {
if (state > 0) {
u.size = offset - u.offset;
buffer_append(buf, &u, sizeof(uint16_t)*2);
buffer_append(buf, t->hdr_change + u.offset,
u.size);
state = 0;
}
}
}
return buf;
}
int mail_transaction_log_append(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r)
{
struct mail_transaction_extra_rec_header extra_rec_hdr;
struct mail_index_view *view = t->view;
struct mail_index *index;
struct mail_transaction_log *log;
struct mail_transaction_log_file *file;
size_t offset;
uoff_t append_offset;
buffer_t *hdr_buf;
unsigned int i;
int ret;
index = mail_index_view_get_index(view);
log = index->log;
if (t->updates == NULL && t->new_cache_file_seq == 0 &&
t->cache_updates == NULL && t->expunges == NULL &&
t->appends == NULL && !t->hdr_changed) {
/* nothing to append */
*log_file_seq_r = 0;
*log_file_offset_r = 0;
return 0;
}
if (log->index->log_locked) {
i_assert(view->external);
} else {
if (mail_transaction_log_lock_head(log) < 0)
return -1;
}
if (log->head->hdr.used_size > MAIL_TRANSACTION_LOG_ROTATE_SIZE &&
log->head->last_mtime <
ioloop_time - MAIL_TRANSACTION_LOG_ROTATE_MIN_TIME) {
/* we might want to rotate, but check first that head file
sequence matches the one in index header, ie. we have
everything synced in index. */
unsigned int lock_id;
uint32_t seq;
if (mail_index_lock_shared(log->index, TRUE, &lock_id) == 0) {
seq = index->hdr->log_file_seq;
mail_index_unlock(log->index, lock_id);
} else {
seq = 0;
}
if (log->head->hdr.file_seq == seq) {
if (mail_transaction_log_rotate(log, F_WRLCK) < 0) {
/* that didn't work. well, try to continue
anyway */
}
}
}
file = log->head;
append_offset = file->hdr.used_size;
if (mail_transaction_log_fix_appends(log, t) < 0) {
if (!log->index->log_locked)
(void)mail_transaction_log_file_lock(file, F_UNLCK);
return -1;
}
ret = 0;
if (t->appends != NULL) {
ret = log_append_buffer(file, t->appends, NULL,
MAIL_TRANSACTION_APPEND,
view->external);
}
if (t->updates != NULL && ret == 0) {
ret = log_append_buffer(file, t->updates, NULL,
MAIL_TRANSACTION_FLAG_UPDATE,
view->external);
}
if (t->new_cache_file_seq != 0) {
ret = log_append_buffer(file, get_cache_reset_buf(t), NULL,
MAIL_TRANSACTION_CACHE_RESET,
view->external);
}
if (t->cache_updates != NULL && ret == 0) {
ret = log_append_buffer(file, t->cache_updates, NULL,
MAIL_TRANSACTION_CACHE_UPDATE,
view->external);
}
hdr_buf = buffer_create_data(pool_datastack_create(),
&extra_rec_hdr, sizeof(extra_rec_hdr));
buffer_set_used_size(hdr_buf, sizeof(extra_rec_hdr));
for (i = 0; i < view->index->extra_records_count; i++) {
if (t->extra_rec_updates[i] == NULL || ret != 0)
continue;
/* FIXME: do data_id mapping conversion */
extra_rec_hdr.idx = i;
ret = log_append_buffer(file, t->extra_rec_updates[i], hdr_buf,
MAIL_TRANSACTION_EXTRA_REC_UPDATE,
view->external);
}
if (t->expunges != NULL && ret == 0) {
ret = log_append_buffer(file, t->expunges, NULL,
MAIL_TRANSACTION_EXPUNGE,
view->external);
}
if (t->hdr_changed && ret == 0) {
ret = log_append_buffer(file, log_get_hdr_update_buffer(t),
NULL, MAIL_TRANSACTION_HEADER_UPDATE,
view->external);
}
if (ret == 0) {
/* rewrite used_size */
offset = offsetof(struct mail_transaction_log_header,
used_size);
ret = pwrite_full(file->fd, &file->hdr.used_size,
sizeof(file->hdr.used_size), offset);
}
if (ret == 0 && (t->updates != NULL || t->appends != NULL) &&
t->hide_transaction) {
mail_index_view_add_synced_transaction(view, file->hdr.file_seq,
append_offset);
}
if (ret < 0) {
file->hdr.used_size = append_offset;
mail_index_file_set_syscall_error(log->index, file->filepath,
"pwrite()");
} else if (fsync(file->fd) < 0) {
/* we don't know how much of it got written,
it may be corrupted now.. */
mail_index_file_set_syscall_error(log->index, file->filepath,
"fsync()");
ret = -1;
}
*log_file_seq_r = file->hdr.file_seq;
*log_file_offset_r = file->hdr.used_size;
if (!log->index->log_locked)
(void)mail_transaction_log_file_lock(file, F_UNLCK);
return ret;
}
int mail_transaction_log_sync_lock(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r)
{
i_assert(!log->index->log_locked);
if (mail_transaction_log_lock_head(log) < 0)
return -1;
log->index->log_locked = TRUE;
*file_seq_r = log->head->hdr.file_seq;
*file_offset_r = log->head->hdr.used_size;
return 0;
}
void mail_transaction_log_sync_unlock(struct mail_transaction_log *log)
{
i_assert(log->index->log_locked);
log->index->log_locked = FALSE;
(void)mail_transaction_log_file_lock(log->head, F_UNLCK);
}
void mail_transaction_log_get_head(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r)
{
i_assert(log->index->log_locked);
*file_seq_r = log->head->hdr.file_seq;
*file_offset_r = log->head->hdr.used_size;
}