src/lib-index/mail-transaction-log.c

	mail-transaction-log.c revision d9fdacd5fb3e07997e5c389739d2054f0c8441d8
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "lib.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "ioloop.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "buffer.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "file-dotlock.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "nfs-workarounds.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "close-keep-errno.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "read-full.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "write-full.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "mmap-util.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "mail-index-private.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "mail-index-view-private.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "mail-transaction-log-private.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "mail-transaction-util.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "mail-index-transaction-private.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include <stddef.h>
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include <stdio.h>
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include <sys/stat.h>
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#define LOG_PREFETCH 1024
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen/* this lock should never exist for a long time.. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#define LOG_DOTLOCK_TIMEOUT 60
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#define LOG_DOTLOCK_STALE_TIMEOUT 60
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#define MAIL_TRANSACTION_LOG_SUFFIX ".log"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#define LOG_NEW_DOTLOCK_SUFFIX ".newlock"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic struct mail_transaction_log_file *
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                     const char *path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_file_read(struct mail_transaction_log_file *file,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                   uoff_t offset);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenvoid
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen                    const char *fmt, ...)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    va_list va;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    file->hdr.indexid = 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        if (pwrite_full(file->fd, &file->hdr.indexid,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                sizeof(file->hdr.indexid), 0) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            mail_index_file_set_syscall_error(file->log->index,
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen                file->filepath, "pwrite()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    va_start(va, fmt);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    t_push();
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    mail_index_set_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                 "Corrupted transaction log file %s: %s",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                 file->filepath, t_strdup_vprintf(fmt, va));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    t_pop();
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    va_end(va);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->log->index->log != NULL) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        /* this may have happened because of broken index.
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen           make sure it's ok. */
87b5c1fc03945708726c175d540fd307f7f78480Phil Carmody        (void)mail_index_fsck(file->log->index);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    int ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->log->dotlock_count > 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        ret = 1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    else {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        ret = file_dotlock_create(&file->log->dotlock_settings,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                      file->filepath, 0,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                      &file->log->dotlock);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (ret > 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->log->dotlock_count++;
d06e111700769927a64b46dc157ac1c122dff1a0Timo Sirainen        file->locked = TRUE;
d06e111700769927a64b46dc157ac1c122dff1a0Timo Sirainen        return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (ret < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_index_file_set_syscall_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                          file->filepath,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                          "file_dotlock_create()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    mail_index_set_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                 "Timeout while waiting for release of "
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                 "dotlock for transaction log file %s",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                 file->filepath);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    file->log->index->index_lock_timeout = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    int ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
4307c886579381dbb1897ea1388ae6978c96f560Timo Sirainen    if (--file->log->dotlock_count > 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    ret = file_dotlock_delete(&file->log->dotlock);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (ret < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_index_file_set_syscall_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            file->filepath, "file_dotlock_delete()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (ret == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_index_set_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            "Dotlock was lost for transaction log file %s",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            file->filepath);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_file_lock(struct mail_transaction_log_file *file)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    int ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->locked)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->locked = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return mail_transaction_log_file_dotlock(file);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    i_assert(file->file_lock == NULL);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                 F_WRLCK, MAIL_INDEX_LOCK_SECS,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                 &file->file_lock);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (ret > 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->locked = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (ret < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_index_file_set_syscall_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                          file->filepath,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                          "mail_index_wait_lock_fd()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    mail_index_set_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        "Timeout while waiting for lock for transaction log file %s",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->filepath);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    file->log->index->index_lock_timeout = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenvoid mail_transaction_log_file_unlock(struct mail_transaction_log_file *file)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (!file->locked)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    file->locked = FALSE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_transaction_log_file_undotlock(file);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    file_unlock(&file->file_lock);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#define INDEX_HAS_MISSING_LOGS(index, file) \
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    !(((file)->hdr.file_seq == (index)->hdr->log_file_seq && \
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen       (index)->hdr->log_file_int_offset >= (file)->hdr.hdr_size) || \
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen      ((file)->hdr.prev_file_seq == (index)->hdr->log_file_seq && \
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen       (file)->hdr.prev_file_offset == (index)->hdr->log_file_int_offset))
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    struct mail_index *index = log->index;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    struct mail_transaction_log_file *file;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    unsigned int lock_id;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    int ret;
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (mail_transaction_log_lock_head(log) < 0)
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen        return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    file = log->head;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    file->refcount++;
6b2f82c7ed4e646217a1ee29e99c83d95762a8d6Phil Carmody
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    ret = mail_index_lock_shared(index, TRUE, &lock_id);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen    if (ret == 0) {
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen        ret = mail_index_map(index, FALSE);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen        if (ret <= 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            ret = -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        else if (INDEX_HAS_MISSING_LOGS(index, file)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            /* broken - fix it by creating a new log file */
2282944443f78bd186809df235da1b7e801f0430Phil Carmody            ret = mail_transaction_log_rotate(log, FALSE);
2282944443f78bd186809df235da1b7e801f0430Phil Carmody        }
2282944443f78bd186809df235da1b7e801f0430Phil Carmody    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen    if (--file->refcount == 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_transaction_logs_clean(log);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    else
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_transaction_log_file_unlock(file);
6b2f82c7ed4e646217a1ee29e99c83d95762a8d6Phil Carmody    return ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainenstruct mail_transaction_log *
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_open_or_create(struct mail_index *index)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    struct mail_transaction_log *log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    const char *path;
2282944443f78bd186809df235da1b7e801f0430Phil Carmody
2282944443f78bd186809df235da1b7e801f0430Phil Carmody    log = i_new(struct mail_transaction_log, 1);
2282944443f78bd186809df235da1b7e801f0430Phil Carmody    log->index = index;
2282944443f78bd186809df235da1b7e801f0430Phil Carmody
2282944443f78bd186809df235da1b7e801f0430Phil Carmody    log->dotlock_settings.timeout = LOG_DOTLOCK_TIMEOUT;
2282944443f78bd186809df235da1b7e801f0430Phil Carmody    log->dotlock_settings.stale_timeout = LOG_DOTLOCK_STALE_TIMEOUT;
2282944443f78bd186809df235da1b7e801f0430Phil Carmody
2282944443f78bd186809df235da1b7e801f0430Phil Carmody    log->new_dotlock_settings = log->dotlock_settings;
2282944443f78bd186809df235da1b7e801f0430Phil Carmody    log->new_dotlock_settings.lock_suffix = LOG_NEW_DOTLOCK_SUFFIX;
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    path = t_strconcat(log->index->filepath,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen               MAIL_TRANSACTION_LOG_SUFFIX, NULL);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    log->head = mail_transaction_log_file_open_or_create(log, path);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    if (log->head == NULL) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        /* fallback to in-memory indexes */
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        if (mail_index_move_to_memory(index) < 0) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody            mail_transaction_log_close(&log);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody            return NULL;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        }
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        log->head = mail_transaction_log_file_open_or_create(log, path);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        i_assert(log->head != NULL);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    }
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    log->head->refcount++;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    if (index->fd != -1 &&
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        INDEX_HAS_MISSING_LOGS(index, log->head)) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        /* head log file isn't same as head index file -
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody           shouldn't happen except in race conditions.
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody           lock them and check again */
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        if (mail_transaction_log_check_file_seq(log) < 0) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody            mail_transaction_log_close(&log);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody            return NULL;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        }
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    }
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    return log;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody}
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmodyvoid mail_transaction_log_close(struct mail_transaction_log **_log)
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody{
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    struct mail_transaction_log *log = *_log;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    mail_transaction_log_views_close(log);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (log->head != NULL)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        log->head->refcount--;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    mail_transaction_logs_clean(log);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    i_assert(log->files == NULL);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen    *_log = NULL;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen    log->index->log = NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    i_free(log);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic void
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainenmail_transaction_log_file_free(struct mail_transaction_log_file *file)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    struct mail_transaction_log_file **p;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    int old_errno = errno;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    mail_transaction_log_file_unlock(file);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        if (*p == file) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            *p = file->next;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            break;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file == file->log->head)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->log->head = NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->buffer != NULL)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        buffer_free(file->buffer);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->mmap_base != NULL) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        if (munmap(file->mmap_base, file->mmap_size) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            mail_index_file_set_syscall_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                              file->filepath,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                              "munmap()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->fd != -1) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        if (close(file->fd) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            mail_index_file_set_syscall_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                              file->filepath,
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody                              "close()");
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody        }
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    i_free(file->filepath);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        i_free(file);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        errno = old_errno;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenint mail_transaction_log_move_to_memory(struct mail_transaction_log *log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    struct mail_transaction_log_file *file = log->head;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file == NULL || MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    /* read the whole file to memory. we might currently be appending
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen       data into it, so we want to read it up to end of file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->buffer_offset = 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->buffer != NULL) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        buffer_free(file->buffer);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->buffer = NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->mmap_base != NULL) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        if (munmap(file->mmap_base, file->mmap_size) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            mail_index_file_set_syscall_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                              file->filepath,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                              "munmap()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->mmap_base = NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (mail_transaction_log_file_read(file, 0) <= 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    /* after we've read the file into memory, make it into in-memory
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen       log file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (close(file->fd) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_index_file_set_syscall_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                          file->filepath, "close()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    file->fd = -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmodystatic int
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmodymail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                   int head, bool ignore_estale)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        struct mail_transaction_log_file *f;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    int ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody    i_assert(!MAIL_INDEX_IS_IN_MEMORY(file->log->index));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (ret < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                if (errno != ESTALE || !ignore_estale) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                        mail_index_file_set_syscall_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                                                          file->filepath,
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen                                                          "pread_full()");
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen                }
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen        return -1;
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen    }
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen    if (ret == 0) {
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen        mail_transaction_log_file_set_corrupted(file,
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen            "unexpected end of file while reading header");
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen        return 0;
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen    }
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen    if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen        /* incompatible version - fix silently */
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen        return 0;
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen    }
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody    if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_transaction_log_file_set_corrupted(file,
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody            "Header size too small");
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody        return 0;
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen    }
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody    if (file->hdr.hdr_size < sizeof(file->hdr)) {
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody        /* @UNSAFE: smaller than we expected - zero out the fields we
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody           shouldn't have filled */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen               sizeof(file->hdr) - file->hdr.hdr_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->hdr.indexid == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        /* corrupted */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_index_set_error(file->log->index,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            "Transaction log file %s: marked corrupted",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            file->filepath);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (file->hdr.indexid != file->log->index->indexid) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        if (file->log->index->fd != -1) {
87b5c1fc03945708726c175d540fd307f7f78480Phil Carmody            /* index file was probably just rebuilt and we don't
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen               know about it yet */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            mail_transaction_log_file_set_corrupted(file,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                "invalid indexid (%u != %u)",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                file->hdr.indexid, file->log->index->indexid);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        /* creating index file. since transaction log is created
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen           first, use the indexid in it to create the main index
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen           to avoid races. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        file->log->index->indexid = file->hdr.indexid;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    /* make sure we already don't have a file with the same sequence
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen       opened. it shouldn't happen unless the old log file was
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen       corrupted.
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen       If we're opening head log file, make sure the sequence is larger
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen       than any existing one. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (head) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        for (f = file->log->files; f != NULL; f = f->next) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            if (f->hdr.file_seq >= file->hdr.file_seq) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                mail_transaction_log_file_set_corrupted(file,
3785910c303507db5f629684e6dde2cc7f83668eTimo Sirainen                    "invalid new transaction log sequence "
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                    "(%u >= %u)",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                    f->hdr.file_seq, file->hdr.file_seq);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    } else {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        for (f = file->log->files; f != NULL; f = f->next) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            if (f->hdr.file_seq == file->hdr.file_seq) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                mail_transaction_log_file_set_corrupted(file,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                    "old transaction log already opened "
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                    "(%u == %u)",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                    f->hdr.file_seq, file->hdr.file_seq);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    return 1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
d868a04630bd7bfe9c1543a7c3f68703b3e276e4Timo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_init_hdr(struct mail_transaction_log *log,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                  struct mail_transaction_log_header *hdr)
46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen{
46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen    struct mail_index *index = log->index;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    unsigned int lock_id;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    memset(hdr, 0, sizeof(*hdr));
d868a04630bd7bfe9c1543a7c3f68703b3e276e4Timo Sirainen    hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;
d868a04630bd7bfe9c1543a7c3f68703b3e276e4Timo Sirainen    hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    hdr->hdr_size = sizeof(struct mail_transaction_log_header);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    hdr->indexid = log->index->indexid;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    hdr->create_stamp = ioloop_time;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (index->fd != -1) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        /* not creating index - make sure we have latest header */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        if (mail_index_map(index, FALSE) <= 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            mail_index_unlock(index, lock_id);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    hdr->prev_file_seq = index->hdr->log_file_seq;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    hdr->prev_file_offset = index->hdr->log_file_int_offset;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    hdr->file_seq = index->hdr->log_file_seq+1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (index->fd != -1)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_index_unlock(index, lock_id);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (log->head != NULL && hdr->file_seq <= log->head->hdr.file_seq) {
573424407a2d3c1453638a643583a7cf10c129e1Phil Carmody        /* make sure the sequence grows */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        hdr->file_seq = log->head->hdr.file_seq+1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenmail_transaction_log_file_create2(struct mail_transaction_log *log,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                  const char *path, int new_fd,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                  struct dotlock **dotlock,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                  dev_t dev, ino_t ino, uoff_t file_size)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    struct mail_index *index = log->index;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    struct mail_transaction_log_header hdr;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    struct stat st;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    const char *path2;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    int old_fd, ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    bool found;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    /* log creation is locked now - see if someone already created it */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    old_fd = nfs_safe_open(path, O_RDWR);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    if (old_fd != -1) {
3281669db44d09a087a203201248abbc81b3cc1aTimo Sirainen        if ((ret = fstat(old_fd, &st)) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                        mail_index_file_set_syscall_error(index, path,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                                                          "fstat()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        } else if (st.st_ino == ino && CMP_DEV_T(st.st_dev, dev) &&
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen               (uoff_t)st.st_size == file_size) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            /* same file, still broken */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        } else {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                        /* file changed, use the new file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen                        (void)file_dotlock_delete(dotlock);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen            return old_fd;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        (void)close(old_fd);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        old_fd = -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
9afa8aff5bda7c24cce6cc19b8232d2a228b4109Timo Sirainen                if (ret < 0) {
46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen                        /* fstat() failure, return after closing fd.. */
46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen                        return -1;
9afa8aff5bda7c24cce6cc19b8232d2a228b4109Timo Sirainen                }
9afa8aff5bda7c24cce6cc19b8232d2a228b4109Timo Sirainen        found = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    } else if (errno != ENOENT) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        mail_index_file_set_syscall_error(index, path, "open()");
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen    } else {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen        found = FALSE;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    }
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    if (mail_transaction_log_init_hdr(log, &hdr) < 0)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        return -1;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    if (write_full(new_fd, &hdr, sizeof(hdr)) < 0) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        mail_index_file_set_syscall_error(index, path,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody                          "write_full()");
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        return -1;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    }
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    /* keep two log files */
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    if (found) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        path2 = t_strconcat(path, ".2", NULL);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        if (rename(path, path2) < 0) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody                        mail_index_set_error(index,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody                            "rename(%s, %s) failed: %m", path, path2);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody            /* ignore the error. we don't care that much about the
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody               second log file and we're going to overwrite this
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody               first one. */
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        }
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    }
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody    if (file_dotlock_replace(dotlock,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody                 DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) <= 0)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody        return -1;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody    /* success */
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody    return new_fd;
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody}
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmodystatic int
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmodymail_transaction_log_file_create(struct mail_transaction_log *log,
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody                 const char *path,
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody                 dev_t dev, ino_t ino, uoff_t file_size)
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody{
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody    struct dotlock *dotlock;
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody    struct stat st;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    mode_t old_mask;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    int fd;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    i_assert(!MAIL_INDEX_IS_IN_MEMORY(log->index));
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    if (stat(log->index->dir, &st) < 0) {
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        if (ENOTFOUND(errno)) {
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody            /* the whole index directory was deleted, which means
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody               the mailbox was deleted by another process.
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody               fail silently. */
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody            mail_index_mark_corrupted(log->index);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody            return -1;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        }
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        mail_index_file_set_syscall_error(log->index, log->index->dir,
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody                          "stat()");
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        return -1;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    }
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    /* With dotlocking we might already have path.lock created, so this
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody       filename has to be different. */
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    old_mask = umask(log->index->mode ^ 0666);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    fd = file_dotlock_open(&log->new_dotlock_settings, path, 0, &dotlock);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    umask(old_mask);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    if (fd == -1) {
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        mail_index_file_set_syscall_error(log->index, path,
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody                          "file_dotlock_open()");
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        return -1;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    }
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    if (log->index->gid != (gid_t)-1 &&
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        fchown(fd, (uid_t)-1, log->index->gid) < 0) {
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        mail_index_file_set_syscall_error(log->index, path, "fchown()");
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        (void)file_dotlock_delete(&dotlock);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        return -1;
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    }
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        /* either fd gets used or the dotlock gets deleted and returned fd
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody           is for the existing file */
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        fd = mail_transaction_log_file_create2(log, path, fd, &dotlock,
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody                                               dev, ino, file_size);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody    if (fd < 0) {
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        if (dotlock != NULL)
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody            (void)file_dotlock_delete(&dotlock);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody        return -1;
    }
    return fd;
}

static void
mail_transaction_log_file_add_to_head(struct mail_transaction_log_file *file)
{
    struct mail_transaction_log *log = file->log;
    struct mail_transaction_log_file **p;

    if (log->index->map != NULL &&
        file->hdr.file_seq == log->index->map->hdr.log_file_seq &&
        log->index->map->hdr.log_file_int_offset != 0) {
        /* we can get a valid log offset from index file. initialize
           sync_offset from it so we don't have to read the whole log
           file from beginning. */
        file->sync_offset = log->index->map->hdr.log_file_int_offset;
    } else {
        file->sync_offset = file->hdr.hdr_size;
    }

    /* append to end of list. */
    for (p = &log->files; *p != NULL; p = &(*p)->next)
        i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
    *p = file;
}

static void
mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
{
    struct mail_transaction_log *log = file->log;
    struct mail_transaction_log_file **p;

    file->sync_offset = file->hdr.hdr_size;

    /* insert it to correct position */
    for (p = &log->files; *p != NULL; p = &(*p)->next) {
        i_assert((*p)->hdr.file_seq != file->hdr.file_seq);
        if ((*p)->hdr.file_seq > file->hdr.file_seq)
            break;
    }

    file->next = *p;
    *p = file;
}

static int
mail_transaction_log_file_fd_open(struct mail_transaction_log *log,
                                  struct mail_transaction_log_file **file_r,
                  const char *path, int fd, bool head,
                  bool ignore_estale)
{
        struct mail_transaction_log_file *file;
    struct stat st;
    int ret;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(log->index));

    *file_r = NULL;

    if (fstat(fd, &st) < 0) {
                if (errno != ESTALE || !ignore_estale) {
                        mail_index_file_set_syscall_error(log->index, path,
                                                          "fstat()");
                }
        close_keep_errno(fd);
        return -1;
    }

    file = i_new(struct mail_transaction_log_file, 1);
    file->log = log;
    file->filepath = i_strdup(path);
    file->fd = fd;
    file->st_dev = st.st_dev;
    file->st_ino = st.st_ino;
    file->last_mtime = st.st_mtime;
    file->last_size = st.st_size;

    ret = mail_transaction_log_file_read_hdr(file, head, ignore_estale);
    if (ret < 0) {
        mail_transaction_log_file_free(file);
        return -1;
    }

    *file_r = file;
    return ret;
}


static struct mail_transaction_log_file *
mail_transaction_log_file_fd_open_or_create(struct mail_transaction_log *log,
                                            const char *path, int fd,
                                            bool *retry_r, bool try_retry)
{
    struct mail_transaction_log_file *file;
    struct stat st;
    int ret;

        *retry_r = FALSE;

    ret = mail_transaction_log_file_fd_open(log, &file, path, fd, TRUE,
                        !try_retry);
        if (ret < 0) {
                *retry_r = errno == ESTALE && try_retry;
                return NULL;
        }

    if (ret == 0) {
        /* corrupted header */
        fd = mail_transaction_log_file_create(log, path, file->st_dev,
                              file->st_ino,
                              file->last_size);
        if (fd == -1)
            ret = -1;
        else if (fstat(fd, &st) < 0) {
                        mail_index_file_set_syscall_error(log->index, path,
                              "fstat()");
            (void)close(fd);
            fd = -1;
            ret = -1;
        }

        if (fd != -1) {
            (void)close(file->fd);
            file->fd = fd;

            file->st_dev = st.st_dev;
            file->st_ino = st.st_ino;
            file->last_mtime = st.st_mtime;
            file->last_size = st.st_size;

            memset(&file->hdr, 0, sizeof(file->hdr));
            ret = mail_transaction_log_file_read_hdr(file, TRUE,
                                 !try_retry);
        }
    }
    if (ret <= 0) {
                *retry_r = errno == ESTALE && try_retry;
        mail_transaction_log_file_free(file);
        return NULL;
    }

        mail_transaction_log_file_add_to_head(file);
    return file;
}

static struct mail_transaction_log_file *
mail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)
{
    struct mail_transaction_log_file *file;

    file = i_new(struct mail_transaction_log_file, 1);
    file->log = log;
    file->filepath = i_strdup("(in-memory transaction log file)");
    file->fd = -1;

    if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {
        i_free(file);
        return NULL;
    }

    file->buffer = buffer_create_dynamic(default_pool, 4096);
    file->buffer_offset = sizeof(file->hdr);

    mail_transaction_log_file_add_to_head(file);
    return file;
}

static struct mail_transaction_log_file *
mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
                     const char *path)
{
        struct mail_transaction_log_file *file;
        unsigned int i;
        bool retry;
        int fd;

    if (MAIL_INDEX_IS_IN_MEMORY(log->index))
        return mail_transaction_log_file_alloc_in_memory(log);

        for (i = 0; ; i++) {
                fd = nfs_safe_open(path, O_RDWR);
                if (fd == -1) {
                        if (errno != ENOENT) {
                                mail_index_file_set_syscall_error(log->index,
                                                                  path,
                                                                  "open()");
                                return NULL;
                        }

                        /* doesn't exist, try creating it */
                        fd = mail_transaction_log_file_create(log, path,
                                                              0, 0, 0);
                        if (fd == -1)
                                return NULL;
                }

                file = mail_transaction_log_file_fd_open_or_create(log, path,
                        fd, &retry, i == MAIL_INDEX_ESTALE_RETRY_COUNT);
                if (file != NULL || !retry)
                        return file;

                /* ESTALE - retry */
        }
}

static struct mail_transaction_log_file *
mail_transaction_log_file_open(struct mail_transaction_log *log,
                   const char *path)
{
    struct mail_transaction_log_file *file;
        unsigned int i;
        int fd, ret;

        for (i = 0;; i++) {
                fd = nfs_safe_open(path, O_RDWR);
                if (fd == -1) {
                        mail_index_file_set_syscall_error(log->index, path,
                                                          "open()");
                        return NULL;
                }

                ret = mail_transaction_log_file_fd_open(log, &file, path,
                        fd, TRUE, i < MAIL_INDEX_ESTALE_RETRY_COUNT);
                if (ret > 0)
                        break;

                /* error / corrupted */
                if (ret == 0)
                        mail_transaction_log_file_free(file);
                else {
                        if (errno == ESTALE &&
                            i < MAIL_INDEX_ESTALE_RETRY_COUNT) {
                                /* try again */
                                continue;
                        }
                }
                return NULL;
        }

    mail_transaction_log_file_add_to_head(file);
    return file;
}

void mail_transaction_logs_clean(struct mail_transaction_log *log)
{
    struct mail_transaction_log_file *file, *next;

    for (file = log->files; file != NULL; file = next) {
        next = file->next;

        i_assert(file->refcount >= 0);
        if (file->refcount == 0)
            mail_transaction_log_file_free(file);
    }
}

int mail_transaction_log_rotate(struct mail_transaction_log *log, bool lock)
{
    struct mail_transaction_log_file *file;
    const char *path = log->head->filepath;
        struct stat st;
        bool retry;
    int fd;

    i_assert(log->head->locked);

    if (MAIL_INDEX_IS_IN_MEMORY(log->index))
        file = mail_transaction_log_file_alloc_in_memory(log);
    else {
                /* we're locked, we shouldn't need to worry about ESTALE
                   problems in here. */
        if (fstat(log->head->fd, &st) < 0) {
            mail_index_file_set_syscall_error(log->index, path,
                              "fstat()");
            return -1;
        }

        fd = mail_transaction_log_file_create(log, path,
                              st.st_dev, st.st_ino,
                              st.st_size);
        if (fd == -1)
            return -1;

                file = mail_transaction_log_file_fd_open_or_create(log, path,
                                    fd, &retry, FALSE);
                if (file == NULL) {
            i_assert(!retry);
                        return -1;
                }
    }

    if (lock) {
        if (mail_transaction_log_file_lock(file) < 0) {
            mail_transaction_logs_clean(log);
            return -1;
        }
    }
    i_assert(file->locked == lock);

    if (--log->head->refcount == 0)
        mail_transaction_logs_clean(log);
    else
        mail_transaction_log_file_unlock(log->head);

    i_assert(log->head != file);
    log->head = file;
    log->head->refcount++;
    return 0;
}

static int mail_transaction_log_refresh(struct mail_transaction_log *log,
                    bool create_if_needed)
{
        struct mail_transaction_log_file *file;
    struct stat st;
    const char *path;

    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(log->head))
        return 0;

    path = t_strconcat(log->index->filepath,
               MAIL_TRANSACTION_LOG_SUFFIX, NULL);
    if (nfs_safe_stat(path, &st) < 0) {
        if (errno != ENOENT) {
            mail_index_file_set_syscall_error(log->index, path,
                              "stat()");
            return -1;
        }
        /* log was deleted. just reopen/recreate it. */
    } else {
        if (log->head != NULL &&
            log->head->st_ino == st.st_ino &&
            CMP_DEV_T(log->head->st_dev, st.st_dev)) {
            /* same file */
            return 0;
        }
    }

    file = create_if_needed ?
        mail_transaction_log_file_open_or_create(log, path) :
        mail_transaction_log_file_open(log, path);
    if (file == NULL)
        return -1;

    i_assert(!file->locked);

    if (log->head != NULL) {
        if (--log->head->refcount == 0)
            mail_transaction_logs_clean(log);
    }

    log->head = file;
    log->head->refcount++;
    return 0;
}

int mail_transaction_log_file_find(struct mail_transaction_log *log,
                   uint32_t file_seq,
                   struct mail_transaction_log_file **file_r)
{
    struct mail_transaction_log_file *file;
    struct stat st;
    const char *path;
    int ret, fd;

    if (file_seq > log->head->hdr.file_seq) {
        /* don't try to recreate log file if it gets lost. we're
           already in trouble and with mmap_disable the creation
           could cause a recursive mail_index_map() call */
        if (mail_transaction_log_refresh(log, FALSE) < 0)
            return -1;
    }

    for (file = log->files; file != NULL; file = file->next) {
        if (file->hdr.file_seq == file_seq) {
            *file_r = file;
            return 1;
        }
    }

    /* see if we have it in log.2 file */
    path = t_strconcat(log->index->filepath,
               MAIL_TRANSACTION_LOG_SUFFIX".2", NULL);
    fd = nfs_safe_open(path, O_RDWR);
    if (fd == -1) {
        if (errno == ENOENT)
            return 0;

        mail_index_file_set_syscall_error(log->index, path, "open()");
        return -1;
    }

    if (fstat(fd, &st) < 0) {
        close_keep_errno(fd);
                if (errno == ESTALE) {
                        /* treat as "doesn't exist" */
                        return 0;
                }
                mail_index_file_set_syscall_error(log->index, path, "fstat()");
        return -1;
    }

    /* see if we have it already opened */
    for (file = log->files; file != NULL; file = file->next) {
        if (file->st_ino == st.st_ino &&
            CMP_DEV_T(file->st_dev, st.st_dev)) {
            if (close(fd) < 0)
                i_error("close() failed: %m");
            return 0;
        }
    }

    ret = mail_transaction_log_file_fd_open(log, &file, path, fd,
                        FALSE, TRUE);
    if (ret <= 0) {
        bool stale = errno == ESTALE;

        if (ret == 0) {
            /* corrupted, delete it */
            if (unlink(file->filepath) < 0 && errno != ENOENT) {
                i_error("unlink(%s) failed: %m",
                    file->filepath);
            }
            mail_transaction_log_file_free(file);
            return 0;
                }
        mail_transaction_log_file_free(file);

        if (stale) {
                        /* treat as "doesn't exist" */
                        return 0;
                }
        return -1;
    }

    /* got it */
    mail_transaction_log_file_add_to_list(file);

    /* but is it what we expected? */
    if (file->hdr.file_seq != file_seq)
        return 0;

    *file_r = file;
    return 1;
}

static int
mail_transaction_log_file_sync(struct mail_transaction_log_file *file)
{
        const struct mail_transaction_header *hdr;
    const void *data;
    size_t size;
    uint32_t hdr_size = 0;

    data = buffer_get_data(file->buffer, &size);

    if (file->sync_offset < file->buffer_offset)
        file->sync_offset = file->buffer_offset;

    while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {
        hdr = CONST_PTR_OFFSET(data, file->sync_offset -
                       file->buffer_offset);
        hdr_size = mail_index_offset_to_uint32(hdr->size);
        if (hdr_size == 0) {
            /* unfinished */
            return 0;
        }
        if (hdr_size < sizeof(*hdr)) {
            mail_transaction_log_file_set_corrupted(file,
                "hdr.size too small (%u)", hdr_size);
            return -1;
        }

        if (file->sync_offset - file->buffer_offset + hdr_size > size)
            break;
        file->sync_offset += hdr_size;
    }

    if (file->sync_offset - file->buffer_offset != size) {
        /* record goes outside the file we've seen. or if
           we're accessing the log file via unlocked mmaped
           memory, it may be just that the memory was updated
           after we checked the file size. */
        if (file->locked || file->mmap_base == NULL) {
            mail_transaction_log_file_set_corrupted(file,
                "hdr.size too large (%u)", hdr_size);
            return -1;
        }
    }
    return 0;
}

static int
mail_transaction_log_file_read(struct mail_transaction_log_file *file,
                   uoff_t offset)
{
    void *data;
    size_t size;
    uint32_t read_offset;
    int ret;

    i_assert(file->mmap_base == NULL);

    if (file->buffer != NULL && file->buffer_offset > offset) {
        /* we have to insert missing data to beginning of buffer */
        size = file->buffer_offset - offset;
        buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
        file->buffer_offset -= size;

        data = buffer_get_space_unsafe(file->buffer, 0, size);
        ret = pread_full(file->fd, data, size, offset);
        if (ret == 0) {
            mail_transaction_log_file_set_corrupted(file,
                "Unexpected end of file");
            return 0;
        }
        if (ret < 0) {
            if (errno == ESTALE) {
                /* log file was deleted in NFS server,
                   fail silently */
                return 0;
            }
            mail_index_file_set_syscall_error(file->log->index,
                              file->filepath,
                              "pread()");
            return -1;
        }
    }

    if (file->buffer == NULL) {
        file->buffer =
            buffer_create_dynamic(default_pool, LOG_PREFETCH);
        file->buffer_offset = offset;
    }

    /* read all records */
    read_offset = file->buffer_offset + buffer_get_used_size(file->buffer);

    do {
        data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);
        ret = pread(file->fd, data, LOG_PREFETCH, read_offset);
        if (ret > 0)
            read_offset += ret;

        size = read_offset - file->buffer_offset;
        buffer_set_used_size(file->buffer, size);
    } while (ret > 0 || (ret < 0 && errno == EINTR));

    if (mail_transaction_log_file_sync(file) < 0)
        return -1;

    if (ret == 0) {
        /* EOF */
        i_assert(file->sync_offset >= file->buffer_offset);
        buffer_set_used_size(file->buffer,
                     file->sync_offset - file->buffer_offset);
        return 1;
    }

    if (errno == ESTALE) {
        /* log file was deleted in NFS server, fail silently */
        return 0;
    }

    mail_index_file_set_syscall_error(file->log->index, file->filepath,
                      "pread()");
    return -1;
}

int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
                  uoff_t start_offset, uoff_t end_offset)
{
    struct mail_index *index = file->log->index;
    size_t size;
    struct stat st;
    int ret, use_mmap;

    i_assert(start_offset <= end_offset);

    if (file->hdr.indexid == 0) {
        /* corrupted */
        return 0;
    }

    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
        return 1;

    if (start_offset < file->hdr.hdr_size) {
        mail_transaction_log_file_set_corrupted(file,
            "offset (%"PRIuUOFF_T") < header size (%u)",
            start_offset, file->hdr.hdr_size);
        return -1;
    }

    /* with mmap_no_write we could alternatively just write to log with
       msync() rather than pwrite(). but since there aren't many such OSes
       left, it's easier to just use mmap_disable behavior with it */
    use_mmap = !index->mmap_disable && !index->mmap_no_write;

    if (file->buffer != NULL && file->buffer_offset <= start_offset) {
        /* see if we already have it */
        size = buffer_get_used_size(file->buffer);
        if (file->buffer_offset + size >= end_offset)
            return 1;
    }

    if (use_mmap) {
        if (fstat(file->fd, &st) < 0) {
            mail_index_file_set_syscall_error(index, file->filepath,
                              "fstat()");
            return -1;
        }
        if (start_offset > (uoff_t)st.st_size) {
            mail_transaction_log_file_set_corrupted(file,
                "start_offset (%"PRIuUOFF_T") > file size "
                "(%"PRIuUOFF_T")", start_offset,
                (uoff_t)st.st_size);
            return -1;
        }

        if (file->mmap_base != NULL &&
            (uoff_t)st.st_size == file->mmap_size &&
            file->buffer_offset <= start_offset &&
            end_offset == (uoff_t)-1) {
            /* it's all mmaped already */
            if (mail_transaction_log_file_sync(file) < 0)
                return -1;
            return 1;
        }
    }

    if (file->buffer != NULL &&
        (file->mmap_base != NULL || use_mmap)) {
        buffer_free(file->buffer);
        file->buffer = NULL;
    }
    if (file->mmap_base != NULL) {
        if (munmap(file->mmap_base, file->mmap_size) < 0) {
            mail_index_file_set_syscall_error(index, file->filepath,
                              "munmap()");
        }
        file->mmap_base = NULL;
    }

    if (!use_mmap) {
        ret = mail_transaction_log_file_read(file, start_offset);
        if (ret <= 0) {
            /* make sure we don't leave ourself in
               inconsistent state */
            if (file->buffer != NULL) {
                buffer_free(file->buffer);
                file->buffer = NULL;
            }
            return ret;
        }
    } else {
        file->mmap_size = st.st_size;
        file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ,
                       MAP_SHARED, file->fd, 0);
        if (file->mmap_base == MAP_FAILED) {
            file->mmap_base = NULL;
            mail_index_file_set_syscall_error(index, file->filepath,
                              "mmap()");
            return -1;
        }

        if (file->mmap_size > mmap_get_page_size()) {
            if (madvise(file->mmap_base, file->mmap_size,
                    MADV_SEQUENTIAL) < 0) {
                mail_index_file_set_syscall_error(index,
                    file->filepath, "madvise()");
            }
        }

        file->buffer = buffer_create_const_data(default_pool,
                            file->mmap_base,
                            file->mmap_size);
        file->buffer_offset = 0;

        if (mail_transaction_log_file_sync(file) < 0)
            return -1;
    }

    if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {
        mail_transaction_log_file_set_corrupted(file,
            "end_offset (%"PRIuUOFF_T") > current sync_offset "
            "(%"PRIuUOFF_T")", end_offset, file->sync_offset);
        return -1;
    }

    return 1;
}

int mail_transaction_log_lock_head(struct mail_transaction_log *log)
{
    struct mail_transaction_log_file *file;
    int ret = 0;

    /* we want to get the head file locked. this is a bit racy,
       since by the time we have it locked a new log file may have been
       created.

       creating new log file requires locking the head file, so if we
       can lock it and don't see another file, we can be sure no-one is
       creating a new log at the moment */

    for (;;) {
        file = log->head;
        if (mail_transaction_log_file_lock(file) < 0)
            return -1;

        file->refcount++;
        ret = mail_transaction_log_refresh(log, TRUE);
        if (--file->refcount == 0) {
            mail_transaction_logs_clean(log);
            file = NULL;
        }

        if (ret == 0 && log->head == file) {
            /* success */
            break;
        }

        if (file != NULL)
            mail_transaction_log_file_unlock(file);

        if (ret < 0)
            break;

        /* try again */
    }

    return ret;
}

int mail_transaction_log_sync_lock(struct mail_transaction_log *log,
                   uint32_t *file_seq_r, uoff_t *file_offset_r)
{
    i_assert(!log->index->log_locked);

    if (mail_transaction_log_lock_head(log) < 0)
        return -1;

    /* update sync_offset */
    if (mail_transaction_log_file_map(log->head, log->head->sync_offset,
                      (uoff_t)-1) < 0) {
        mail_transaction_log_file_unlock(log->head);
        return -1;
    }

    log->index->log_locked = TRUE;
    *file_seq_r = log->head->hdr.file_seq;
    *file_offset_r = log->head->sync_offset;
    return 0;
}

void mail_transaction_log_sync_unlock(struct mail_transaction_log *log)
{
    i_assert(log->index->log_locked);

    log->index->log_locked = FALSE;
    mail_transaction_log_file_unlock(log->head);
}

void mail_transaction_log_get_head(struct mail_transaction_log *log,
                   uint32_t *file_seq_r, uoff_t *file_offset_r)
{
    i_assert(log->index->log_locked);

    *file_seq_r = log->head->hdr.file_seq;
    *file_offset_r = log->head->sync_offset;
}

bool mail_transaction_log_is_head_prev(struct mail_transaction_log *log,
                       uint32_t file_seq, uoff_t file_offset)
{
    return log->head->hdr.prev_file_seq == file_seq &&
        log->head->hdr.prev_file_offset == file_offset;
}