src/lib-index/mail-transaction-log.c

	mail-transaction-log.c revision 37598dd7a0613a03b12170c450b53c70f2b3d438
2454dfa32c93c20a8522c6ed42fe057baaac9f9aStephan Bosch/* Copyright (C) 2003-2004 Timo Sirainen */
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "lib.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "ioloop.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "buffer.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "file-dotlock.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "nfs-workarounds.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "close-keep-errno.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "read-full.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "write-full.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mmap-util.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mail-index-private.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mail-index-view-private.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mail-transaction-log-private.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mail-transaction-util.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mail-index-transaction-private.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include <stddef.h>
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include <stdio.h>
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen#include <sys/stat.h>
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen#define LOG_PREFETCH 1024
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen/* this lock should never exist for a long time.. */
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen#define LOG_DOTLOCK_TIMEOUT 60
5a8ee853d0b62692a6e624b125d08d87a79e001fTimo Sirainen#define LOG_DOTLOCK_STALE_TIMEOUT 60
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen#define MAIL_TRANSACTION_LOG_SUFFIX ".log"
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen#define LOG_NEW_DOTLOCK_SUFFIX ".newlock"
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenstatic struct mail_transaction_log_file *
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                     const char *path);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenstatic int
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmail_transaction_log_file_read(struct mail_transaction_log_file *file,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                   uoff_t offset);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenvoid
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                    const char *fmt, ...)
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen{
5a8ee853d0b62692a6e624b125d08d87a79e001fTimo Sirainen    va_list va;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    file->hdr.indexid = 0;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        if (pwrite_full(file->fd, &file->hdr.indexid,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                sizeof(file->hdr.indexid), 0) < 0) {
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen            mail_index_file_set_syscall_error(file->log->index,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                file->filepath, "pwrite()");
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        }
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    }
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    va_start(va, fmt);
58a770f1e0ab553a0dba9cad9d6f3a6cdf2dc855Timo Sirainen    t_push();
58a770f1e0ab553a0dba9cad9d6f3a6cdf2dc855Timo Sirainen    mail_index_set_error(file->log->index,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                 "Corrupted transaction log file %s: %s",
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                 file->filepath, t_strdup_vprintf(fmt, va));
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    t_pop();
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    va_end(va);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    if (file->log->index->log != NULL) {
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        /* this may have happened because of broken index.
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen           make sure it's ok. */
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        (void)mail_index_fsck(file->log->index);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    }
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen}
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenstatic int
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen{
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    int ret;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    if (file->log->dotlock_count > 0)
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        ret = 1;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    else {
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        ret = file_dotlock_create(&file->log->dotlock_settings,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                      file->filepath, 0,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                      &file->log->dotlock);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    }
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    if (ret > 0) {
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        file->log->dotlock_count++;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        file->locked = TRUE;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        return 0;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    }
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    if (ret < 0) {
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen                          file->filepath,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                          "file_dotlock_create()");
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        return -1;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    }
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    mail_index_set_error(file->log->index,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                 "Timeout while waiting for release of "
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                 "dotlock for transaction log file %s",
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                 file->filepath);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    file->log->index->index_lock_timeout = TRUE;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    return -1;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen}
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenstatic int
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen{
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    int ret;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    if (--file->log->dotlock_count > 0)
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        return 0;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    ret = file_dotlock_delete(&file->log->dotlock);
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    if (ret < 0) {
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen            file->filepath, "file_dotlock_delete()");
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        return -1;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    }
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    if (ret == 0) {
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        mail_index_set_error(file->log->index,
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen            "Dotlock was lost for transaction log file %s",
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen            file->filepath);
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        return -1;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    }
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    return 0;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen}
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainenstatic int
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainenmail_transaction_log_file_lock(struct mail_transaction_log_file *file)
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen{
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen    int ret;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    if (file->locked)
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        return 0;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        file->locked = TRUE;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        return 0;
a8ebb72c0fba1a6a71104e530bf5903d5f149351Timo Sirainen    }
a8ebb72c0fba1a6a71104e530bf5903d5f149351Timo Sirainen
a8ebb72c0fba1a6a71104e530bf5903d5f149351Timo Sirainen    if (file->log->index->lock_method == MAIL_INDEX_LOCK_DOTLOCK)
a8ebb72c0fba1a6a71104e530bf5903d5f149351Timo Sirainen        return mail_transaction_log_file_dotlock(file);
a8ebb72c0fba1a6a71104e530bf5903d5f149351Timo Sirainen
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen    ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen                 F_WRLCK, MAIL_INDEX_LOCK_SECS);
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    if (ret > 0) {
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        file->locked = TRUE;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        return 0;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    }
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    if (ret < 0) {
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen                          file->filepath,
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen                          "mail_index_wait_lock_fd()");
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        return -1;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    }
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    mail_index_set_error(file->log->index,
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        "Timeout while waiting for lock for transaction log file %s",
8ae72ad7d0c69e972cfa65d1e2ce4e3e9a8b765cTimo Sirainen        file->filepath);
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    file->log->index->index_lock_timeout = TRUE;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    return -1;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen}
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainenvoid mail_transaction_log_file_unlock(struct mail_transaction_log_file *file)
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen{
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    int ret;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    if (!file->locked)
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        return;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    file->locked = FALSE;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen        return;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    if (file->log->index->lock_method == MAIL_INDEX_LOCK_DOTLOCK) {
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen        mail_transaction_log_file_undotlock(file);
8ae72ad7d0c69e972cfa65d1e2ce4e3e9a8b765cTimo Sirainen        return;
99df8a838cd9c5257ea5a2554383a9a999191e38Pali Rohár    }
99df8a838cd9c5257ea5a2554383a9a999191e38Pali Rohár
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen                 F_UNLCK, 0);
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    if (ret <= 0) {
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen        mail_index_file_set_syscall_error(file->log->index,
99df8a838cd9c5257ea5a2554383a9a999191e38Pali Rohár                          file->filepath,
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen                          "mail_index_wait_lock_fd()");
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    }
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen}
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#define INDEX_HAS_MISSING_LOGS(index, file) \
baf3e87e186453fda13bd21f7cbcb2efc8492e8bTimo Sirainen    !(((file)->hdr.file_seq == (index)->hdr->log_file_seq && \
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen       (index)->hdr->log_file_int_offset >= (file)->hdr.hdr_size) || \
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen      ((file)->hdr.prev_file_seq == (index)->hdr->log_file_seq && \
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen       (file)->hdr.prev_file_offset == (index)->hdr->log_file_int_offset))
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainenstatic int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)
{
    struct mail_index *index = log->index;
    struct mail_transaction_log_file *file;
    unsigned int lock_id;
    int ret;

    if (mail_transaction_log_lock_head(log) < 0)
        return -1;

    file = log->head;
    file->refcount++;

    ret = mail_index_lock_shared(index, TRUE, &lock_id);
    if (ret == 0) {
        ret = mail_index_map(index, FALSE);
        if (ret <= 0)
            ret = -1;
        else if (INDEX_HAS_MISSING_LOGS(index, file)) {
            /* broken - fix it by creating a new log file */
            ret = mail_transaction_log_rotate(log, FALSE);
        }
    }

    if (--file->refcount == 0)
        mail_transaction_logs_clean(log);
    else
        mail_transaction_log_file_unlock(file);
    return ret;
}

struct mail_transaction_log *
mail_transaction_log_open_or_create(struct mail_index *index)
{
    struct mail_transaction_log *log;
    const char *path;

    log = i_new(struct mail_transaction_log, 1);
    log->index = index;

    log->dotlock_settings.timeout = LOG_DOTLOCK_TIMEOUT;
    log->dotlock_settings.stale_timeout = LOG_DOTLOCK_STALE_TIMEOUT;

    log->new_dotlock_settings = log->dotlock_settings;
    log->new_dotlock_settings.lock_suffix = LOG_NEW_DOTLOCK_SUFFIX;

    path = t_strconcat(log->index->filepath,
               MAIL_TRANSACTION_LOG_SUFFIX, NULL);
    log->head = mail_transaction_log_file_open_or_create(log, path);
    if (log->head == NULL) {
        /* fallback to in-memory indexes */
        if (mail_index_move_to_memory(index) < 0) {
            mail_transaction_log_close(&log);
            return NULL;
        }
        log->head = mail_transaction_log_file_open_or_create(log, path);
        i_assert(log->head != NULL);
    }

    if (index->fd != -1 &&
        INDEX_HAS_MISSING_LOGS(index, log->head)) {
        /* head log file isn't same as head index file -
           shouldn't happen except in race conditions.
           lock them and check again */
        if (mail_transaction_log_check_file_seq(log) < 0) {
            mail_transaction_log_close(&log);
            return NULL;
        }
    }
    return log;
}

void mail_transaction_log_close(struct mail_transaction_log **_log)
{
    struct mail_transaction_log *log = *_log;

    mail_transaction_log_views_close(log);

    if (log->head != NULL)
        log->head->refcount--;
    mail_transaction_logs_clean(log);
    i_assert(log->tail == NULL);

    *_log = NULL;
    log->index->log = NULL;
    i_free(log);
}

static void
mail_transaction_log_file_free(struct mail_transaction_log_file *file)
{
        int old_errno = errno;

    mail_transaction_log_file_unlock(file);

    if (file == file->log->head)
        file->log->head = NULL;
    if (file == file->log->tail)
        file->log->tail = file->next;

    if (file->buffer != NULL)
        buffer_free(file->buffer);

    if (file->mmap_base != NULL) {
        if (munmap(file->mmap_base, file->mmap_size) < 0) {
            mail_index_file_set_syscall_error(file->log->index,
                              file->filepath,
                              "munmap()");
        }
    }

    if (file->fd != -1) {
        if (close(file->fd) < 0) {
            mail_index_file_set_syscall_error(file->log->index,
                              file->filepath,
                              "close()");
        }
    }

    i_free(file->filepath);
        i_free(file);

        errno = old_errno;
}

int mail_transaction_log_move_to_memory(struct mail_transaction_log *log)
{
    struct mail_transaction_log_file *file = log->head;

    if (file == NULL || MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
        return 0;

    /* read the whole file to memory. we might currently be appending
       data into it, so we want to read it up to end of file */
        file->buffer_offset = 0;

    if (file->buffer != NULL) {
        buffer_free(file->buffer);
        file->buffer = NULL;
    }

    if (file->mmap_base != NULL) {
        if (munmap(file->mmap_base, file->mmap_size) < 0) {
            mail_index_file_set_syscall_error(file->log->index,
                              file->filepath,
                              "munmap()");
        }
        file->mmap_base = NULL;
    }

    if (mail_transaction_log_file_read(file, 0) <= 0)
        return -1;

    /* after we've read the file into memory, make it into in-memory
       log file */
    if (close(file->fd) < 0) {
        mail_index_file_set_syscall_error(file->log->index,
                          file->filepath, "close()");
    }
    file->fd = -1;
    return 0;
}

static int
mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
                   int head, bool ignore_estale)
{
        struct mail_transaction_log_file *f;
    int ret;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(file->log->index));

    ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);
    if (ret < 0) {
                if (errno != ESTALE || !ignore_estale) {
                        mail_index_file_set_syscall_error(file->log->index,
                                                          file->filepath,
                                                          "pread_full()");
                }
        return -1;
    }
    if (ret == 0) {
        mail_transaction_log_file_set_corrupted(file,
            "unexpected end of file while reading header");
        return 0;
    }

    if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {
        /* incompatible version - fix silently */
        return 0;
    }
    if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
        mail_transaction_log_file_set_corrupted(file,
            "Header size too small");
        return 0;
    }
    if (file->hdr.hdr_size < sizeof(file->hdr)) {
        /* @UNSAFE: smaller than we expected - zero out the fields we
           shouldn't have filled */
        memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
               sizeof(file->hdr) - file->hdr.hdr_size);
    }

    if (file->hdr.indexid == 0) {
        /* corrupted */
        mail_index_set_error(file->log->index,
            "Transaction log file %s: marked corrupted",
            file->filepath);
        return 0;
    }
    if (file->hdr.indexid != file->log->index->indexid) {
        if (file->log->index->fd != -1) {
            /* index file was probably just rebuilt and we don't
               know about it yet */
            mail_transaction_log_file_set_corrupted(file,
                "invalid indexid (%u != %u)",
                file->hdr.indexid, file->log->index->indexid);
            return 0;
        }

        /* creating index file. since transaction log is created
           first, use the indexid in it to create the main index
           to avoid races. */
        file->log->index->indexid = file->hdr.indexid;
    }

    /* make sure we already don't have a file with the same sequence
       opened. it shouldn't happen unless the old log file was
       corrupted.

       If we're opening head log file, make sure the sequence is larger
       than any existing one. */
    if (head) {
        for (f = file->log->tail; f != NULL; f = f->next) {
            if (f->hdr.file_seq >= file->hdr.file_seq) {
                mail_transaction_log_file_set_corrupted(file,
                    "invalid new transaction log sequence "
                    "(%u >= %u)",
                    f->hdr.file_seq, file->hdr.file_seq);
                return 0;
            }
        }
    } else {
        for (f = file->log->tail; f != NULL; f = f->next) {
            if (f->hdr.file_seq == file->hdr.file_seq) {
                mail_transaction_log_file_set_corrupted(file,
                    "old transaction log already opened "
                    "(%u == %u)",
                    f->hdr.file_seq, file->hdr.file_seq);
                return 0;
            }
        }
    }

    return 1;
}

static int
mail_transaction_log_init_hdr(struct mail_transaction_log *log,
                  struct mail_transaction_log_header *hdr)
{
    struct mail_index *index = log->index;
    unsigned int lock_id;

    memset(hdr, 0, sizeof(*hdr));
    hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;
    hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;
    hdr->hdr_size = sizeof(struct mail_transaction_log_header);
    hdr->indexid = log->index->indexid;
    hdr->create_stamp = ioloop_time;

    if (index->fd != -1) {
        /* not creating index - make sure we have latest header */
        if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
            return -1;
        if (mail_index_map(index, FALSE) <= 0) {
            mail_index_unlock(index, lock_id);
            return -1;
        }
    }
    hdr->prev_file_seq = index->hdr->log_file_seq;
    hdr->prev_file_offset = index->hdr->log_file_int_offset;
    hdr->file_seq = index->hdr->log_file_seq+1;

    if (index->fd != -1)
        mail_index_unlock(index, lock_id);

    if (log->head != NULL && hdr->file_seq <= log->head->hdr.file_seq) {
        /* make sure the sequence grows */
        hdr->file_seq = log->head->hdr.file_seq+1;
    }
    return 0;
}

static int
mail_transaction_log_file_create2(struct mail_transaction_log *log,
                  const char *path, int new_fd,
                  struct dotlock **dotlock,
                  dev_t dev, ino_t ino, uoff_t file_size)
{
    struct mail_index *index = log->index;
    struct mail_transaction_log_header hdr;
    struct stat st;
    const char *path2;
    int old_fd, ret;
    bool found;

    /* log creation is locked now - see if someone already created it */
    old_fd = nfs_safe_open(path, O_RDWR);
    if (old_fd != -1) {
        if ((ret = fstat(old_fd, &st)) < 0) {
                        mail_index_file_set_syscall_error(index, path,
                                                          "fstat()");
        } else if (st.st_ino == ino && CMP_DEV_T(st.st_dev, dev) &&
               (uoff_t)st.st_size == file_size) {
            /* same file, still broken */
        } else {
                        /* file changed, use the new file */
                        (void)file_dotlock_delete(dotlock);
            return old_fd;
        }

        (void)close(old_fd);
        old_fd = -1;

                if (ret < 0) {
                        /* fstat() failure, return after closing fd.. */
                        return -1;
                }
        found = TRUE;
    } else if (errno != ENOENT) {
        mail_index_file_set_syscall_error(index, path, "open()");
        return -1;
    } else {
        found = FALSE;
    }

    if (mail_transaction_log_init_hdr(log, &hdr) < 0)
        return -1;

    if (write_full(new_fd, &hdr, sizeof(hdr)) < 0) {
        mail_index_file_set_syscall_error(index, path,
                          "write_full()");
        return -1;
    }

    /* keep two log files */
    if (found) {
        path2 = t_strconcat(path, ".2", NULL);
        if (rename(path, path2) < 0) {
                        mail_index_set_error(index,
                            "rename(%s, %s) failed: %m", path, path2);
            /* ignore the error. we don't care that much about the
               second log file and we're going to overwrite this
               first one. */
        }
    }

    if (file_dotlock_replace(dotlock,
                 DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) <= 0)
        return -1;

    /* success */
    return new_fd;
}

static int
mail_transaction_log_file_create(struct mail_transaction_log *log,
                 const char *path,
                 dev_t dev, ino_t ino, uoff_t file_size)
{
    struct dotlock *dotlock;
    struct stat st;
    mode_t old_mask;
    int fd;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(log->index));

    if (stat(log->index->dir, &st) < 0) {
        if (ENOTFOUND(errno)) {
            /* the whole index directory was deleted, which means
               the mailbox was deleted by another process.
               fail silently. */
            mail_index_mark_corrupted(log->index);
            return -1;
        }
        mail_index_file_set_syscall_error(log->index, log->index->dir,
                          "stat()");
        return -1;
    }

    /* With dotlocking we might already have path.lock created, so this
       filename has to be different. */
    old_mask = umask(log->index->mode ^ 0666);
    fd = file_dotlock_open(&log->new_dotlock_settings, path, 0, &dotlock);
    umask(old_mask);

    if (fd == -1) {
        mail_index_file_set_syscall_error(log->index, path,
                          "file_dotlock_open()");
        return -1;
    }

    if (log->index->gid != (gid_t)-1 &&
        fchown(fd, (uid_t)-1, log->index->gid) < 0) {
        mail_index_file_set_syscall_error(log->index, path, "fchown()");
        (void)file_dotlock_delete(&dotlock);
        return -1;
    }

        /* either fd gets used or the dotlock gets deleted and returned fd
           is for the existing file */
        fd = mail_transaction_log_file_create2(log, path, fd, &dotlock,
                                               dev, ino, file_size);
    if (fd < 0) {
        if (dotlock != NULL)
            (void)file_dotlock_delete(&dotlock);
        return -1;
    }
    return fd;
}

static void
mail_transaction_log_file_add_to_head(struct mail_transaction_log_file *file)
{
    struct mail_transaction_log *log = file->log;
    struct mail_transaction_log_file **p;

    if (log->index->map != NULL &&
        file->hdr.file_seq == log->index->map->hdr.log_file_seq &&
        log->index->map->hdr.log_file_int_offset != 0) {
        /* we can get a valid log offset from index file. initialize
           sync_offset from it so we don't have to read the whole log
           file from beginning. */
        file->sync_offset = log->index->map->hdr.log_file_int_offset;
    } else {
        file->sync_offset = file->hdr.hdr_size;
    }

    /* append to end of list. */
    for (p = &log->tail; *p != NULL; p = &(*p)->next)
        i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
    *p = file;
}

static void
mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
{
    struct mail_transaction_log *log = file->log;
    struct mail_transaction_log_file **p;

    file->sync_offset = file->hdr.hdr_size;

    /* insert it to correct position */
    for (p = &log->tail; *p != NULL; p = &(*p)->next) {
        i_assert((*p)->hdr.file_seq != file->hdr.file_seq);
        if ((*p)->hdr.file_seq > file->hdr.file_seq)
            break;
    }

    file->next = *p;
    *p = file;
}

static int
mail_transaction_log_file_fd_open(struct mail_transaction_log *log,
                                  struct mail_transaction_log_file **file_r,
                  const char *path, int fd, bool head,
                  bool ignore_estale)
{
        struct mail_transaction_log_file *file;
    struct stat st;
    int ret;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(log->index));

    *file_r = NULL;

    if (fstat(fd, &st) < 0) {
                if (errno != ESTALE || !ignore_estale) {
                        mail_index_file_set_syscall_error(log->index, path,
                                                          "fstat()");
                }
        close_keep_errno(fd);
        return -1;
    }

    file = i_new(struct mail_transaction_log_file, 1);
    file->refcount = 1;
    file->log = log;
    file->filepath = i_strdup(path);
    file->fd = fd;
    file->st_dev = st.st_dev;
    file->st_ino = st.st_ino;
    file->last_mtime = st.st_mtime;
    file->last_size = st.st_size;

    ret = mail_transaction_log_file_read_hdr(file, head, ignore_estale);
    if (ret < 0) {
        mail_transaction_log_file_free(file);
        return -1;
    }

    *file_r = file;
    return ret;
}


static struct mail_transaction_log_file *
mail_transaction_log_file_fd_open_or_create(struct mail_transaction_log *log,
                                            const char *path, int fd,
                                            bool *retry_r, bool try_retry)
{
    struct mail_transaction_log_file *file;
    struct stat st;
    int ret;

        *retry_r = FALSE;

    ret = mail_transaction_log_file_fd_open(log, &file, path, fd, TRUE,
                        !try_retry);
        if (ret < 0) {
                *retry_r = errno == ESTALE && try_retry;
                return NULL;
        }

    if (ret == 0) {
        /* corrupted header */
        fd = mail_transaction_log_file_create(log, path, file->st_dev,
                              file->st_ino,
                              file->last_size);
        if (fd == -1)
            ret = -1;
        else if (fstat(fd, &st) < 0) {
                        mail_index_file_set_syscall_error(log->index, path,
                              "fstat()");
            (void)close(fd);
            fd = -1;
            ret = -1;
        }

        if (fd != -1) {
            (void)close(file->fd);
            file->fd = fd;

            file->st_dev = st.st_dev;
            file->st_ino = st.st_ino;
            file->last_mtime = st.st_mtime;
            file->last_size = st.st_size;

            memset(&file->hdr, 0, sizeof(file->hdr));
            ret = mail_transaction_log_file_read_hdr(file, TRUE,
                                 !try_retry);
        }
    }
    if (ret <= 0) {
                *retry_r = errno == ESTALE && try_retry;
        mail_transaction_log_file_free(file);
        return NULL;
    }

        mail_transaction_log_file_add_to_head(file);
    return file;
}

static struct mail_transaction_log_file *
mail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)
{
    struct mail_transaction_log_file *file;

    file = i_new(struct mail_transaction_log_file, 1);
    file->refcount = 1;
    file->log = log;
    file->filepath = i_strdup("(in-memory transaction log file)");
    file->fd = -1;

    if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {
        i_free(file);
        return NULL;
    }

    file->buffer = buffer_create_dynamic(default_pool, 4096);
    file->buffer_offset = sizeof(file->hdr);

    mail_transaction_log_file_add_to_head(file);
    return file;
}

static struct mail_transaction_log_file *
mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
                     const char *path)
{
        struct mail_transaction_log_file *file;
        unsigned int i;
        bool retry;
        int fd;

    if (MAIL_INDEX_IS_IN_MEMORY(log->index))
        return mail_transaction_log_file_alloc_in_memory(log);

        for (i = 0; ; i++) {
                fd = nfs_safe_open(path, O_RDWR);
                if (fd == -1) {
                        if (errno != ENOENT) {
                                mail_index_file_set_syscall_error(log->index,
                                                                  path,
                                                                  "open()");
                                return NULL;
                        }

                        /* doesn't exist, try creating it */
                        fd = mail_transaction_log_file_create(log, path,
                                                              0, 0, 0);
                        if (fd == -1)
                                return NULL;
                }

                file = mail_transaction_log_file_fd_open_or_create(log, path,
                        fd, &retry, i == MAIL_INDEX_ESTALE_RETRY_COUNT);
                if (file != NULL || !retry)
                        return file;

                /* ESTALE - retry */
        }
}

static struct mail_transaction_log_file *
mail_transaction_log_file_open(struct mail_transaction_log *log,
                   const char *path)
{
    struct mail_transaction_log_file *file;
        unsigned int i;
        int fd, ret;

        for (i = 0;; i++) {
                fd = nfs_safe_open(path, O_RDWR);
                if (fd == -1) {
                        mail_index_file_set_syscall_error(log->index, path,
                                                          "open()");
                        return NULL;
                }

                ret = mail_transaction_log_file_fd_open(log, &file, path,
                        fd, TRUE, i < MAIL_INDEX_ESTALE_RETRY_COUNT);
                if (ret > 0)
                        break;

                /* error / corrupted */
                if (ret == 0)
                        mail_transaction_log_file_free(file);
                else {
                        if (errno == ESTALE &&
                            i < MAIL_INDEX_ESTALE_RETRY_COUNT) {
                                /* try again */
                                continue;
                        }
                }
                return NULL;
        }

    mail_transaction_log_file_add_to_head(file);
    return file;
}

void mail_transaction_logs_clean(struct mail_transaction_log *log)
{
    struct mail_transaction_log_file **p, *next;

    for (p = &log->tail; *p != NULL; ) {
        if ((*p)->refcount != 0)
                        p = &(*p)->next;
        else {
            next = (*p)->next;
            mail_transaction_log_file_free(*p);
            *p = next;
        }
    }
}

int mail_transaction_log_rotate(struct mail_transaction_log *log, bool lock)
{
    struct mail_transaction_log_file *file;
    const char *path = log->head->filepath;
        struct stat st;
        bool retry;
    int fd;

    i_assert(log->head->locked);

    if (MAIL_INDEX_IS_IN_MEMORY(log->index))
        file = mail_transaction_log_file_alloc_in_memory(log);
    else {
                /* we're locked, we shouldn't need to worry about ESTALE
                   problems in here. */
        if (fstat(log->head->fd, &st) < 0) {
            mail_index_file_set_syscall_error(log->index, path,
                              "fstat()");
            return -1;
        }

        fd = mail_transaction_log_file_create(log, path,
                              st.st_dev, st.st_ino,
                              st.st_size);
        if (fd == -1)
            return -1;

                file = mail_transaction_log_file_fd_open_or_create(log, path,
                                    fd, &retry, FALSE);
                if (file == NULL) {
            i_assert(!retry);
                        return -1;
                }
    }

    if (lock) {
        if (mail_transaction_log_file_lock(file) < 0) {
            file->refcount--;
            mail_transaction_logs_clean(log);
            return -1;
        }
    }
    i_assert(file->locked == lock);

    if (--log->head->refcount == 0)
        mail_transaction_logs_clean(log);
    else
        mail_transaction_log_file_unlock(log->head);

    i_assert(log->head != file);
    log->head = file;
    return 0;
}

static int mail_transaction_log_refresh(struct mail_transaction_log *log,
                    bool create_if_needed)
{
        struct mail_transaction_log_file *file;
    struct stat st;
    const char *path;

    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(log->head))
        return 0;

    path = t_strconcat(log->index->filepath,
               MAIL_TRANSACTION_LOG_SUFFIX, NULL);
    if (nfs_safe_stat(path, &st) < 0) {
        if (errno != ENOENT) {
            mail_index_file_set_syscall_error(log->index, path,
                              "stat()");
            return -1;
        }
        /* log was deleted. just reopen/recreate it. */
    } else {
        if (log->head != NULL &&
            log->head->st_ino == st.st_ino &&
            CMP_DEV_T(log->head->st_dev, st.st_dev)) {
            /* same file */
            return 0;
        }
    }

    file = create_if_needed ?
        mail_transaction_log_file_open_or_create(log, path) :
        mail_transaction_log_file_open(log, path);
    if (file == NULL)
        return -1;

    i_assert(!file->locked);

    if (log->head != NULL) {
        if (--log->head->refcount == 0)
            mail_transaction_logs_clean(log);
    }

    log->head = file;
    return 0;
}

int mail_transaction_log_file_find(struct mail_transaction_log *log,
                   uint32_t file_seq,
                   struct mail_transaction_log_file **file_r)
{
    struct mail_transaction_log_file *file;
    struct stat st;
    const char *path;
    int ret, fd;

    if (file_seq > log->head->hdr.file_seq) {
        /* don't try to recreate log file if it gets lost. we're
           already in trouble and with mmap_disable the creation
           could cause a recursive mail_index_map() call */
        if (mail_transaction_log_refresh(log, FALSE) < 0)
            return -1;
    }

    for (file = log->tail; file != NULL; file = file->next) {
        if (file->hdr.file_seq == file_seq) {
            *file_r = file;
            return 1;
        }
    }

    /* see if we have it in log.2 file */
    path = t_strconcat(log->index->filepath,
               MAIL_TRANSACTION_LOG_SUFFIX".2", NULL);
    fd = nfs_safe_open(path, O_RDWR);
    if (fd == -1) {
        if (errno == ENOENT)
            return 0;

        mail_index_file_set_syscall_error(log->index, path, "open()");
        return -1;
    }

    if (fstat(fd, &st) < 0) {
        close_keep_errno(fd);
                if (errno == ESTALE) {
                        /* treat as "doesn't exist" */
                        return 0;
                }
                mail_index_file_set_syscall_error(log->index, path, "fstat()");
        return -1;
    }

    /* see if we have it already opened */
    for (file = log->tail; file != NULL; file = file->next) {
        if (file->st_ino == st.st_ino &&
            CMP_DEV_T(file->st_dev, st.st_dev)) {
            if (close(fd) < 0)
                i_error("close() failed: %m");
            return 0;
        }
    }

    ret = mail_transaction_log_file_fd_open(log, &file, path, fd,
                        FALSE, TRUE);
    if (ret <= 0) {
        bool stale = errno == ESTALE;

        if (ret == 0) {
            /* corrupted, delete it */
            if (unlink(file->filepath) < 0 && errno != ENOENT) {
                i_error("unlink(%s) failed: %m",
                    file->filepath);
            }
            mail_transaction_log_file_free(file);
            return 0;
                }
        mail_transaction_log_file_free(file);

        if (stale) {
                        /* treat as "doesn't exist" */
                        return 0;
                }
        return -1;
    }

    /* got it */
    mail_transaction_log_file_add_to_list(file);

    /* but is it what we expected? */
    if (file->hdr.file_seq != file_seq)
        return 0;

    *file_r = file;
    return 1;
}

static int
mail_transaction_log_file_sync(struct mail_transaction_log_file *file)
{
        const struct mail_transaction_header *hdr;
    const void *data;
    size_t size;
    uint32_t hdr_size = 0;

    data = buffer_get_data(file->buffer, &size);

    if (file->sync_offset < file->buffer_offset)
        file->sync_offset = file->buffer_offset;

    while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {
        hdr = CONST_PTR_OFFSET(data, file->sync_offset -
                       file->buffer_offset);
        hdr_size = mail_index_offset_to_uint32(hdr->size);
        if (hdr_size == 0) {
            /* unfinished */
            return 0;
        }
        if (hdr_size < sizeof(*hdr)) {
            mail_transaction_log_file_set_corrupted(file,
                "hdr.size too small (%u)", hdr_size);
            return -1;
        }

        if (file->sync_offset - file->buffer_offset + hdr_size > size)
            break;
        file->sync_offset += hdr_size;
    }

    if (file->sync_offset - file->buffer_offset != size) {
        /* record goes outside the file we've seen. or if
           we're accessing the log file via unlocked mmaped
           memory, it may be just that the memory was updated
           after we checked the file size. */
        if (file->locked || file->mmap_base == NULL) {
            mail_transaction_log_file_set_corrupted(file,
                "hdr.size too large (%u)", hdr_size);
            return -1;
        }
    }
    return 0;
}

static int
mail_transaction_log_file_read(struct mail_transaction_log_file *file,
                   uoff_t offset)
{
    void *data;
    size_t size;
    uint32_t read_offset;
    int ret;

    i_assert(file->mmap_base == NULL);

    if (file->buffer != NULL && file->buffer_offset > offset) {
        /* we have to insert missing data to beginning of buffer */
        size = file->buffer_offset - offset;
        buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
        file->buffer_offset -= size;

        data = buffer_get_space_unsafe(file->buffer, 0, size);
        ret = pread_full(file->fd, data, size, offset);
        if (ret == 0) {
            mail_transaction_log_file_set_corrupted(file,
                "Unexpected end of file");
            return 0;
        }
        if (ret < 0) {
            if (errno == ESTALE) {
                /* log file was deleted in NFS server,
                   fail silently */
                return 0;
            }
            mail_index_file_set_syscall_error(file->log->index,
                              file->filepath,
                              "pread()");
            return -1;
        }
    }

    if (file->buffer == NULL) {
        file->buffer =
            buffer_create_dynamic(default_pool, LOG_PREFETCH);
        file->buffer_offset = offset;
    }

    /* read all records */
    read_offset = file->buffer_offset + buffer_get_used_size(file->buffer);

    do {
        data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);
        ret = pread(file->fd, data, LOG_PREFETCH, read_offset);
        if (ret > 0)
            read_offset += ret;

        size = read_offset - file->buffer_offset;
        buffer_set_used_size(file->buffer, size);
    } while (ret > 0 || (ret < 0 && errno == EINTR));

    if (mail_transaction_log_file_sync(file) < 0)
        return -1;

    if (ret == 0) {
        /* EOF */
        i_assert(file->sync_offset >= file->buffer_offset);
        buffer_set_used_size(file->buffer,
                     file->sync_offset - file->buffer_offset);
        return 1;
    }

    if (errno == ESTALE) {
        /* log file was deleted in NFS server, fail silently */
        return 0;
    }

    mail_index_file_set_syscall_error(file->log->index, file->filepath,
                      "pread()");
    return -1;
}

int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
                  uoff_t start_offset, uoff_t end_offset)
{
    struct mail_index *index = file->log->index;
    size_t size;
    struct stat st;
    int ret, use_mmap;

    i_assert(start_offset <= end_offset);

    if (file->hdr.indexid == 0) {
        /* corrupted */
        return 0;
    }

    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
        return 1;

    if (start_offset < file->hdr.hdr_size) {
        mail_transaction_log_file_set_corrupted(file,
            "offset (%"PRIuUOFF_T") < header size (%u)",
            start_offset, file->hdr.hdr_size);
        return -1;
    }

    /* with mmap_no_write we could alternatively just write to log with
       msync() rather than pwrite(). but since there aren't many such OSes
       left, it's easier to just use mmap_disable behavior with it */
    use_mmap = !index->mmap_disable && !index->mmap_no_write;

    if (file->buffer != NULL && file->buffer_offset <= start_offset) {
        /* see if we already have it */
        size = buffer_get_used_size(file->buffer);
        if (file->buffer_offset + size >= end_offset)
            return 1;
    }

    if (use_mmap) {
        if (fstat(file->fd, &st) < 0) {
            mail_index_file_set_syscall_error(index, file->filepath,
                              "fstat()");
            return -1;
        }
        if (start_offset > (uoff_t)st.st_size) {
            mail_transaction_log_file_set_corrupted(file,
                "start_offset (%"PRIuUOFF_T") > file size "
                "(%"PRIuUOFF_T")", start_offset,
                (uoff_t)st.st_size);
            return -1;
        }

        if (file->mmap_base != NULL &&
            (uoff_t)st.st_size == file->mmap_size &&
            file->buffer_offset <= start_offset &&
            end_offset == (uoff_t)-1) {
            /* it's all mmaped already */
            if (mail_transaction_log_file_sync(file) < 0)
                return -1;
            return 1;
        }
    }

    if (file->buffer != NULL &&
        (file->mmap_base != NULL || use_mmap)) {
        buffer_free(file->buffer);
        file->buffer = NULL;
    }
    if (file->mmap_base != NULL) {
        if (munmap(file->mmap_base, file->mmap_size) < 0) {
            mail_index_file_set_syscall_error(index, file->filepath,
                              "munmap()");
        }
        file->mmap_base = NULL;
    }

    if (!use_mmap) {
        ret = mail_transaction_log_file_read(file, start_offset);
        if (ret <= 0) {
            /* make sure we don't leave ourself in
               inconsistent state */
            if (file->buffer != NULL) {
                buffer_free(file->buffer);
                file->buffer = NULL;
            }
            return ret;
        }
    } else {
        file->mmap_size = st.st_size;
        file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ,
                       MAP_SHARED, file->fd, 0);
        if (file->mmap_base == MAP_FAILED) {
            file->mmap_base = NULL;
            mail_index_file_set_syscall_error(index, file->filepath,
                              "mmap()");
            return -1;
        }

        if (file->mmap_size > mmap_get_page_size()) {
            if (madvise(file->mmap_base, file->mmap_size,
                    MADV_SEQUENTIAL) < 0) {
                mail_index_file_set_syscall_error(index,
                    file->filepath, "madvise()");
            }
        }

        file->buffer = buffer_create_const_data(default_pool,
                            file->mmap_base,
                            file->mmap_size);
        file->buffer_offset = 0;

        if (mail_transaction_log_file_sync(file) < 0)
            return -1;
    }

    if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {
        mail_transaction_log_file_set_corrupted(file,
            "end_offset (%"PRIuUOFF_T") > current sync_offset "
            "(%"PRIuUOFF_T")", end_offset, file->sync_offset);
        return -1;
    }

    return 1;
}

int mail_transaction_log_lock_head(struct mail_transaction_log *log)
{
    struct mail_transaction_log_file *file;
    int ret = 0;

    /* we want to get the head file locked. this is a bit racy,
       since by the time we have it locked a new log file may have been
       created.

       creating new log file requires locking the head file, so if we
       can lock it and don't see another file, we can be sure no-one is
       creating a new log at the moment */

    for (;;) {
        file = log->head;
        if (mail_transaction_log_file_lock(file) < 0)
            return -1;

        file->refcount++;
        ret = mail_transaction_log_refresh(log, TRUE);
        if (--file->refcount == 0) {
            mail_transaction_logs_clean(log);
            file = NULL;
        }

        if (ret == 0 && log->head == file) {
            /* success */
            break;
        }

        if (file != NULL)
            mail_transaction_log_file_unlock(file);

        if (ret < 0)
            break;

        /* try again */
    }

    return ret;
}

int mail_transaction_log_sync_lock(struct mail_transaction_log *log,
                   uint32_t *file_seq_r, uoff_t *file_offset_r)
{
    i_assert(!log->index->log_locked);

    if (mail_transaction_log_lock_head(log) < 0)
        return -1;

    /* update sync_offset */
    if (mail_transaction_log_file_map(log->head, log->head->sync_offset,
                      (uoff_t)-1) < 0) {
        mail_transaction_log_file_unlock(log->head);
        return -1;
    }

    log->index->log_locked = TRUE;
    *file_seq_r = log->head->hdr.file_seq;
    *file_offset_r = log->head->sync_offset;
    return 0;
}

void mail_transaction_log_sync_unlock(struct mail_transaction_log *log)
{
    i_assert(log->index->log_locked);

    log->index->log_locked = FALSE;
    mail_transaction_log_file_unlock(log->head);
}

void mail_transaction_log_get_head(struct mail_transaction_log *log,
                   uint32_t *file_seq_r, uoff_t *file_offset_r)
{
    i_assert(log->index->log_locked);

    *file_seq_r = log->head->hdr.file_seq;
    *file_offset_r = log->head->sync_offset;
}

bool mail_transaction_log_is_head_prev(struct mail_transaction_log *log,
                       uint32_t file_seq, uoff_t file_offset)
{
    return log->head->hdr.prev_file_seq == file_seq &&
        log->head->hdr.prev_file_offset == file_offset;
}