src/lib-index/mail-transaction-log-file.c

	mail-transaction-log-file.c revision c026384095b555cc86d032b043d107cc371aacec
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2003-2008 Dovecot authors, see the included COPYING file */
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen#include "lib.h"
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen#include "ioloop.h"
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen#include "buffer.h"
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen#include "file-dotlock.h"
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen#include "nfs-workarounds.h"
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen#include "read-full.h"
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen#include "write-full.h"
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen#include "mmap-util.h"
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen#include "mail-index-private.h"
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen#include "mail-transaction-log-private.h"
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen#define LOG_PREFETCH 1024
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen#define MEMORY_LOG_NAME "(in-memory transaction log file)"
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainenvoid
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                    const char *fmt, ...)
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen{
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    va_list va;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen    file->corrupted = TRUE;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    file->hdr.indexid = 0;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        /* indexid=0 marks the log file as corrupted */
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        if (pwrite_full(file->fd, &file->hdr.indexid,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen                sizeof(file->hdr.indexid),
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen                offsetof(struct mail_transaction_log_header,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen                     indexid)) < 0) {
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen            mail_index_file_set_syscall_error(file->log->index,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen                file->filepath, "pwrite()");
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen        }
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    }
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    va_start(va, fmt);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    T_BEGIN {
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen        mail_index_set_error(file->log->index,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen                     "Corrupted transaction log file %s: %s",
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen                     file->filepath, t_strdup_vprintf(fmt, va));
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    } T_END;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    va_end(va);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen}
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainenstruct mail_transaction_log_file *
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainenmail_transaction_log_file_alloc(struct mail_transaction_log *log,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen                const char *path)
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen{
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    struct mail_transaction_log_file *file;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    file = i_new(struct mail_transaction_log_file, 1);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    file->log = log;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    file->filepath = i_strdup(path);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    file->fd = -1;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    return file;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen}
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainenvoid mail_transaction_log_file_free(struct mail_transaction_log_file **_file)
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen{
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    struct mail_transaction_log_file *file = *_file;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    struct mail_transaction_log_file **p;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    int old_errno = errno;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    *_file = NULL;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    mail_transaction_log_file_unlock(file);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen        if (*p == file) {
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen            *p = file->next;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen            break;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen        }
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    }
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    if (file == file->log->head)
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen        file->log->head = NULL;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    if (file->buffer != NULL)
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen        buffer_free(&file->buffer);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen    if (file->mmap_base != NULL) {
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen        if (munmap(file->mmap_base, file->mmap_size) < 0) {
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen            mail_index_file_set_syscall_error(file->log->index,
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen                              file->filepath,
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen                              "munmap()");
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        }
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen    }
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    if (file->fd != -1) {
737561538a2dcdcda948a1da2830a612d8263a23Timo Sirainen        if (close(file->fd) < 0) {
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen            mail_index_file_set_syscall_error(file->log->index,
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen                              file->filepath,
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen                              "close()");
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen        }
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen    }
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen
ecf44c74416ffa4e7c331e49a1e283be6b1aa668Timo Sirainen    i_free(file->filepath);
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen        i_free(file);
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen        errno = old_errno;
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen}
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainenstatic void
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainenmail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen{
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen    struct mail_transaction_log *log = file->log;
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen    struct mail_transaction_log_file **p;
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen    struct mail_index_map *map = log->index->map;
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    if (map != NULL && file->hdr.file_seq == map->hdr.log_file_seq &&
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen        map->hdr.log_file_head_offset != 0) {
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        /* we can get a valid log offset from index file. initialize
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen           sync_offset from it so we don't have to read the whole log
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen           file from beginning. */
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        if (map->hdr.log_file_head_offset >= file->hdr.hdr_size)
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen            file->sync_offset = map->hdr.log_file_head_offset;
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen        else {
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen            mail_index_set_error(log->index,
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen                "%s: log_file_head_offset too small",
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen                log->index->filepath);
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen            file->sync_offset = file->hdr.hdr_size;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        }
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        file->saved_tail_offset = map->hdr.log_file_tail_offset;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    } else {
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        file->sync_offset = file->hdr.hdr_size;
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen    }
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen    /* insert it to correct position */
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen    for (p = &log->files; *p != NULL; p = &(*p)->next) {
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen        if ((*p)->hdr.file_seq > file->hdr.file_seq)
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen            break;
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen        i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen    }
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen    file->next = *p;
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen    *p = file;
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen}
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainenstatic int
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainenmail_transaction_log_init_hdr(struct mail_transaction_log *log,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                  struct mail_transaction_log_header *hdr)
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen{
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen    struct mail_index *index = log->index;
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen
0950aed81d1e9618264e6aa4d214d89e005ec8d6Timo Sirainen    memset(hdr, 0, sizeof(*hdr));
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    hdr->hdr_size = sizeof(struct mail_transaction_log_header);
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    hdr->indexid = log->index->indexid;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    hdr->create_stamp = ioloop_time;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    if (index->fd != -1) {
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen        /* not creating index - make sure we have latest header */
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        if (!index->mapping) {
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen            if (mail_index_map(index,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                       MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                return -1;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        } else {
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen            /* if we got here from mapping, the .log file is
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen               corrupted. use whatever values we got from index
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen               file */
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        }
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    }
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    if (index->map != NULL) {
737561538a2dcdcda948a1da2830a612d8263a23Timo Sirainen        hdr->prev_file_seq = index->map->hdr.log_file_seq;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        hdr->prev_file_offset = index->map->hdr.log_file_head_offset;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        hdr->file_seq = index->map->hdr.log_file_seq + 1;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    } else {
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        hdr->file_seq = 1;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    }
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    if (log->head != NULL && hdr->file_seq <= log->head->hdr.file_seq) {
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen        /* make sure the sequence grows */
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        hdr->file_seq = log->head->hdr.file_seq+1;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    }
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    return 0;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen}
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainenstruct mail_transaction_log_file *
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainenmail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen{
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    struct mail_transaction_log_file *file;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    file = mail_transaction_log_file_alloc(log, MEMORY_LOG_NAME);
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        i_free(file);
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        return NULL;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    }
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen    file->buffer = buffer_create_dynamic(default_pool, 4096);
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen    file->buffer_offset = sizeof(file->hdr);
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    mail_transaction_log_file_add_to_list(file);
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    return file;
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen}
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainenstatic int
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen{
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen    int ret;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    if (file->log->dotlock_count > 0)
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        ret = 1;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    else {
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        ret = file_dotlock_create(&file->log->dotlock_settings,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                      file->filepath, 0,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                      &file->log->dotlock);
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    }
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    if (ret > 0) {
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen        file->log->dotlock_count++;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        file->locked = TRUE;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        return 0;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    }
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    if (ret < 0) {
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                          file->filepath,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                          "file_dotlock_create()");
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        return -1;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    }
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    mail_index_set_error(file->log->index,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                 "Timeout while waiting for "
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                 "dotlock for transaction log file %s",
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen                 file->filepath);
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    file->log->index->index_lock_timeout = TRUE;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    return -1;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen}
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainenstatic int
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen{
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    int ret;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    if (--file->log->dotlock_count > 0)
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        return 0;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    ret = file_dotlock_delete(&file->log->dotlock);
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    if (ret < 0) {
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen            file->filepath, "file_dotlock_delete()");
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen        return -1;
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen    }
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen    if (ret == 0) {
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        mail_index_set_error(file->log->index,
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen            "Dotlock was lost for transaction log file %s",
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen            file->filepath);
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        return -1;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    }
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    return 0;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen}
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainenint mail_transaction_log_file_lock(struct mail_transaction_log_file *file)
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen{
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    int ret;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
fcaf124d4a727424a338cccfd4274c2393818cd3Timo Sirainen    if (file->locked)
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        return 0;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        file->locked = TRUE;
6c00502d4ece417ead501db8f0ee3e8287ba4459Timo Sirainen        return 0;
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    }
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen    if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)
cc5fc3c813c527ca32fa49cb96edf59b859ded20Timo Sirainen        return mail_transaction_log_file_dotlock(file);
643a81fff9003cba13deb49a565a3c8171da524dTimo Sirainen
    i_assert(file->file_lock == NULL);
    ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
                 F_WRLCK, MAIL_TRANSCATION_LOG_LOCK_TIMEOUT,
                 &file->file_lock);
    if (ret > 0) {
        file->locked = TRUE;
        return 0;
    }
    if (ret < 0) {
        mail_index_file_set_syscall_error(file->log->index,
                          file->filepath,
                          "mail_index_wait_lock_fd()");
        return -1;
    }

    mail_index_set_error(file->log->index,
        "Timeout while waiting for lock for transaction log file %s",
        file->filepath);
    file->log->index->index_lock_timeout = TRUE;
    return -1;
}

void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file)
{
    if (!file->locked)
        return;

    file->locked = FALSE;

    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
        return;

    if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
        mail_transaction_log_file_undotlock(file);
        return;
    }

    file_unlock(&file->file_lock);
}

static int
mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
                   bool ignore_estale)
{
        struct mail_transaction_log_file *f;
    int ret;

    i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));

    if (file->corrupted)
        return 0;

    ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);
    if (ret < 0) {
                if (errno != ESTALE || !ignore_estale) {
                        mail_index_file_set_syscall_error(file->log->index,
                                                          file->filepath,
                                                          "pread_full()");
                }
        return -1;
    }
    if (ret == 0) {
        mail_transaction_log_file_set_corrupted(file,
            "unexpected end of file while reading header");
        return 0;
    }

    if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {
        /* incompatible version - fix silently */
        return 0;
    }
    if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
        mail_transaction_log_file_set_corrupted(file,
            "Header size too small");
        return 0;
    }
    if (file->hdr.hdr_size < sizeof(file->hdr)) {
        /* @UNSAFE: smaller than we expected - zero out the fields we
           shouldn't have filled */
        memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
               sizeof(file->hdr) - file->hdr.hdr_size);
    }

    if (file->hdr.indexid == 0) {
        /* corrupted */
        file->corrupted = TRUE;
        mail_index_set_error(file->log->index,
            "Transaction log file %s: marked corrupted",
            file->filepath);
        return 0;
    }
    if (file->hdr.indexid != file->log->index->indexid) {
        if (file->log->index->indexid != 0) {
            /* index file was probably just rebuilt and we don't
               know about it yet */
            mail_transaction_log_file_set_corrupted(file,
                "indexid changed %u -> %u",
                file->log->index->indexid, file->hdr.indexid);
            return 0;
        }

        /* creating index file. since transaction log is created
           first, use the indexid in it to create the main index
           to avoid races. */
        file->log->index->indexid = file->hdr.indexid;
    }

    /* make sure we already don't have a file with the same sequence
       opened. it shouldn't happen unless the old log file was
       corrupted. */
    for (f = file->log->files; f != NULL; f = f->next) {
        if (f->hdr.file_seq == file->hdr.file_seq) {
            /* mark the old file corrupted. we can't safely remove
               it from the list however, so return failure. */
            mail_transaction_log_file_set_corrupted(f,
                "duplicate transaction log sequence (%u)",
                f->hdr.file_seq);
            return 0;
        }
    }

    return 1;
}

static int
mail_transaction_log_file_stat(struct mail_transaction_log_file *file,
                   bool ignore_estale)
{
    struct stat st;

    if (fstat(file->fd, &st) < 0) {
                if (errno != ESTALE || !ignore_estale) {
            mail_index_file_set_syscall_error(file->log->index,
                file->filepath, "fstat()");
                }
        return -1;
    }

    file->st_dev = st.st_dev;
    file->st_ino = st.st_ino;
    file->last_mtime = st.st_mtime;
    file->last_size = st.st_size;
    return 0;
}

static bool
mail_transaction_log_file_is_dupe(struct mail_transaction_log_file *file)
{
    struct mail_transaction_log_file *tmp;

    for (tmp = file->log->files; tmp != NULL; tmp = tmp->next) {
        if (tmp->st_ino == file->st_ino &&
            CMP_DEV_T(tmp->st_dev, file->st_dev))
            return TRUE;
    }
    return FALSE;
}

static int
mail_transaction_log_file_create2(struct mail_transaction_log_file *file,
                  int new_fd, bool reset,
                  struct dotlock **dotlock)
{
    struct mail_index *index = file->log->index;
    struct stat st;
    const char *path2;
    int fd, ret;
    bool rename_existing;

    if (index->nfs_flush) {
        /* although we check also mtime and file size below, it's done
           only to fix broken log files. we don't bother flushing
           attribute cache just for that. */
        nfs_flush_file_handle_cache(file->filepath);
    }

    /* log creation is locked now - see if someone already created it.
       note that if we're rotating, we need to keep the log locked until
       the file has been rewritten. and because fcntl() locks are stupid,
       if we go and open()+close() the file and we had it already opened,
       its locks are lost. so we use stat() to check if the file has been
       recreated, although it almost never is. */
    if (reset)
        rename_existing = FALSE;
    else if (nfs_safe_stat(file->filepath, &st) < 0) {
        if (errno != ENOENT) {
            mail_index_file_set_syscall_error(index, file->filepath,
                              "stat()");
            return -1;
        }
        rename_existing = FALSE;
    } else if (st.st_ino == file->st_ino &&
           CMP_DEV_T(st.st_dev, file->st_dev) &&
           /* inode/dev checks are enough when we're rotating the file,
              but not when we're replacing a broken log file */
           st.st_mtime == file->last_mtime &&
           (uoff_t)st.st_size == file->last_size) {
        /* no-one else recreated the file */
        rename_existing = TRUE;
    } else {
        /* recreated. use the file if its header is ok */
        fd = nfs_safe_open(file->filepath, O_RDWR);
        if (fd == -1) {
            if (errno != ENOENT) {
                mail_index_file_set_syscall_error(index,
                    file->filepath, "open()");
                return -1;
            }
        } else {
            file->fd = fd;
            if (mail_transaction_log_file_read_hdr(file,
                                   FALSE) > 0 &&
                mail_transaction_log_file_stat(file, FALSE) == 0) {
                /* yes, it was ok */
                (void)file_dotlock_delete(dotlock);
                mail_transaction_log_file_add_to_list(file);
                return 0;
            }
            file->fd = -1;
            if (close(fd) < 0) {
                mail_index_file_set_syscall_error(index,
                    file->filepath, "close()");
            }
        }
        rename_existing = FALSE;
    }

    if (mail_transaction_log_init_hdr(file->log, &file->hdr) < 0)
        return -1;

    if (reset) {
        file->hdr.prev_file_seq = 0;
        file->hdr.prev_file_offset = 0;
    }

    if (write_full(new_fd, &file->hdr, sizeof(file->hdr)) < 0) {
        mail_index_file_set_syscall_error(index, file->filepath,
                          "write_full()");
        return -1;
    }

    if (index->nfs_flush) {
        /* the header isn't important, so don't bother calling
           fdatasync() unless NFS is used */
        if (fdatasync(new_fd) < 0) {
            mail_index_file_set_syscall_error(index, file->filepath,
                              "fdatasync()");
            return -1;
        }
    }

    file->fd = new_fd;
    ret = mail_transaction_log_file_stat(file, FALSE);

    if (file->log->head != NULL && file->log->head->locked) {
        /* we'll need to preserve the lock */
        if (mail_transaction_log_file_lock(file) < 0)
            ret = -1;
    }

    /* if we return -1 the dotlock deletion code closes the fd */
    file->fd = -1;
    if (ret < 0)
        return -1;

    /* keep two log files */
    if (rename_existing) {
        /* rename() would be nice and easy way to do this, except then
           there's a race condition between the rename and
           file_dotlock_replace(). during that time the log file
           doesn't exist, which could cause problems. */
        path2 = t_strconcat(file->filepath, ".2", NULL);
        if (unlink(path2) < 0 && errno != ENOENT) {
                        mail_index_set_error(index, "unlink(%s) failed: %m",
                         path2);
            /* try to link() anyway */
        }
        if (nfs_safe_link(file->filepath, path2, FALSE) < 0 &&
            errno != ENOENT && errno != EEXIST) {
                        mail_index_set_error(index, "link(%s, %s) failed: %m",
                         file->filepath, path2);
            /* ignore the error. we don't care that much about the
               second log file and we're going to overwrite this
               first one. */
        }
    }

    if (file_dotlock_replace(dotlock,
                 DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) <= 0)
        return -1;

    /* success */
    file->fd = new_fd;
        mail_transaction_log_file_add_to_list(file);
    return 0;
}

int mail_transaction_log_file_create(struct mail_transaction_log_file *file,
                     bool reset)
{
    struct mail_index *index = file->log->index;
    struct dotlock *dotlock;
    mode_t old_mask;
    int fd;

    i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

    /* With dotlocking we might already have path.lock created, so this
       filename has to be different. */
    old_mask = umask(index->mode ^ 0666);
    fd = file_dotlock_open(&file->log->new_dotlock_settings,
                   file->filepath, 0, &dotlock);
    umask(old_mask);

    if (fd == -1) {
        mail_index_file_set_syscall_error(index, file->filepath,
                          "file_dotlock_open()");
        return -1;
    }

    if (index->gid != (gid_t)-1 &&
        fchown(fd, (uid_t)-1, index->gid) < 0) {
        mail_index_file_set_syscall_error(index, file->filepath,
                          "fchown()");
        (void)file_dotlock_delete(&dotlock);
        return -1;
    }

        /* either fd gets used or the dotlock gets deleted and returned fd
           is for the existing file */
        if (mail_transaction_log_file_create2(file, fd, reset, &dotlock) < 0) {
        if (dotlock != NULL)
            (void)file_dotlock_delete(&dotlock);
        return -1;
    }
    return 0;
}

int mail_transaction_log_file_open(struct mail_transaction_log_file *file,
                   bool check_existing)
{
        unsigned int i;
    bool ignore_estale;
    int ret;

        for (i = 0;; i++) {
                file->fd = nfs_safe_open(file->filepath, O_RDWR);
                if (file->fd == -1) {
            if (errno == ENOENT)
                return 0;

            mail_index_file_set_syscall_error(file->log->index,
                file->filepath, "open()");
            return -1;
                }

        ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
        if (mail_transaction_log_file_stat(file, ignore_estale) < 0)
            ret = -1;
        else if (check_existing &&
             mail_transaction_log_file_is_dupe(file))
            return 0;
        else {
            ret = mail_transaction_log_file_read_hdr(file,
                                 ignore_estale);
        }
        if (ret > 0) {
            /* success */
            break;
        }

        if (ret == 0) {
            /* corrupted */
            if (unlink(file->filepath) < 0 && errno != ENOENT) {
                mail_index_set_error(file->log->index,
                             "unlink(%s) failed: %m",
                             file->filepath);
            }
            return 0;
        }
        if (errno != ESTALE ||
            i == MAIL_INDEX_ESTALE_RETRY_COUNT) {
            /* syscall error */
            return -1;
        }

        /* ESTALE - try again */
        }

    mail_transaction_log_file_add_to_list(file);
    return 1;
}

static int
log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
                       const void *data, unsigned int size)
{
    const struct mail_transaction_header_update *u = data;
    const struct mail_index_header *ihdr;
    const unsigned int offset_pos =
        offsetof(struct mail_index_header, log_file_tail_offset);
    const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);
    uint32_t sync_offset;

    i_assert(offset_size == sizeof(sync_offset));

    if (size < sizeof(*u) || size < sizeof(*u) + u->size) {
        mail_transaction_log_file_set_corrupted(file,
            "header update extends beyond record size");
        return -1;
    }

    if (u->offset <= offset_pos &&
        u->offset + u->size >= offset_pos + offset_size) {
        memcpy(&sync_offset,
               CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),
               sizeof(sync_offset));

        if (sync_offset < file->saved_tail_offset) {
            mail_transaction_log_file_set_corrupted(file,
                "log_file_tail_offset shrank");
            return -1;
        }
        file->saved_tail_offset = sync_offset;
        if (sync_offset > file->max_tail_offset)
            file->max_tail_offset = sync_offset;
        return 1;
    }
    return 0;
}

static int
log_file_track_mailbox_sync_offset(struct mail_transaction_log_file *file,
                   const struct mail_transaction_header *hdr,
                   unsigned int trans_size)
{
    int ret;

    i_assert((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0);

    if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
        MAIL_TRANSACTION_HEADER_UPDATE) {
        /* see if this updates mailbox_sync_offset */
        ret = log_file_track_mailbox_sync_offset_hdr(file, hdr + 1,
                                 trans_size -
                                 sizeof(*hdr));
        if (ret != 0)
            return ret < 0 ? -1 : 0;
    }

    if (file->max_tail_offset == file->sync_offset) {
        /* external transactions aren't synced to mailbox. we can
           update mailbox sync offset to skip this transaction to
           avoid re-reading it at the next sync. */
        file->max_tail_offset += trans_size;
    }
    return 0;
}

static int
mail_transaction_log_file_sync(struct mail_transaction_log_file *file)
{
        const struct mail_transaction_header *hdr;
    const void *data;
    struct stat st;
    size_t size, avail;
    uint32_t trans_size = 0;

    data = buffer_get_data(file->buffer, &size);

    if (file->sync_offset < file->buffer_offset)
        file->sync_offset = file->buffer_offset;

    while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {
        hdr = CONST_PTR_OFFSET(data, file->sync_offset -
                       file->buffer_offset);
        trans_size = mail_index_offset_to_uint32(hdr->size);
        if (trans_size == 0) {
            /* unfinished */
            return 1;
        }
        if (trans_size < sizeof(*hdr)) {
            mail_transaction_log_file_set_corrupted(file,
                "hdr.size too small (%u)", trans_size);
            return -1;
        }

        if (file->sync_offset - file->buffer_offset + trans_size > size)
            break;

        /* transaction has been fully written */
        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
            if (log_file_track_mailbox_sync_offset(file, hdr,
                                   trans_size) < 0)
                return -1;
        }
        file->sync_offset += trans_size;
        trans_size = 0;
    }

    if (file->mmap_base != NULL && !file->locked) {
        /* Now that all the mmaped pages have page faulted, check if
           the file had changed while doing that. Only after the last
           page has faulted, the size returned by fstat() can be
           trusted. Otherwise it might point to a page boundary while
           the next page is still being written.

           Without this check we might see partial transactions,
           sometimes causing "Extension record updated without intro
           prefix" errors. */
        if (fstat(file->fd, &st) < 0) {
            mail_index_file_set_syscall_error(file->log->index,
                              file->filepath,
                              "fstat()");
            return -1;
        }
        if ((uoff_t)st.st_size != file->last_size) {
            file->last_size = st.st_size;
            return 0;
        }
    }

    avail = file->sync_offset - file->buffer_offset;
    if (avail != size) {
        /* There's more data than we could sync at the moment. If the
           last record's size wasn't valid, we can't know if it will
           be updated unless we've locked the log. */
        if (trans_size != 0) {
            /* pread()s or the above fstat() check for mmaps should
               have guaranteed that this doesn't happen */
            mail_transaction_log_file_set_corrupted(file,
                "hdr.size too large (%u)", trans_size);
            return -1;
        } else if (file->locked) {
            mail_transaction_log_file_set_corrupted(file,
                "Unexpected garbage at EOF");
            return -1;
        }

        /* The size field will be updated soon */
        mail_index_flush_read_cache(file->log->index, file->filepath,
                        file->fd, file->locked);
    }

    if (file->next != NULL &&
        file->hdr.file_seq == file->next->hdr.prev_file_seq &&
        file->next->hdr.prev_file_offset != file->sync_offset) {
        mail_index_set_error(file->log->index,
            "Invalid transaction log size "
            "(%"PRIuUOFF_T" vs %u): %s", file->sync_offset,
            file->log->head->hdr.prev_file_offset, file->filepath);
        return -1;
    }

    return 1;
}

static int
mail_transaction_log_file_insert_read(struct mail_transaction_log_file *file,
                      uoff_t offset)
{
    void *data;
    size_t size;
    ssize_t ret;

    size = file->buffer_offset - offset;
    buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);

    data = buffer_get_space_unsafe(file->buffer, 0, size);
    ret = pread_full(file->fd, data, size, offset);
    if (ret > 0) {
        /* success */
        file->buffer_offset -= size;
        return 1;
    }

    /* failure. don't leave ourself to inconsistent state */
    buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1);
    buffer_set_used_size(file->buffer, file->buffer->used - size);

    if (ret == 0) {
        mail_transaction_log_file_set_corrupted(file, "file shrank");
        return 0;
    } else if (errno == ESTALE) {
        /* log file was deleted in NFS server, fail silently */
        return 0;
    } else {
        mail_index_file_set_syscall_error(file->log->index,
                          file->filepath, "pread()");
        return -1;
    }
}

static int
mail_transaction_log_file_read_more(struct mail_transaction_log_file *file)
{
    void *data;
    size_t size;
    uint32_t read_offset;
    ssize_t ret;

    read_offset = file->buffer_offset + buffer_get_used_size(file->buffer);

    do {
        data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);
        ret = pread(file->fd, data, LOG_PREFETCH, read_offset);
        if (ret > 0)
            read_offset += ret;

        size = read_offset - file->buffer_offset;
        buffer_set_used_size(file->buffer, size);
    } while (ret > 0 || (ret < 0 && errno == EINTR));

    file->last_size = read_offset;

    if (ret < 0) {
        if (errno == ESTALE) {
            /* log file was deleted in NFS server, fail silently */
            return 0;
        }

        mail_index_file_set_syscall_error(file->log->index,
                          file->filepath, "pread()");
        return -1;
    }
    return 1;
}

static bool
mail_transaction_log_file_need_nfs_flush(struct mail_transaction_log_file *file)
{
    const struct mail_index_header *hdr = &file->log->index->map->hdr;
    uoff_t max_offset = file->last_size;

    if (file->next != NULL &&
        file->hdr.file_seq == file->next->hdr.prev_file_seq &&
        file->next->hdr.prev_file_offset != max_offset) {
        /* we already have a newer log file which says that we haven't
           synced the entire file. */
        return TRUE;
    }

    if (file->hdr.file_seq == hdr->log_file_seq &&
        max_offset < hdr->log_file_head_offset)
        return TRUE;

    return FALSE;
}

static int
mail_transaction_log_file_read(struct mail_transaction_log_file *file,
                   uoff_t start_offset, bool nfs_flush)
{
    int ret;

    i_assert(file->mmap_base == NULL);

    /* NFS: if file isn't locked, we're optimistic that we can read enough
       data without flushing attribute cache. if after reading we notice
       that we really should have read more, flush the cache and try again.
       if file is locked, the attribute cache was already flushed when
       refreshing the log. */
    if (file->log->index->nfs_flush && nfs_flush) {
        if (!file->locked)
            nfs_flush_attr_cache_unlocked(file->filepath);
        else {
            nfs_flush_attr_cache_fd_locked(file->filepath,
                               file->fd);
        }
    }

    if (file->buffer != NULL && file->buffer_offset > start_offset) {
        /* we have to insert missing data to beginning of buffer */
        ret = mail_transaction_log_file_insert_read(file, start_offset);
        if (ret <= 0)
            return ret;
    }

    if (file->buffer == NULL) {
        file->buffer =
            buffer_create_dynamic(default_pool, LOG_PREFETCH);
        file->buffer_offset = start_offset;
    }

    if ((ret = mail_transaction_log_file_read_more(file)) <= 0)
        return ret;

    if (file->log->index->nfs_flush && !nfs_flush &&
        mail_transaction_log_file_need_nfs_flush(file)) {
        /* we didn't read enough data. flush and try again. */
        return mail_transaction_log_file_read(file, start_offset, TRUE);
    }

    if ((ret = mail_transaction_log_file_sync(file)) <= 0) {
        i_assert(ret != 0); /* happens only with mmap */
        return -1;
    }

    i_assert(file->sync_offset >= file->buffer_offset);
    buffer_set_used_size(file->buffer,
                 file->sync_offset - file->buffer_offset);
    return 1;
}

static int
log_file_map_check_offsets(struct mail_transaction_log_file *file,
               uoff_t start_offset, uoff_t end_offset)
{
    if (start_offset > file->sync_offset) {
        /* broken start offset */
        mail_index_set_error(file->log->index,
            "%s: start_offset (%"PRIuUOFF_T") > "
            "current sync_offset (%"PRIuUOFF_T")",
            file->filepath, start_offset, file->sync_offset);
        return 0;
    }
    if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {
        mail_index_set_error(file->log->index,
            "%s: end_offset (%"PRIuUOFF_T") > "
            "current sync_offset (%"PRIuUOFF_T")",
            file->filepath, start_offset, file->sync_offset);
        return 0;
    }

    return 1;
}

static int
mail_transaction_log_file_mmap(struct mail_transaction_log_file *file)
{
    if (file->buffer != NULL) {
        /* in case we just switched to mmaping */
        buffer_free(&file->buffer);
    }
    file->mmap_size = file->last_size;
    file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, MAP_SHARED,
                   file->fd, 0);
    if (file->mmap_base == MAP_FAILED) {
        file->mmap_base = NULL;
        file->mmap_size = 0;
        mail_index_file_set_syscall_error(file->log->index,
                          file->filepath, "mmap()");
        return -1;
    }

    if (file->mmap_size > mmap_get_page_size()) {
        if (madvise(file->mmap_base, file->mmap_size,
                MADV_SEQUENTIAL) < 0) {
            mail_index_file_set_syscall_error(file->log->index,
                file->filepath, "madvise()");
        }
    }

    file->buffer = buffer_create_const_data(default_pool,
                        file->mmap_base,
                        file->mmap_size);
    file->buffer_offset = 0;
    return 0;
}

static void
mail_transaction_log_file_munmap(struct mail_transaction_log_file *file)
{
    if (file->mmap_base == NULL)
        return;

    if (munmap(file->mmap_base, file->mmap_size) < 0) {
        mail_index_file_set_syscall_error(file->log->index,
                          file->filepath, "munmap()");
    }
    file->mmap_base = NULL;
    file->mmap_size = 0;
    buffer_free(&file->buffer);
}

static int
mail_transaction_log_file_map_mmap(struct mail_transaction_log_file *file,
                   uoff_t start_offset)
{
    struct stat st;
    int ret;

    /* we are going to mmap() this file, but it's not necessarily
       mmaped currently. */
    i_assert(file->buffer_offset == 0 || file->mmap_base == NULL);
    i_assert(file->mmap_size == 0 || file->mmap_base != NULL);

    if (fstat(file->fd, &st) < 0) {
        mail_index_file_set_syscall_error(file->log->index,
                          file->filepath, "fstat()");
        return -1;
    }
    file->last_size = st.st_size;

    if ((uoff_t)st.st_size < file->sync_offset) {
        mail_transaction_log_file_set_corrupted(file,
                            "file size shrank");
        return 0;
    }

    if (file->buffer != NULL &&
        (uoff_t)st.st_size == file->buffer_offset + file->buffer->used) {
        /* we already have the whole file mapped */
        if ((ret = mail_transaction_log_file_sync(file)) < 0)
            return 0;
        if (ret > 0)
            return 1;
        /* size changed, re-mmap */
    }

    do {
        mail_transaction_log_file_munmap(file);

        if (file->last_size - start_offset < mmap_get_page_size()) {
            /* just reading the file is probably faster */
            return mail_transaction_log_file_read(file,
                                  start_offset,
                                  FALSE);
        }

        if (mail_transaction_log_file_mmap(file) < 0)
            return -1;
        if ((ret = mail_transaction_log_file_sync(file)) < 0)
            return 0;
    } while (ret == 0);

    return 1;
}

int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
                  uoff_t start_offset, uoff_t end_offset)
{
    struct mail_index *index = file->log->index;
    size_t size;
    int ret;

    if (file->hdr.indexid == 0) {
        /* corrupted */
        return 0;
    }

    i_assert(start_offset >= file->hdr.hdr_size);
    i_assert(start_offset <= end_offset);

    if (index->log_locked && file == file->log->head &&
        end_offset == (uoff_t)-1) {
        /* we're not interested of going further than sync_offset */
        if (log_file_map_check_offsets(file, start_offset,
                           end_offset) == 0)
            return 0;
        i_assert(start_offset <= file->sync_offset);
        end_offset = file->sync_offset;
    }

    if (file->buffer != NULL && file->buffer_offset <= start_offset) {
        /* see if we already have it */
        size = buffer_get_used_size(file->buffer);
        if (file->buffer_offset + size >= end_offset)
            return 1;
    }

    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
        if (start_offset < file->buffer_offset) {
            /* we had moved the log to memory but failed to read
               the beginning of the log file */
            mail_index_set_error(index,
                "%s: Beginning of the log isn't available",
                file->filepath);
            return 0;
        }
        return log_file_map_check_offsets(file, start_offset,
                          end_offset);
    }

    if (!index->mmap_disable)
        ret = mail_transaction_log_file_map_mmap(file, start_offset);
    else {
        mail_transaction_log_file_munmap(file);
        ret = mail_transaction_log_file_read(file, start_offset, FALSE);
    }

    return ret <= 0 ? ret :
        log_file_map_check_offsets(file, start_offset, end_offset);
}

void mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file
                          *file)
{
    buffer_t *buf;

    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
        return;

    if (file->mmap_base != NULL) {
        /* just copy to memory */
        i_assert(file->buffer_offset == 0);

        buf = buffer_create_dynamic(default_pool, file->mmap_size);
        buffer_append(buf, file->mmap_base, file->mmap_size);
        buffer_free(&file->buffer);
        file->buffer = buf;

        /* and lose the mmap */
        if (munmap(file->mmap_base, file->mmap_size) < 0) {
            mail_index_file_set_syscall_error(file->log->index,
                              file->filepath,
                              "munmap()");
        }
        file->mmap_base = NULL;
    } else if (file->buffer_offset != 0) {
        /* we don't have the full log in the memory. read it. */
        (void)mail_transaction_log_file_read(file, 0, FALSE);
    }

    if (close(file->fd) < 0) {
        mail_index_file_set_syscall_error(file->log->index,
                          file->filepath, "close()");
    }
    file->fd = -1;
}