src/lib-index/mail-transaction-log.c

	mail-transaction-log.c revision fe5799cb926d65e895d7916d1ecacf5d09ac11e6
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2003-2017 Dovecot authors, see the included COPYING file */
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include "lib.h"
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include "ioloop.h"
bdd36cfdba3ff66d25570a9ff568d69e1eb543cfTimo Sirainen#include "buffer.h"
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include "file-dotlock.h"
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include "nfs-workarounds.h"
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include "mmap-util.h"
633a1bf10559b9a8d85bfab662fa2a7ba97d45baAki Tuomi#include "mail-index-private.h"
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include "mail-transaction-log-private.h"
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include <stddef.h>
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include <stdio.h>
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk#include <sys/stat.h>
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volkstatic void
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volkmail_transaction_log_set_head(struct mail_transaction_log *log,
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk                  struct mail_transaction_log_file *file)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk{
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_assert(log->head != file);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    file->refcount++;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log->head = file;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_assert(log->files != NULL);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_assert(log->files->next != NULL || log->files == file);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk}
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volkstruct mail_transaction_log *
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainenmail_transaction_log_alloc(struct mail_index *index)
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi{
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    struct mail_transaction_log *log;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log = i_new(struct mail_transaction_log, 1);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log->index = index;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    return log;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk}
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volkstatic void mail_transaction_log_2_unlink_old(struct mail_transaction_log *log)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk{
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    struct stat st;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    uint32_t log2_rotate_time = log->index->map->hdr.log2_rotate_time;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (MAIL_INDEX_IS_IN_MEMORY(log->index))
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        return;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (log2_rotate_time == 0) {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        if (nfs_safe_stat(log->filepath2, &st) == 0)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk            log2_rotate_time = st.st_mtime;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        else if (errno == ENOENT)
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen            log2_rotate_time = (uint32_t)-1;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        else {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk            mail_index_set_error(log->index,
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk                "stat(%s) failed: %m", log->filepath2);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk            return;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (log2_rotate_time != (uint32_t)-1 &&
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        ioloop_time - (time_t)log2_rotate_time >= (time_t)log->index->log_rotate_log2_stale_secs &&
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        !log->index->readonly) {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        i_unlink_if_exists(log->filepath2);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        log2_rotate_time = (uint32_t)-1;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (log2_rotate_time != log->index->map->hdr.log2_rotate_time) {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        /* Write this as part of the next sync's transaction. We're
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk           here because we're already opening a sync lock, so it'll
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk           always happen. It's also required especially with mdbox map
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk           index, which doesn't like changes done outside syncing. */
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        log->index->pending_log2_rotate_time = log2_rotate_time;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk}
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volkint mail_transaction_log_open(struct mail_transaction_log *log)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk{
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    struct mail_transaction_log_file *file;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    const char *reason;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    int ret;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_free(log->filepath);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_free(log->filepath2);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log->filepath = i_strconcat(log->index->filepath,
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk                    MAIL_TRANSACTION_LOG_SUFFIX, NULL);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log->filepath2 = i_strconcat(log->filepath, ".2", NULL);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    /* these settings aren't available at alloc() time, so we need to
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk       set them here: */
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log->nfs_flush =
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        (log->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (log->open_file != NULL)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        mail_transaction_log_file_free(&log->open_file);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (MAIL_INDEX_IS_IN_MEMORY(log->index))
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        return 0;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    file = mail_transaction_log_file_alloc(log, log->filepath);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if ((ret = mail_transaction_log_file_open(file, &reason)) <= 0) {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        /* leave the file for _create() */
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        log->open_file = file;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        return ret;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    mail_transaction_log_set_head(log, file);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    return 1;
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi}
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainenint mail_transaction_log_create(struct mail_transaction_log *log, bool reset)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk{
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    struct mail_transaction_log_file *file;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    if (MAIL_INDEX_IS_IN_MEMORY(log->index)) {
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        file = mail_transaction_log_file_alloc_in_memory(log);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        mail_transaction_log_set_head(log, file);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        return 0;
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    }
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    file = mail_transaction_log_file_alloc(log, log->filepath);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    if (log->open_file != NULL) {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        /* remember what file we tried to open. if someone else created
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk           a new file, use it instead of recreating it */
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        file->st_ino = log->open_file->st_ino;
633a1bf10559b9a8d85bfab662fa2a7ba97d45baAki Tuomi        file->st_dev = log->open_file->st_dev;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        file->last_size = log->open_file->last_size;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        file->last_mtime = log->open_file->last_mtime;
633a1bf10559b9a8d85bfab662fa2a7ba97d45baAki Tuomi        mail_transaction_log_file_free(&log->open_file);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    }
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    if (mail_transaction_log_file_create(file, reset) < 0) {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        mail_transaction_log_file_free(&file);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        return -1;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    mail_transaction_log_set_head(log, file);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    return 1;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk}
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volkvoid mail_transaction_log_close(struct mail_transaction_log *log)
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen{
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen    i_assert(log->views == NULL);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (log->open_file != NULL)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        mail_transaction_log_file_free(&log->open_file);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (log->head != NULL)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        log->head->refcount--;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen    mail_transaction_logs_clean(log);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_assert(log->files == NULL);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk}
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volkvoid mail_transaction_log_free(struct mail_transaction_log **_log)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk{
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    struct mail_transaction_log *log = *_log;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    *_log = NULL;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    mail_transaction_log_close(log);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log->index->log = NULL;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen    i_free(log->filepath);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_free(log->filepath2);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_free(log);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk}
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volkint mail_transaction_log_move_to_memory(struct mail_transaction_log *log)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk{
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    struct mail_transaction_log_file *file;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (!log->index->initial_mapped && log->files != NULL &&
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        log->files->hdr.prev_file_seq != 0) {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        /* we couldn't read dovecot.index and we don't have the first
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk           .log file, so just start from scratch */
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        mail_transaction_log_close(log);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_free(log->filepath);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_free(log->filepath2);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log->filepath = i_strconcat(log->index->filepath,
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi                    MAIL_TRANSACTION_LOG_SUFFIX, NULL);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    log->filepath2 = i_strconcat(log->filepath, ".2", NULL);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    if (log->head != NULL)
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        return mail_transaction_log_file_move_to_memory(log->head);
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    else {
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        file = mail_transaction_log_file_alloc_in_memory(log);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        mail_transaction_log_set_head(log, file);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        return 0;
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    }
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi}
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomivoid mail_transaction_log_indexid_changed(struct mail_transaction_log *log)
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi{
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    struct mail_transaction_log_file *file;
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    mail_transaction_logs_clean(log);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    for (file = log->files; file != NULL; file = file->next) {
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        if (file->hdr.indexid != log->index->indexid) {
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi            mail_transaction_log_file_set_corrupted(file,
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi                "indexid changed: %u -> %u",
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi                file->hdr.indexid, log->index->indexid);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        }
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen    if (log->head != NULL &&
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        log->head->hdr.indexid != log->index->indexid) {
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        if (--log->head->refcount == 0)
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi            mail_transaction_log_file_free(&log->head);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        (void)mail_transaction_log_create(log, FALSE);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    }
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen}
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
633a1bf10559b9a8d85bfab662fa2a7ba97d45baAki Tuomivoid mail_transaction_logs_clean(struct mail_transaction_log *log)
633a1bf10559b9a8d85bfab662fa2a7ba97d45baAki Tuomi{
633a1bf10559b9a8d85bfab662fa2a7ba97d45baAki Tuomi    struct mail_transaction_log_file *file, *next;
633a1bf10559b9a8d85bfab662fa2a7ba97d45baAki Tuomi
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    /* remove only files from the beginning. this way if a view has
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk       referenced an old file, it can still find the new files even if
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk       there aren't any references to it currently. */
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    for (file = log->files; file != NULL; file = next) {
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        next = file->next;
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk        i_assert(file->refcount >= 0);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        if (file->refcount > 0)
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi            break;
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        mail_transaction_log_file_free(&file);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    }
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    /* sanity check: we shouldn't have locked refcount=0 files */
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    for (; file != NULL; file = file->next) {
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi        i_assert(!file->locked || file->refcount > 0);
6eb1a7a7ae2c1dfff6731956ade08f9a4a7c791aAki Tuomi    }
381daab1e3b56a0bc94d2191cf62beba0df51af9Pascal Volk    i_assert(log->head == NULL || log->files != NULL);
}

bool mail_transaction_log_want_rotate(struct mail_transaction_log *log)
{
    struct mail_transaction_log_file *file = log->head;

    if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION ||
        (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
         file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) {
        /* upgrade immediately to a new log file format */
        return TRUE;
    }

    if (file->sync_offset > log->index->log_rotate_max_size) {
        /* file is too large, definitely rotate */
        return TRUE;
    }
    if (file->sync_offset < log->index->log_rotate_min_size) {
        /* file is still too small */
        return FALSE;
    }
    /* rotate if the timestamp is old enough */
    return file->hdr.create_stamp <
        ioloop_time - log->index->log_rotate_min_created_ago_secs;
}

int mail_transaction_log_rotate(struct mail_transaction_log *log, bool reset)
{
    struct mail_transaction_log_file *file;
    const char *path = log->head->filepath;
    struct stat st;
    int ret;

    i_assert(log->head->locked);

    if (MAIL_INDEX_IS_IN_MEMORY(log->index)) {
        file = mail_transaction_log_file_alloc_in_memory(log);
        if (reset) {
            file->hdr.prev_file_seq = 0;
            file->hdr.prev_file_offset = 0;
        }
    } else {
                /* we're locked, we shouldn't need to worry about ESTALE
                   problems in here. */
        if (fstat(log->head->fd, &st) < 0) {
            mail_index_file_set_syscall_error(log->index,
                log->head->filepath, "fstat()");
            return -1;
        }

        file = mail_transaction_log_file_alloc(log, path);

        file->st_dev = st.st_dev;
        file->st_ino = st.st_ino;
        file->last_mtime = st.st_mtime;
        file->last_size = st.st_size;

        if ((ret = mail_transaction_log_file_create(file, reset)) < 0) {
            mail_transaction_log_file_free(&file);
            return -1;
        }
        if (ret == 0) {
            mail_index_set_error(log->index,
                "Transaction log %s was recreated while we had it locked - "
                "locking is broken (lock_method=%s)", path,
                file_lock_method_to_str(log->index->lock_method));
            mail_transaction_log_file_free(&file);
            return -1;
        }
        i_assert(file->locked);
    }

    if (--log->head->refcount == 0)
        mail_transaction_logs_clean(log);
    else {
        /* the newly created log file is already locked */
        mail_transaction_log_file_unlock(log->head,
            !log->index->log_sync_locked ? "rotating" :
            "rotating while syncing");
    }
    mail_transaction_log_set_head(log, file);
    return 0;
}

static int
mail_transaction_log_refresh(struct mail_transaction_log *log, bool nfs_flush,
                 const char **reason_r)
{
        struct mail_transaction_log_file *file;
    struct stat st;

    i_assert(log->head != NULL);

    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(log->head)) {
        *reason_r = "Log is in memory";
        return 0;
    }

    if (nfs_flush && log->nfs_flush)
        nfs_flush_file_handle_cache(log->filepath);
    if (nfs_safe_stat(log->filepath, &st) < 0) {
        if (errno != ENOENT) {
            mail_index_file_set_syscall_error(log->index,
                              log->filepath,
                              "stat()");
            *reason_r = t_strdup_printf("stat(%s) failed: %m", log->filepath);
            return -1;
        }
        /* We shouldn't lose dovecot.index.log unless the mailbox was
           deleted or renamed. Just fail this and let the mailbox
           opening code figure out whether to create a new log file
           or not. Anything else can cause unwanted behavior (e.g.
           mailbox deletion not fully finishing due to .nfs* files and
           an IDLEing IMAP process creating the index back here). */
        log->index->index_deleted = TRUE;
        *reason_r = "Trasnaction log lost while it was open";
        return -1;
    } else if (log->head->st_ino == st.st_ino &&
           CMP_DEV_T(log->head->st_dev, st.st_dev)) {
        /* NFS: log files get rotated to .log.2 files instead
           of being unlinked, so we don't bother checking if
           the existing file has already been unlinked here
           (in which case inodes could match but point to
           different files) */
        *reason_r = "Log inode is unchanged";
        return 0;
    }

    file = mail_transaction_log_file_alloc(log, log->filepath);
    if (mail_transaction_log_file_open(file, reason_r) <= 0) {
        mail_transaction_log_file_free(&file);
        return -1;
    }

    i_assert(!file->locked);

    if (--log->head->refcount == 0)
        mail_transaction_logs_clean(log);
    mail_transaction_log_set_head(log, file);
    *reason_r = "Log reopened";
    return 0;
}

void mail_transaction_log_get_mailbox_sync_pos(struct mail_transaction_log *log,
                           uint32_t *file_seq_r,
                           uoff_t *file_offset_r)
{
    *file_seq_r = log->head->hdr.file_seq;
    *file_offset_r = log->head->max_tail_offset;
}

void mail_transaction_log_set_mailbox_sync_pos(struct mail_transaction_log *log,
                           uint32_t file_seq,
                           uoff_t file_offset)
{
    i_assert(file_seq == log->head->hdr.file_seq);
    i_assert(file_offset >= log->head->saved_tail_offset);

    if (file_offset >= log->head->max_tail_offset)
        log->head->max_tail_offset = file_offset;
}

int mail_transaction_log_find_file(struct mail_transaction_log *log,
                   uint32_t file_seq, bool nfs_flush,
                   struct mail_transaction_log_file **file_r,
                   const char **reason_r)
{
    struct mail_transaction_log_file *file;
    const char *reason;
    int ret;

    if (file_seq > log->head->hdr.file_seq) {
        /* see if the .log file has been recreated */
        if (log->head->locked) {
            /* transaction log is locked. there's no way a newer
               file exists. */
            *reason_r = "Log is locked - newer log can't exist";
            return 0;
        }

        if (mail_transaction_log_refresh(log, FALSE, &reason) < 0) {
            *reason_r = reason;
            return -1;
        }
        if (file_seq > log->head->hdr.file_seq) {
            if (!nfs_flush || !log->nfs_flush) {
                *reason_r = t_strdup_printf(
                    "Requested newer log than exists: %s", reason);
                return 0;
            }
            /* try again, this time flush attribute cache */
            if (mail_transaction_log_refresh(log, TRUE, &reason) < 0) {
                *reason_r = t_strdup_printf(
                    "Log refresh with NFS flush failed: %s", reason);
                return -1;
            }
            if (file_seq > log->head->hdr.file_seq) {
                *reason_r = t_strdup_printf(
                    "Requested newer log than exists - "
                    "still after NFS flush: %s", reason);
                return 0;
            }
        }
    }

    for (file = log->files; file != NULL; file = file->next) {
        if (file->hdr.file_seq == file_seq) {
            *file_r = file;
            return 1;
        }
    }

    if (MAIL_INDEX_IS_IN_MEMORY(log->index)) {
        *reason_r = "Logs are only in memory";
        return 0;
    }

    /* see if we have it in log.2 file */
    file = mail_transaction_log_file_alloc(log, log->filepath2);
    if ((ret = mail_transaction_log_file_open(file, reason_r)) <= 0) {
        mail_transaction_log_file_free(&file);
        return ret;
    }

    /* but is it what we expected? */
    if (file->hdr.file_seq != file_seq) {
        *reason_r = t_strdup_printf(".log.2 contains file_seq=%u",
                        file->hdr.file_seq);
        return 0;
    }

    *file_r = file;
    return 1;
}

int mail_transaction_log_lock_head(struct mail_transaction_log *log,
                   const char *lock_reason)
{
    struct mail_transaction_log_file *file;
    time_t lock_wait_started, lock_secs = 0;
    const char *reason;
    int ret = 0;

    /* we want to get the head file locked. this is a bit racy,
       since by the time we have it locked a new log file may have been
       created.

       creating new log file requires locking the head file, so if we
       can lock it and don't see another file, we can be sure no-one is
       creating a new log at the moment */

    lock_wait_started = time(NULL);
    for (;;) {
        file = log->head;
        if (mail_transaction_log_file_lock(file) < 0)
            return -1;

        file->refcount++;
        ret = mail_transaction_log_refresh(log, TRUE, &reason);
        if (--file->refcount == 0) {
            mail_transaction_log_file_unlock(file, t_strdup_printf(
                "trying to lock head for %s", lock_reason));
            mail_transaction_logs_clean(log);
            file = NULL;
        }

        if (ret == 0 && log->head == file) {
            /* success */
            i_assert(file != NULL);
            lock_secs = file->lock_created - lock_wait_started;
            break;
        }

        if (file != NULL) {
            mail_transaction_log_file_unlock(file, t_strdup_printf(
                "trying to lock head for %s", lock_reason));
        }
        if (ret < 0)
            break;

        /* try again */
    }
    if (lock_secs > MAIL_TRANSACTION_LOG_LOCK_WARN_SECS) {
        i_warning("Locking transaction log file %s took %ld seconds (%s)",
              log->head->filepath, (long)lock_secs, lock_reason);
    }

    i_assert(ret < 0 || log->head != NULL);
    return ret;
}

int mail_transaction_log_sync_lock(struct mail_transaction_log *log,
                   const char *lock_reason,
                   uint32_t *file_seq_r, uoff_t *file_offset_r)
{
    const char *reason;

    i_assert(!log->index->log_sync_locked);

    if (!log->log_2_unlink_checked) {
        /* we need to check once in a while if .log.2 should be deleted
           to avoid wasting space on such old files. but we also don't
           want to waste time on checking it when the same mailbox
           gets opened over and over again rapidly (e.g. pop3). so
           do this only when there have actually been some changes
           to mailbox (i.e. when it's being locked here) */
        log->log_2_unlink_checked = TRUE;
        mail_transaction_log_2_unlink_old(log);
    }

    if (mail_transaction_log_lock_head(log, lock_reason) < 0)
        return -1;

    /* update sync_offset */
    if (mail_transaction_log_file_map(log->head, log->head->sync_offset,
                      (uoff_t)-1, &reason) <= 0) {
        mail_index_set_error(log->index,
            "Failed to map transaction log %s at "
            "sync_offset=%"PRIuUOFF_T" after locking: %s",
            log->head->filepath, log->head->sync_offset, reason);
        mail_transaction_log_file_unlock(log->head, t_strdup_printf(
            "%s - map failed", lock_reason));
        return -1;
    }

    log->index->log_sync_locked = TRUE;
    *file_seq_r = log->head->hdr.file_seq;
    *file_offset_r = log->head->sync_offset;
    return 0;
}

void mail_transaction_log_sync_unlock(struct mail_transaction_log *log,
                      const char *lock_reason)
{
    i_assert(log->index->log_sync_locked);

    log->index->log_sync_locked = FALSE;
    mail_transaction_log_file_unlock(log->head, lock_reason);
}

void mail_transaction_log_get_head(struct mail_transaction_log *log,
                   uint32_t *file_seq_r, uoff_t *file_offset_r)
{
    *file_seq_r = log->head->hdr.file_seq;
    *file_offset_r = log->head->sync_offset;
}

void mail_transaction_log_get_tail(struct mail_transaction_log *log,
                   uint32_t *file_seq_r)
{
    struct mail_transaction_log_file *tail, *file = log->files;

    for (tail = file; file->next != NULL; file = file->next) {
        if (file->hdr.file_seq + 1 != file->next->hdr.file_seq)
            tail = file->next;
    }
    *file_seq_r = tail->hdr.file_seq;
}

bool mail_transaction_log_is_head_prev(struct mail_transaction_log *log,
                       uint32_t file_seq, uoff_t file_offset)
{
    return log->head->hdr.prev_file_seq == file_seq &&
        log->head->hdr.prev_file_offset == file_offset;
}

int mail_transaction_log_get_mtime(struct mail_transaction_log *log,
                   time_t *mtime_r)
{
    struct stat st;

    *mtime_r = 0;
    if (stat(log->filepath, &st) < 0) {
        if (errno == ENOENT)
            return 0;

        mail_index_file_set_syscall_error(log->index, log->filepath,
                          "stat()");
        return -1;
    }
    *mtime_r = st.st_mtime;
    return 0;
}

int mail_transaction_log_unlink(struct mail_transaction_log *log)
{
    if (unlink(log->filepath) < 0 &&
        errno != ENOENT && errno != ESTALE) {
        mail_index_file_set_syscall_error(log->index, log->filepath,
                          "unlink()");
        return -1;
    }
    return 0;
}

void mail_transaction_log_get_dotlock_set(struct mail_transaction_log *log,
                      struct dotlock_settings *set_r)
{
    struct mail_index *index = log->index;

    i_zero(set_r);
    set_r->timeout = I_MIN(MAIL_TRANSACTION_LOG_LOCK_TIMEOUT,
                   index->max_lock_timeout_secs);
    set_r->stale_timeout = MAIL_TRANSACTION_LOG_LOCK_CHANGE_TIMEOUT;
    set_r->nfs_flush = (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;
    set_r->use_excl_lock =
        (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
}