mail-transaction-log-file.c revision d98109d230575391c1dfe1bea3800d3ff0ac3f11
89a126810703c666309310d0f3189e9834d70b5bTimo Sirainen/* Copyright (c) 2003-2007 Dovecot authors, see the included COPYING file */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "lib.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "ioloop.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "buffer.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "file-dotlock.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "nfs-workarounds.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "read-full.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "write-full.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "mmap-util.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "mail-index-private.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "mail-transaction-log-private.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#define LOG_PREFETCH 1024
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#define MEMORY_LOG_NAME "(in-memory transaction log file)"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenvoid
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen const char *fmt, ...)
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen va_list va;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->corrupted = TRUE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->hdr.indexid = 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* indexid=0 marks the log file as corrupted */
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen if (pwrite_full(file->fd, &file->hdr.indexid,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen sizeof(file->hdr.indexid),
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen offsetof(struct mail_transaction_log_header,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen indexid)) < 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath, "pwrite()");
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen va_start(va, fmt);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen t_push();
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_set_error(file->log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "Corrupted transaction log file %s: %s",
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath, t_strdup_vprintf(fmt, va));
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen t_pop();
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen va_end(va);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstruct mail_transaction_log_file *
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenmail_transaction_log_file_alloc(struct mail_transaction_log *log,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen const char *path)
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_transaction_log_file *file;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file = i_new(struct mail_transaction_log_file, 1);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->log = log;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath = i_strdup(path);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->fd = -1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return file;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenvoid mail_transaction_log_file_free(struct mail_transaction_log_file **_file)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_transaction_log_file *file = *_file;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_transaction_log_file **p;
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen int old_errno = errno;
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen *_file = NULL;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_transaction_log_file_unlock(file);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (*p == file) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen *p = file->next;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen break;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file == file->log->head)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->log->head = NULL;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->buffer != NULL)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen buffer_free(&file->buffer);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->mmap_base != NULL) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "munmap()");
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->fd != -1) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (close(file->fd) < 0) {
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen file->filepath,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "close()");
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen }
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen i_free(file->filepath);
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen i_free(file);
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen errno = old_errno;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainenstatic void
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenmail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen struct mail_transaction_log *log = file->log;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_transaction_log_file **p;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_index_map *map = log->index->map;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (map != NULL && file->hdr.file_seq == map->hdr.log_file_seq &&
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen map->hdr.log_file_head_offset != 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* we can get a valid log offset from index file. initialize
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen sync_offset from it so we don't have to read the whole log
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file from beginning. */
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen if (map->hdr.log_file_head_offset >= file->hdr.hdr_size)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->sync_offset = map->hdr.log_file_head_offset;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen else {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_set_error(log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "%s: log_file_head_offset too small",
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen log->index->filepath);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->sync_offset = file->hdr.hdr_size;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->saved_tail_offset = map->hdr.log_file_tail_offset;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen } else {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->sync_offset = file->hdr.hdr_size;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* insert it to correct position */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen for (p = &log->files; *p != NULL; p = &(*p)->next) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if ((*p)->hdr.file_seq > file->hdr.file_seq)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen break;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen file->next = *p;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen *p = file;
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen}
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstatic int
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenmail_transaction_log_init_hdr(struct mail_transaction_log *log,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_transaction_log_header *hdr)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_index *index = log->index;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen memset(hdr, 0, sizeof(*hdr));
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen hdr->hdr_size = sizeof(struct mail_transaction_log_header);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen hdr->indexid = log->index->indexid;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen hdr->create_stamp = ioloop_time;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (index->fd != -1) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* not creating index - make sure we have latest header */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (!index->mapping) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (mail_index_map(index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return -1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen } else {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* if we got here from mapping, the .log file is
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen corrupted. use whatever values we got from index
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (index->map != NULL) {
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen hdr->prev_file_seq = index->map->hdr.log_file_seq;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen hdr->prev_file_offset = index->map->hdr.log_file_head_offset;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen hdr->file_seq = index->map->hdr.log_file_seq + 1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen } else {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen hdr->file_seq = 1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (log->head != NULL && hdr->file_seq <= log->head->hdr.file_seq) {
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen /* make sure the sequence grows */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen hdr->file_seq = log->head->hdr.file_seq+1;
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstruct mail_transaction_log_file *
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenmail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_transaction_log_file *file;
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen file = mail_transaction_log_file_alloc(log, MEMORY_LOG_NAME);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen i_free(file);
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen return NULL;
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen }
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen file->buffer = buffer_create_dynamic(default_pool, 4096);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->buffer_offset = sizeof(file->hdr);
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen mail_transaction_log_file_add_to_list(file);
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen return file;
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen}
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainenstatic int
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen int ret;
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->log->dotlock_count > 0)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen ret = 1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen else {
064ccad9cea2a75a0d6caa2c679b6cd22be58174Timo Sirainen ret = file_dotlock_create(&file->log->dotlock_settings,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath, 0,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen &file->log->dotlock);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (ret > 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->log->dotlock_count++;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->locked = TRUE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (ret < 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "file_dotlock_create()");
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return -1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen mail_index_set_error(file->log->index,
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen "Timeout while waiting for release of "
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen "dotlock for transaction log file %s",
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen file->filepath);
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen file->log->index->index_lock_timeout = TRUE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return -1;
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen}
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstatic int
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen int ret;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (--file->log->dotlock_count > 0)
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen return 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen ret = file_dotlock_delete(&file->log->dotlock);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (ret < 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath, "file_dotlock_delete()");
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return -1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (ret == 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_set_error(file->log->index,
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen "Dotlock was lost for transaction log file %s",
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return -1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenint mail_transaction_log_file_lock(struct mail_transaction_log_file *file)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen int ret;
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen if (file->locked)
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen return 0;
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
c2f2a6cdf7dd55a4aabe72495cabb4deaa59cffcTimo Sirainen file->locked = TRUE;
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen return 0;
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen }
675b2b1c4587a79478062b05725da61afa5d8045Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return mail_transaction_log_file_dotlock(file);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen i_assert(file->file_lock == NULL);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen F_WRLCK, MAIL_TRANSCATION_LOG_LOCK_TIMEOUT,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen &file->file_lock);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (ret > 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->locked = TRUE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (ret < 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "mail_index_wait_lock_fd()");
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return -1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_set_error(file->log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "Timeout while waiting for lock for transaction log file %s",
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->log->index->index_lock_timeout = TRUE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return -1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenvoid mail_transaction_log_file_unlock(struct mail_transaction_log_file *file)
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (!file->locked)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->locked = FALSE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_transaction_log_file_undotlock(file);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file_unlock(&file->file_lock);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstatic int
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenmail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen bool ignore_estale)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct mail_transaction_log_file *f;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen int ret;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->corrupted)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (ret < 0) {
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen if (errno != ESTALE || !ignore_estale) {
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen file->filepath,
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen "pread_full()");
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen }
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen return -1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (ret == 0) {
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen mail_transaction_log_file_set_corrupted(file,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "unexpected end of file while reading header");
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen }
f3aff3c6bf32d5bb0a61993b76a3fbe5ad798c09Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* incompatible version - fix silently */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_transaction_log_file_set_corrupted(file,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "Header size too small");
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
e161a5225abda0837b5deb8746ef808ba5e98d94Timo Sirainen }
e161a5225abda0837b5deb8746ef808ba5e98d94Timo Sirainen if (file->hdr.hdr_size < sizeof(file->hdr)) {
e161a5225abda0837b5deb8746ef808ba5e98d94Timo Sirainen /* @UNSAFE: smaller than we expected - zero out the fields we
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen shouldn't have filled */
e161a5225abda0837b5deb8746ef808ba5e98d94Timo Sirainen memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen sizeof(file->hdr) - file->hdr.hdr_size);
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen }
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen if (file->hdr.indexid == 0) {
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen /* corrupted */
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen file->corrupted = TRUE;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen mail_index_set_error(file->log->index,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen "Transaction log file %s: marked corrupted",
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen file->filepath);
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen return 0;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->hdr.indexid != file->log->index->indexid) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (file->log->index->indexid != 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* index file was probably just rebuilt and we don't
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen know about it yet */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_transaction_log_file_set_corrupted(file,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "indexid changed %u -> %u",
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->log->index->indexid, file->hdr.indexid);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* creating index file. since transaction log is created
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen first, use the indexid in it to create the main index
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen to avoid races. */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->log->index->indexid = file->hdr.indexid;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* make sure we already don't have a file with the same sequence
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen opened. it shouldn't happen unless the old log file was
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen corrupted. */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen for (f = file->log->files; f != NULL; f = f->next) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (f->hdr.file_seq == file->hdr.file_seq) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen /* mark the old file corrupted. we can't safely remove
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen it from the list however, so return failure. */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_transaction_log_file_set_corrupted(f,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen "duplicate transaction log sequence (%u)",
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen f->hdr.file_seq);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 0;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen }
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen return 1;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstatic int
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainenmail_transaction_log_file_stat(struct mail_transaction_log_file *file,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen bool ignore_estale)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen struct stat st;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (fstat(file->fd, &st) < 0) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen if (errno != ESTALE || !ignore_estale) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen file->filepath, "fstat()");
}
return -1;
}
file->st_dev = st.st_dev;
file->st_ino = st.st_ino;
file->last_mtime = st.st_mtime;
file->last_size = st.st_size;
return 0;
}
static bool
mail_transaction_log_file_is_dupe(struct mail_transaction_log_file *file)
{
struct mail_transaction_log_file *tmp;
for (tmp = file->log->files; tmp != NULL; tmp = tmp->next) {
if (tmp->st_ino == file->st_ino &&
CMP_DEV_T(tmp->st_dev, file->st_dev))
return TRUE;
}
return FALSE;
}
static int
mail_transaction_log_file_create2(struct mail_transaction_log_file *file,
int new_fd, bool reset,
struct dotlock **dotlock)
{
struct mail_index *index = file->log->index;
struct stat st;
const char *path2;
int fd, ret;
bool rename_existing;
if (index->nfs_flush)
nfs_flush_attr_cache(file->filepath, TRUE);
/* log creation is locked now - see if someone already created it.
note that if we're rotating, we need to keep the log locked until
the file has been rewritten. and because fcntl() locks are stupid,
if we go and open()+close() the file and we had it already opened,
its locks are lost. so we use stat() to check if the file has been
recreated, although it almost never is. */
if (reset)
rename_existing = FALSE;
else if (nfs_safe_stat(file->filepath, &st) < 0) {
if (errno != ENOENT) {
mail_index_file_set_syscall_error(index, file->filepath,
"stat()");
return -1;
}
rename_existing = FALSE;
} else if (st.st_ino == file->st_ino &&
CMP_DEV_T(st.st_dev, file->st_dev) &&
/* inode/dev checks are enough when we're rotating the file,
but not when we're replacing a broken log file */
st.st_mtime == file->last_mtime &&
(uoff_t)st.st_size == file->last_size) {
/* no-one else recreated the file */
rename_existing = TRUE;
} else {
/* recreated. use the file if its header is ok */
fd = nfs_safe_open(file->filepath, O_RDWR);
if (fd == -1) {
if (errno != ENOENT) {
mail_index_file_set_syscall_error(index,
file->filepath, "open()");
return -1;
}
} else {
file->fd = fd;
if (mail_transaction_log_file_read_hdr(file,
FALSE) > 0 &&
mail_transaction_log_file_stat(file, FALSE) == 0) {
/* yes, it was ok */
(void)file_dotlock_delete(dotlock);
mail_transaction_log_file_add_to_list(file);
return 0;
}
file->fd = -1;
if (close(fd) < 0) {
mail_index_file_set_syscall_error(index,
file->filepath, "close()");
}
}
rename_existing = FALSE;
}
if (mail_transaction_log_init_hdr(file->log, &file->hdr) < 0)
return -1;
if (reset) {
file->hdr.prev_file_seq = 0;
file->hdr.prev_file_offset = 0;
}
if (write_full(new_fd, &file->hdr, sizeof(file->hdr)) < 0) {
mail_index_file_set_syscall_error(index, file->filepath,
"write_full()");
return -1;
}
if (index->nfs_flush) {
/* the header isn't important, so don't bother calling
fdatasync() unless NFS is used */
if (fdatasync(new_fd) < 0) {
mail_index_file_set_syscall_error(index, file->filepath,
"fdatasync()");
return -1;
}
}
file->fd = new_fd;
ret = mail_transaction_log_file_stat(file, FALSE);
/* if we return -1 the dotlock deletion code closes the fd */
file->fd = -1;
if (ret < 0)
return -1;
/* keep two log files */
if (rename_existing) {
/* rename() would be nice and easy way to do this, except then
there's a race condition between the rename and
file_dotlock_replace(). during that time the log file
doesn't exist, which could cause problems. */
path2 = t_strconcat(file->filepath, ".2", NULL);
if (unlink(path2) < 0 && errno != ENOENT) {
mail_index_set_error(index, "unlink(%s) failed: %m",
path2);
/* try to link() anyway */
}
if (nfs_safe_link(file->filepath, path2) < 0 &&
errno != ENOENT && errno != EEXIST) {
mail_index_set_error(index, "link(%s, %s) failed: %m",
file->filepath, path2);
/* ignore the error. we don't care that much about the
second log file and we're going to overwrite this
first one. */
}
}
if (file_dotlock_replace(dotlock,
DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) <= 0)
return -1;
/* success */
file->fd = new_fd;
mail_transaction_log_file_add_to_list(file);
return 0;
}
int mail_transaction_log_file_create(struct mail_transaction_log_file *file,
bool reset)
{
struct mail_index *index = file->log->index;
struct dotlock *dotlock;
mode_t old_mask;
int fd;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
/* With dotlocking we might already have path.lock created, so this
filename has to be different. */
old_mask = umask(index->mode ^ 0666);
fd = file_dotlock_open(&file->log->new_dotlock_settings,
file->filepath, 0, &dotlock);
umask(old_mask);
if (fd == -1) {
mail_index_file_set_syscall_error(index, file->filepath,
"file_dotlock_open()");
return -1;
}
if (index->gid != (gid_t)-1 &&
fchown(fd, (uid_t)-1, index->gid) < 0) {
mail_index_file_set_syscall_error(index, file->filepath,
"fchown()");
(void)file_dotlock_delete(&dotlock);
return -1;
}
/* either fd gets used or the dotlock gets deleted and returned fd
is for the existing file */
if (mail_transaction_log_file_create2(file, fd, reset, &dotlock) < 0) {
if (dotlock != NULL)
(void)file_dotlock_delete(&dotlock);
return -1;
}
return 0;
}
int mail_transaction_log_file_open(struct mail_transaction_log_file *file,
bool check_existing)
{
unsigned int i;
bool ignore_estale;
int ret;
for (i = 0;; i++) {
file->fd = nfs_safe_open(file->filepath, O_RDWR);
if (file->fd == -1) {
if (errno == ENOENT)
return 0;
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "open()");
return -1;
}
ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
if (mail_transaction_log_file_stat(file, ignore_estale) < 0)
ret = -1;
else if (check_existing &&
mail_transaction_log_file_is_dupe(file))
return 0;
else {
ret = mail_transaction_log_file_read_hdr(file,
ignore_estale);
}
if (ret > 0) {
/* success */
break;
}
if (ret == 0) {
/* corrupted */
if (unlink(file->filepath) < 0 && errno != ENOENT) {
mail_index_set_error(file->log->index,
"unlink(%s) failed: %m",
file->filepath);
}
return 0;
}
if (errno != ESTALE ||
i == MAIL_INDEX_ESTALE_RETRY_COUNT) {
/* syscall error */
return -1;
}
/* ESTALE - try again */
}
mail_transaction_log_file_add_to_list(file);
return 1;
}
static int
log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
const void *data, unsigned int size)
{
const struct mail_transaction_header_update *u = data;
const struct mail_index_header *ihdr;
const unsigned int offset_pos =
offsetof(struct mail_index_header, log_file_tail_offset);
const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);
uint32_t sync_offset;
i_assert(offset_size == sizeof(sync_offset));
if (size < sizeof(*u) || size < sizeof(*u) + u->size) {
mail_transaction_log_file_set_corrupted(file,
"header update extends beyond record size");
return -1;
}
if (u->offset <= offset_pos &&
u->offset + u->size >= offset_pos + offset_size) {
memcpy(&sync_offset,
CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),
sizeof(sync_offset));
if (sync_offset < file->saved_tail_offset) {
mail_transaction_log_file_set_corrupted(file,
"log_file_tail_offset shrank");
return -1;
}
file->saved_tail_offset = sync_offset;
if (sync_offset > file->max_tail_offset)
file->max_tail_offset = sync_offset;
return 1;
}
return 0;
}
static int
log_file_track_mailbox_sync_offset(struct mail_transaction_log_file *file,
const struct mail_transaction_header *hdr,
unsigned int trans_size)
{
int ret;
i_assert((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0);
if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
MAIL_TRANSACTION_HEADER_UPDATE) {
/* see if this updates mailbox_sync_offset */
ret = log_file_track_mailbox_sync_offset_hdr(file, hdr + 1,
trans_size -
sizeof(*hdr));
if (ret != 0)
return ret < 0 ? -1 : 0;
}
if (file->max_tail_offset == file->sync_offset) {
/* external transactions aren't synced to mailbox. we can
update mailbox sync offset to skip this transaction to
avoid re-reading it at the next sync. */
file->max_tail_offset += trans_size;
}
return 0;
}
static int
mail_transaction_log_file_sync(struct mail_transaction_log_file *file)
{
const struct mail_transaction_header *hdr;
const void *data;
struct stat st;
size_t size, avail;
uint32_t trans_size = 0;
data = buffer_get_data(file->buffer, &size);
if (file->sync_offset < file->buffer_offset)
file->sync_offset = file->buffer_offset;
while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {
hdr = CONST_PTR_OFFSET(data, file->sync_offset -
file->buffer_offset);
trans_size = mail_index_offset_to_uint32(hdr->size);
if (trans_size == 0) {
/* unfinished */
return 1;
}
if (trans_size < sizeof(*hdr)) {
mail_transaction_log_file_set_corrupted(file,
"hdr.size too small (%u)", trans_size);
return -1;
}
if (file->sync_offset - file->buffer_offset + trans_size > size)
break;
/* transaction has been fully written */
if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
if (log_file_track_mailbox_sync_offset(file, hdr,
trans_size) < 0)
return -1;
}
file->sync_offset += trans_size;
trans_size = 0;
}
if (file->mmap_base != NULL && !file->locked) {
/* Now that all the mmaped pages have page faulted, check if
the file had changed while doing that. Only after the last
page has faulted, the size returned by fstat() can be
trusted. Otherwise it might point to a page boundary while
the next page is still being written.
Without this check we might see partial transactions,
sometimes causing "Extension record updated without intro
prefix" errors. */
if (fstat(file->fd, &st) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"fstat()");
return -1;
}
if ((uoff_t)st.st_size != file->last_size) {
file->last_size = st.st_size;
return 0;
}
}
avail = file->sync_offset - file->buffer_offset;
if (avail != size) {
/* There's more data than we could sync at the moment. If the
last record's size wasn't valid, we can't know if it will
be updated unless we've locked the log. */
if (trans_size != 0) {
/* pread()s or the above fstat() check for mmaps should
have guaranteed that this doesn't happen */
mail_transaction_log_file_set_corrupted(file,
"hdr.size too large (%u)", trans_size);
return -1;
} else if (file->locked) {
mail_transaction_log_file_set_corrupted(file,
"Unexpected garbage at EOF");
return -1;
}
if (file->log->index->nfs_flush) {
/* The size field will be updated soon */
nfs_flush_read_cache(file->filepath, file->fd,
F_UNLCK, FALSE);
}
}
if (file->next != NULL &&
file->hdr.file_seq == file->next->hdr.prev_file_seq &&
file->next->hdr.prev_file_offset != file->sync_offset) {
mail_index_set_error(file->log->index,
"Invalid transaction log size "
"(%"PRIuUOFF_T" vs %u): %s", file->sync_offset,
file->log->head->hdr.prev_file_offset, file->filepath);
return -1;
}
return 1;
}
static int
mail_transaction_log_file_insert_read(struct mail_transaction_log_file *file,
uoff_t offset)
{
void *data;
size_t size;
ssize_t ret;
size = file->buffer_offset - offset;
buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
data = buffer_get_space_unsafe(file->buffer, 0, size);
ret = pread_full(file->fd, data, size, offset);
if (ret > 0) {
/* success */
file->buffer_offset -= size;
return 1;
}
/* failure. don't leave ourself to inconsistent state */
buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1);
buffer_set_used_size(file->buffer, file->buffer->used - size);
if (ret == 0) {
mail_transaction_log_file_set_corrupted(file, "file shrank");
return 0;
} else if (errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
return 0;
} else {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "pread()");
return -1;
}
}
static int
mail_transaction_log_file_read_more(struct mail_transaction_log_file *file)
{
void *data;
size_t size;
uint32_t read_offset;
ssize_t ret;
read_offset = file->buffer_offset + buffer_get_used_size(file->buffer);
do {
data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);
ret = pread(file->fd, data, LOG_PREFETCH, read_offset);
if (ret > 0)
read_offset += ret;
size = read_offset - file->buffer_offset;
buffer_set_used_size(file->buffer, size);
} while (ret > 0 || (ret < 0 && errno == EINTR));
file->last_size = read_offset;
if (ret < 0) {
if (errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
return 0;
}
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "pread()");
return -1;
}
return 1;
}
static bool
mail_transaction_log_file_need_nfs_flush(struct mail_transaction_log_file *file)
{
const struct mail_index_header *hdr = &file->log->index->map->hdr;
uoff_t max_offset = file->last_size;
if (file->next != NULL &&
file->hdr.file_seq == file->next->hdr.prev_file_seq &&
file->next->hdr.prev_file_offset != max_offset) {
/* we already have a newer log file which says that we haven't
synced the entire file. */
return TRUE;
}
if (file->hdr.file_seq == hdr->log_file_seq &&
max_offset < hdr->log_file_head_offset)
return TRUE;
return FALSE;
}
static int
mail_transaction_log_file_read(struct mail_transaction_log_file *file,
uoff_t start_offset, bool nfs_flush)
{
int ret;
i_assert(file->mmap_base == NULL);
/* NFS: if file isn't locked, we're optimistic that we can read enough
data without flushing attribute cache. if after reading we notice
that we really should have read more, flush the cache and try again.
if file is locked, the attribute cache was already flushed when
refreshing the log. */
if (file->log->index->nfs_flush && nfs_flush)
nfs_flush_attr_cache_fd(file->filepath, file->fd);
if (file->buffer != NULL && file->buffer_offset > start_offset) {
/* we have to insert missing data to beginning of buffer */
ret = mail_transaction_log_file_insert_read(file, start_offset);
if (ret <= 0)
return ret;
}
if (file->buffer == NULL) {
file->buffer =
buffer_create_dynamic(default_pool, LOG_PREFETCH);
file->buffer_offset = start_offset;
}
if ((ret = mail_transaction_log_file_read_more(file)) <= 0)
return ret;
if (file->log->index->nfs_flush && !nfs_flush &&
mail_transaction_log_file_need_nfs_flush(file)) {
/* we didn't read enough data. flush and try again. */
return mail_transaction_log_file_read(file, start_offset, TRUE);
}
if ((ret = mail_transaction_log_file_sync(file)) <= 0) {
i_assert(ret != 0); /* happens only with mmap */
return -1;
}
i_assert(file->sync_offset >= file->buffer_offset);
buffer_set_used_size(file->buffer,
file->sync_offset - file->buffer_offset);
return 1;
}
static int
log_file_map_check_offsets(struct mail_transaction_log_file *file,
uoff_t start_offset, uoff_t end_offset)
{
if (start_offset > file->sync_offset) {
/* broken start offset */
mail_index_set_error(file->log->index,
"%s: start_offset (%"PRIuUOFF_T") > "
"current sync_offset (%"PRIuUOFF_T")",
file->filepath, start_offset, file->sync_offset);
return 0;
}
if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {
mail_index_set_error(file->log->index,
"%s: end_offset (%"PRIuUOFF_T") > "
"current sync_offset (%"PRIuUOFF_T")",
file->filepath, start_offset, file->sync_offset);
return 0;
}
return 1;
}
static int
mail_transaction_log_file_mmap(struct mail_transaction_log_file *file)
{
if (file->buffer != NULL) {
/* in case we just switched to mmaping */
buffer_free(&file->buffer);
}
file->mmap_size = file->last_size;
file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, MAP_SHARED,
file->fd, 0);
if (file->mmap_base == MAP_FAILED) {
file->mmap_base = NULL;
file->mmap_size = 0;
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "mmap()");
return -1;
}
if (file->mmap_size > mmap_get_page_size()) {
if (madvise(file->mmap_base, file->mmap_size,
MADV_SEQUENTIAL) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "madvise()");
}
}
file->buffer = buffer_create_const_data(default_pool,
file->mmap_base,
file->mmap_size);
file->buffer_offset = 0;
return 0;
}
static void
mail_transaction_log_file_munmap(struct mail_transaction_log_file *file)
{
if (file->mmap_base == NULL)
return;
if (munmap(file->mmap_base, file->mmap_size) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "munmap()");
}
file->mmap_base = NULL;
file->mmap_size = 0;
buffer_free(&file->buffer);
}
int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
uoff_t start_offset, uoff_t end_offset)
{
struct mail_index *index = file->log->index;
size_t size;
struct stat st;
int ret;
if (file->hdr.indexid == 0) {
/* corrupted */
return 0;
}
i_assert(start_offset >= file->hdr.hdr_size);
i_assert(start_offset <= end_offset);
if (index->log_locked && file == file->log->head &&
end_offset == (uoff_t)-1) {
/* we're not interested of going further than sync_offset */
if (log_file_map_check_offsets(file, start_offset,
end_offset) == 0)
return 0;
i_assert(start_offset <= file->sync_offset);
end_offset = file->sync_offset;
}
if (file->buffer != NULL && file->buffer_offset <= start_offset) {
/* see if we already have it */
size = buffer_get_used_size(file->buffer);
if (file->buffer_offset + size >= end_offset)
return 1;
}
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
if (start_offset < file->buffer_offset) {
/* we had moved the log to memory but failed to read
the beginning of the log file */
mail_index_set_error(index,
"%s: Beginning of the log isn't available",
file->filepath);
return 0;
}
return log_file_map_check_offsets(file, start_offset,
end_offset);
}
if (!index->mmap_disable) {
/* we are going to mmap() this file, but it's not necessarily
mmaped currently. */
i_assert(file->buffer_offset == 0 || file->mmap_base == NULL);
i_assert(file->mmap_size == 0 || file->mmap_base != NULL);
if (fstat(file->fd, &st) < 0) {
mail_index_file_set_syscall_error(index, file->filepath,
"fstat()");
return -1;
}
file->last_size = st.st_size;
if ((uoff_t)st.st_size < file->sync_offset) {
mail_transaction_log_file_set_corrupted(file,
"file size shrank");
return 0;
}
if ((uoff_t)st.st_size == file->mmap_size) {
/* we already have the whole file mmaped */
if ((ret = mail_transaction_log_file_sync(file)) < 0)
return 0;
if (ret > 0) {
return log_file_map_check_offsets(file,
start_offset,
end_offset);
}
/* size changed, re-mmap */
}
}
if (index->mmap_disable) {
mail_transaction_log_file_munmap(file);
ret = mail_transaction_log_file_read(file, start_offset, FALSE);
if (ret <= 0)
return ret;
} else {
do {
mail_transaction_log_file_munmap(file);
if (mail_transaction_log_file_mmap(file) < 0)
return -1;
if ((ret = mail_transaction_log_file_sync(file)) < 0)
return 0;
} while (ret == 0);
}
return log_file_map_check_offsets(file, start_offset, end_offset);
}
void mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file
*file)
{
buffer_t *buf;
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
return;
if (file->mmap_base != NULL) {
/* just copy to memory */
i_assert(file->buffer_offset == 0);
buf = buffer_create_dynamic(default_pool, file->mmap_size);
buffer_append(buf, file->mmap_base, file->mmap_size);
buffer_free(&file->buffer);
file->buffer = buf;
/* and lose the mmap */
if (munmap(file->mmap_base, file->mmap_size) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"munmap()");
}
file->mmap_base = NULL;
} else if (file->buffer_offset != 0) {
/* we don't have the full log in the memory. read it. */
(void)mail_transaction_log_file_read(file, 0, FALSE);
}
if (close(file->fd) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "close()");
}
file->fd = -1;
}