1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "lib.h"

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "ioloop.h"

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "buffer.h"

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "file-dotlock.h"

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "nfs-workarounds.h"

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "read-full.h"

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "write-full.h"

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "mmap-util.h"

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "mail-index-private.h"

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "mail-transaction-log-private.h"

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#define LOG_PREFETCH 1024

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#define MEMORY_LOG_NAME "(in-memory transaction log file)"

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen const char *fmt, ...)

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen{

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen va_list va;

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen file->corrupted = TRUE;

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen file->hdr.indexid = 0;

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen /* indexid=0 marks the log file as corrupted */

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (pwrite_full(file->fd, &file->hdr.indexid,

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen sizeof(file->hdr.indexid),

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen offsetof(struct mail_transaction_log_header,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen indexid)) < 0) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_index_file_set_syscall_error(file->log->index,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath, "pwrite()");

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen va_start(va, fmt);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen t_push();

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_index_set_error(file->log->index,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen "Corrupted transaction log file %s: %s",

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath, t_strdup_vprintf(fmt, va));

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen t_pop();

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen va_end(va);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstruct mail_transaction_log_file *

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenmail_transaction_log_file_alloc(struct mail_transaction_log *log,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const char *path)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_log_file *file;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file = i_new(struct mail_transaction_log_file, 1);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->log = log;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath = i_strdup(path);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->fd = -1;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return file;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenvoid mail_transaction_log_file_free(struct mail_transaction_log_file **_file)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_log_file *file = *_file;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_log_file **p;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen int old_errno = errno;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen *_file = NULL;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_transaction_log_file_unlock(file);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen for (p = &file->log->files; *p != NULL; p = &(*p)->next) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (*p == file) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen *p = file->next;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen break;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (file == file->log->head)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->log->head = NULL;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (file->buffer != NULL)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_free(&file->buffer);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (file->mmap_base != NULL) {

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0) {

5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->filepath,

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen "munmap()");

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (file->fd != -1) {

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (close(file->fd) < 0) {

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->filepath,

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen "close()");

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen i_free(file->filepath);

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen i_free(file);

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen errno = old_errno;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen}

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainenstatic void

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainenmail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen{

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct mail_transaction_log *log = file->log;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct mail_transaction_log_file **p;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct mail_index_map *map = log->index->map;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (map != NULL && file->hdr.file_seq == map->hdr.log_file_seq &&

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen map->hdr.log_file_head_offset != 0) {

5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen /* we can get a valid log offset from index file. initialize

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (map->hdr.log_file_head_offset >= file->hdr.hdr_size)

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->sync_offset = map->hdr.log_file_head_offset;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen else {

eab880d75fd73a80c7803289796d13e08e4b52cbTimo Sirainen mail_index_set_error(log->index,

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen "%s: log_file_head_offset too small",

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen log->index->filepath);

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->sync_offset = file->hdr.hdr_size;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->saved_tail_offset = map->hdr.log_file_tail_offset;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen } else {

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->sync_offset = file->hdr.hdr_size;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen /* insert it to correct position */

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen for (p = &log->files; *p != NULL; p = &(*p)->next) {

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if ((*p)->hdr.file_seq > file->hdr.file_seq)

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen break;

86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen i_assert((*p)->hdr.file_seq < file->hdr.file_seq);

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->next = *p;

bed3fcccce6a15dfd7491e82df10acd5f5c0bc9bTimo Sirainen *p = file;

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen}

e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainen

e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainenstatic int

e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainenmail_transaction_log_init_hdr(struct mail_transaction_log *log,

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct mail_transaction_log_header *hdr)

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen{

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_index *index = log->index;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen memset(hdr, 0, sizeof(*hdr));

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;

02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr->hdr_size = sizeof(struct mail_transaction_log_header);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr->indexid = log->index->indexid;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr->create_stamp = ioloop_time;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (index->fd != -1) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* not creating index - make sure we have latest header */

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!index->mapping) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (mail_index_map(index,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return -1;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen } else {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* if we got here from mapping, the .log file is

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen }

02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen if (index->map != NULL) {

2738626e69754ba1278cb351c873f4187787fdfcTimo Sirainen hdr->prev_file_seq = index->map->hdr.log_file_seq;

2738626e69754ba1278cb351c873f4187787fdfcTimo Sirainen hdr->prev_file_offset = index->map->hdr.log_file_head_offset;

2738626e69754ba1278cb351c873f4187787fdfcTimo Sirainen hdr->file_seq = index->map->hdr.log_file_seq + 1;

2738626e69754ba1278cb351c873f4187787fdfcTimo Sirainen } else {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr->file_seq = 1;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (log->head != NULL && hdr->file_seq <= log->head->hdr.file_seq) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* make sure the sequence grows */

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr->file_seq = log->head->hdr.file_seq+1;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstruct mail_transaction_log_file *

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenmail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_log_file *file;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file = mail_transaction_log_file_alloc(log, MEMORY_LOG_NAME);

1df526903ed039e8ff966a223c43b8d04eddf3c7Phil Carmody if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_free(file);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return NULL;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->buffer = buffer_create_dynamic(default_pool, 4096);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->buffer_offset = sizeof(file->hdr);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_transaction_log_file_add_to_list(file);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return file;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic int

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen int ret;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (file->log->dotlock_count > 0)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ret = 1;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen else {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ret = file_dotlock_create(&file->log->dotlock_settings,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath, 0,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen &file->log->dotlock);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ret > 0) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->log->dotlock_count++;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->locked = TRUE;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ret < 0) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_index_file_set_syscall_error(file->log->index,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen "file_dotlock_create()");

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return -1;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_index_set_error(file->log->index,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen "Timeout while waiting for release of "

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen "dotlock for transaction log file %s",

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->log->index->index_lock_timeout = TRUE;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return -1;

4b6ec040fd7999b8be22aee9c7b4bbd4b11c0a5fTimo Sirainen}

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic int

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen int ret;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (--file->log->dotlock_count > 0)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ret = file_dotlock_delete(&file->log->dotlock);

4b6ec040fd7999b8be22aee9c7b4bbd4b11c0a5fTimo Sirainen if (ret < 0) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_index_file_set_syscall_error(file->log->index,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath, "file_dotlock_delete()");

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return -1;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

4b6ec040fd7999b8be22aee9c7b4bbd4b11c0a5fTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ret == 0) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_index_set_error(file->log->index,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen "Dotlock was lost for transaction log file %s",

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return -1;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenint mail_transaction_log_file_lock(struct mail_transaction_log_file *file)

4b6ec040fd7999b8be22aee9c7b4bbd4b11c0a5fTimo Sirainen{

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen int ret;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (file->locked)

4b6ec040fd7999b8be22aee9c7b4bbd4b11c0a5fTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->locked = TRUE;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return mail_transaction_log_file_dotlock(file);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_assert(file->file_lock == NULL);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen F_WRLCK, MAIL_TRANSCATION_LOG_LOCK_TIMEOUT,

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen &file->file_lock);

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen if (ret > 0) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->locked = TRUE;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (ret < 0) {

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen file->filepath,

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen "mail_index_wait_lock_fd()");

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen return -1;

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen }

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen mail_index_set_error(file->log->index,

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen "Timeout while waiting for lock for transaction log file %s",

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen file->filepath);

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen file->log->index->index_lock_timeout = TRUE;

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen return -1;

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen}

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainenvoid mail_transaction_log_file_unlock(struct mail_transaction_log_file *file)

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen{

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (!file->locked)

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen return;

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen

3a017aa592823edf0363d77f13458d569637915eTimo Sirainen file->locked = FALSE;

4860b8746b3b7846a9fe65b8c2907ba8aebd422fTimo Sirainen

4860b8746b3b7846a9fe65b8c2907ba8aebd422fTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))

4860b8746b3b7846a9fe65b8c2907ba8aebd422fTimo Sirainen return;

4860b8746b3b7846a9fe65b8c2907ba8aebd422fTimo Sirainen

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {

3a017aa592823edf0363d77f13458d569637915eTimo Sirainen mail_transaction_log_file_undotlock(file);

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen return;

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen }

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen file_unlock(&file->file_lock);

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic int

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainenmail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen bool ignore_estale)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen struct mail_transaction_log_file *f;

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen int ret;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen

e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen if (file->corrupted)

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);

0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (ret < 0) {

ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen if (errno != ESTALE || !ignore_estale) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_index_file_set_syscall_error(file->log->index,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen "pread_full()");

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return -1;

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen }

b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen if (ret == 0) {

54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen mail_transaction_log_file_set_corrupted(file,

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen "unexpected end of file while reading header");

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen return 0;

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen /* incompatible version - fix silently */

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen return 0;

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen }

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {

a42d489d99bcf7d18cfd0352fdf0c9348cb224fcTimo Sirainen mail_transaction_log_file_set_corrupted(file,

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen "Header size too small");

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (file->hdr.hdr_size < sizeof(file->hdr)) {

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* @UNSAFE: smaller than we expected - zero out the fields we

4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,

4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen sizeof(file->hdr) - file->hdr.hdr_size);

4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen }

1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen

if (file->hdr.indexid == 0) {

/* corrupted */

file->corrupted = TRUE;

mail_index_set_error(file->log->index,

"Transaction log file %s: marked corrupted",

file->filepath);

return 0;

}

if (file->hdr.indexid != file->log->index->indexid) {

if (file->log->index->indexid != 0) {

/* index file was probably just rebuilt and we don't

mail_transaction_log_file_set_corrupted(file,

"indexid changed %u -> %u",

file->log->index->indexid, file->hdr.indexid);

return 0;

}

/* creating index file. since transaction log is created

file->log->index->indexid = file->hdr.indexid;

}

/* make sure we already don't have a file with the same sequence

for (f = file->log->files; f != NULL; f = f->next) {

if (f->hdr.file_seq == file->hdr.file_seq) {

/* mark the old file corrupted. we can't safely remove

mail_transaction_log_file_set_corrupted(f,

"duplicate transaction log sequence (%u)",

f->hdr.file_seq);

return 0;

}

}

return 1;

}

static int

mail_transaction_log_file_stat(struct mail_transaction_log_file *file,

bool ignore_estale)

{

struct stat st;

if (fstat(file->fd, &st) < 0) {

if (errno != ESTALE || !ignore_estale) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "fstat()");

}

return -1;

}

file->st_dev = st.st_dev;

file->st_ino = st.st_ino;

file->last_mtime = st.st_mtime;

file->last_size = st.st_size;

return 0;

}

static bool

mail_transaction_log_file_is_dupe(struct mail_transaction_log_file *file)

{

struct mail_transaction_log_file *tmp;

for (tmp = file->log->files; tmp != NULL; tmp = tmp->next) {

if (tmp->st_ino == file->st_ino &&

CMP_DEV_T(tmp->st_dev, file->st_dev))

return TRUE;

}

return FALSE;

}

static int

mail_transaction_log_file_create2(struct mail_transaction_log_file *file,

int new_fd, bool reset,

struct dotlock **dotlock)

{

struct mail_index *index = file->log->index;

struct stat st;

const char *path2;

int fd, ret;

bool rename_existing;

if (index->nfs_flush) {

/* although we check also mtime and file size below, it's done

nfs_flush_file_handle_cache(file->filepath);

}

/* log creation is locked now - see if someone already created it.

if (reset)

rename_existing = FALSE;

else if (nfs_safe_stat(file->filepath, &st) < 0) {

if (errno != ENOENT) {

mail_index_file_set_syscall_error(index, file->filepath,

"stat()");

return -1;

}

rename_existing = FALSE;

} else if (st.st_ino == file->st_ino &&

CMP_DEV_T(st.st_dev, file->st_dev) &&

/* inode/dev checks are enough when we're rotating the file,

st.st_mtime == file->last_mtime &&

(uoff_t)st.st_size == file->last_size) {

/* no-one else recreated the file */

rename_existing = TRUE;

} else {

/* recreated. use the file if its header is ok */

fd = nfs_safe_open(file->filepath, O_RDWR);

if (fd == -1) {

if (errno != ENOENT) {

mail_index_file_set_syscall_error(index,

file->filepath, "open()");

return -1;

}

} else {

file->fd = fd;

if (mail_transaction_log_file_read_hdr(file,

FALSE) > 0 &&

mail_transaction_log_file_stat(file, FALSE) == 0) {

/* yes, it was ok */

(void)file_dotlock_delete(dotlock);

mail_transaction_log_file_add_to_list(file);

return 0;

}

file->fd = -1;

if (close(fd) < 0) {

mail_index_file_set_syscall_error(index,

file->filepath, "close()");

}

}

rename_existing = FALSE;

}

if (mail_transaction_log_init_hdr(file->log, &file->hdr) < 0)

return -1;

if (reset) {

file->hdr.prev_file_seq = 0;

file->hdr.prev_file_offset = 0;

}

if (write_full(new_fd, &file->hdr, sizeof(file->hdr)) < 0) {

mail_index_file_set_syscall_error(index, file->filepath,

"write_full()");

return -1;

}

if (index->nfs_flush) {

/* the header isn't important, so don't bother calling

if (fdatasync(new_fd) < 0) {

mail_index_file_set_syscall_error(index, file->filepath,

"fdatasync()");

return -1;

}

}

file->fd = new_fd;

ret = mail_transaction_log_file_stat(file, FALSE);

/* if we return -1 the dotlock deletion code closes the fd */

file->fd = -1;

if (ret < 0)

return -1;

/* keep two log files */

if (rename_existing) {

/* rename() would be nice and easy way to do this, except then

path2 = t_strconcat(file->filepath, ".2", NULL);

if (unlink(path2) < 0 && errno != ENOENT) {

mail_index_set_error(index, "unlink(%s) failed: %m",

path2);

/* try to link() anyway */

}

if (nfs_safe_link(file->filepath, path2, FALSE) < 0 &&

errno != ENOENT && errno != EEXIST) {

mail_index_set_error(index, "link(%s, %s) failed: %m",

file->filepath, path2);

/* ignore the error. we don't care that much about the

}

}

if (file_dotlock_replace(dotlock,

DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) <= 0)

return -1;

/* success */

file->fd = new_fd;

mail_transaction_log_file_add_to_list(file);

return 0;

}

int mail_transaction_log_file_create(struct mail_transaction_log_file *file,

bool reset)

{

struct mail_index *index = file->log->index;

struct dotlock *dotlock;

mode_t old_mask;

int fd;

i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

/* With dotlocking we might already have path.lock created, so this

old_mask = umask(index->mode ^ 0666);

fd = file_dotlock_open(&file->log->new_dotlock_settings,

file->filepath, 0, &dotlock);

umask(old_mask);

if (fd == -1) {

mail_index_file_set_syscall_error(index, file->filepath,

"file_dotlock_open()");

return -1;

}

if (index->gid != (gid_t)-1 &&

fchown(fd, (uid_t)-1, index->gid) < 0) {

mail_index_file_set_syscall_error(index, file->filepath,

"fchown()");

(void)file_dotlock_delete(&dotlock);

return -1;

}

/* either fd gets used or the dotlock gets deleted and returned fd

if (mail_transaction_log_file_create2(file, fd, reset, &dotlock) < 0) {

if (dotlock != NULL)

(void)file_dotlock_delete(&dotlock);

return -1;

}

return 0;

}

int mail_transaction_log_file_open(struct mail_transaction_log_file *file,

bool check_existing)

{

unsigned int i;

bool ignore_estale;

int ret;

for (i = 0;; i++) {

file->fd = nfs_safe_open(file->filepath, O_RDWR);

if (file->fd == -1) {

if (errno == ENOENT)

return 0;

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "open()");

return -1;

}

ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT;

if (mail_transaction_log_file_stat(file, ignore_estale) < 0)

ret = -1;

else if (check_existing &&

mail_transaction_log_file_is_dupe(file))

return 0;

else {

ret = mail_transaction_log_file_read_hdr(file,

ignore_estale);

}

if (ret > 0) {

/* success */

break;

}

if (ret == 0) {

/* corrupted */

if (unlink(file->filepath) < 0 && errno != ENOENT) {

mail_index_set_error(file->log->index,

"unlink(%s) failed: %m",

file->filepath);

}

return 0;

}

if (errno != ESTALE ||

i == MAIL_INDEX_ESTALE_RETRY_COUNT) {

/* syscall error */

return -1;

}

/* ESTALE - try again */

}

mail_transaction_log_file_add_to_list(file);

return 1;

}

static int

log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,

const void *data, unsigned int size)

{

const struct mail_transaction_header_update *u = data;

const struct mail_index_header *ihdr;

const unsigned int offset_pos =

offsetof(struct mail_index_header, log_file_tail_offset);

const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);

uint32_t sync_offset;

i_assert(offset_size == sizeof(sync_offset));

if (size < sizeof(*u) || size < sizeof(*u) + u->size) {

mail_transaction_log_file_set_corrupted(file,

"header update extends beyond record size");

return -1;

}

if (u->offset <= offset_pos &&

u->offset + u->size >= offset_pos + offset_size) {

memcpy(&sync_offset,

CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),

sizeof(sync_offset));

if (sync_offset < file->saved_tail_offset) {

mail_transaction_log_file_set_corrupted(file,

"log_file_tail_offset shrank");

return -1;

}

file->saved_tail_offset = sync_offset;

if (sync_offset > file->max_tail_offset)

file->max_tail_offset = sync_offset;

return 1;

}

return 0;

}

static int

log_file_track_mailbox_sync_offset(struct mail_transaction_log_file *file,

const struct mail_transaction_header *hdr,

unsigned int trans_size)

{

int ret;

i_assert((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0);

if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==

MAIL_TRANSACTION_HEADER_UPDATE) {

/* see if this updates mailbox_sync_offset */

ret = log_file_track_mailbox_sync_offset_hdr(file, hdr + 1,

trans_size -

sizeof(*hdr));

if (ret != 0)

return ret < 0 ? -1 : 0;

}

if (file->max_tail_offset == file->sync_offset) {

/* external transactions aren't synced to mailbox. we can

file->max_tail_offset += trans_size;

}

return 0;

}

static int

mail_transaction_log_file_sync(struct mail_transaction_log_file *file)

{

const struct mail_transaction_header *hdr;

const void *data;

struct stat st;

size_t size, avail;

uint32_t trans_size = 0;

data = buffer_get_data(file->buffer, &size);

if (file->sync_offset < file->buffer_offset)

file->sync_offset = file->buffer_offset;

while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {

hdr = CONST_PTR_OFFSET(data, file->sync_offset -

file->buffer_offset);

trans_size = mail_index_offset_to_uint32(hdr->size);

if (trans_size == 0) {

/* unfinished */

return 1;

}

if (trans_size < sizeof(*hdr)) {

mail_transaction_log_file_set_corrupted(file,

"hdr.size too small (%u)", trans_size);

return -1;

}

if (file->sync_offset - file->buffer_offset + trans_size > size)

break;

/* transaction has been fully written */

if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {

if (log_file_track_mailbox_sync_offset(file, hdr,

trans_size) < 0)

return -1;

}

file->sync_offset += trans_size;

trans_size = 0;

}

if (file->mmap_base != NULL && !file->locked) {

/* Now that all the mmaped pages have page faulted, check if

if (fstat(file->fd, &st) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath,

"fstat()");

return -1;

}

if ((uoff_t)st.st_size != file->last_size) {

file->last_size = st.st_size;

return 0;

}

}

avail = file->sync_offset - file->buffer_offset;

if (avail != size) {

/* There's more data than we could sync at the moment. If the

if (trans_size != 0) {

/* pread()s or the above fstat() check for mmaps should

mail_transaction_log_file_set_corrupted(file,

"hdr.size too large (%u)", trans_size);

return -1;

} else if (file->locked) {

mail_transaction_log_file_set_corrupted(file,

"Unexpected garbage at EOF");

return -1;

}

/* The size field will be updated soon */

mail_index_flush_read_cache(file->log->index, file->filepath,

file->fd, file->locked);

}

if (file->next != NULL &&

file->hdr.file_seq == file->next->hdr.prev_file_seq &&

file->next->hdr.prev_file_offset != file->sync_offset) {

mail_index_set_error(file->log->index,

"Invalid transaction log size "

"(%"PRIuUOFF_T" vs %u): %s", file->sync_offset,

file->log->head->hdr.prev_file_offset, file->filepath);

return -1;

}

return 1;

}

static int

mail_transaction_log_file_insert_read(struct mail_transaction_log_file *file,

uoff_t offset)

{

void *data;

size_t size;

ssize_t ret;

size = file->buffer_offset - offset;

buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);

data = buffer_get_space_unsafe(file->buffer, 0, size);

ret = pread_full(file->fd, data, size, offset);

if (ret > 0) {

/* success */

file->buffer_offset -= size;

return 1;

}

/* failure. don't leave ourself to inconsistent state */

buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1);

buffer_set_used_size(file->buffer, file->buffer->used - size);

if (ret == 0) {

mail_transaction_log_file_set_corrupted(file, "file shrank");

return 0;

} else if (errno == ESTALE) {

/* log file was deleted in NFS server, fail silently */

return 0;

} else {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "pread()");

return -1;

}

}

static int

mail_transaction_log_file_read_more(struct mail_transaction_log_file *file)

{

void *data;

size_t size;

uint32_t read_offset;

ssize_t ret;

read_offset = file->buffer_offset + buffer_get_used_size(file->buffer);

do {

data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);

ret = pread(file->fd, data, LOG_PREFETCH, read_offset);

if (ret > 0)

read_offset += ret;

size = read_offset - file->buffer_offset;

buffer_set_used_size(file->buffer, size);

} while (ret > 0 || (ret < 0 && errno == EINTR));

file->last_size = read_offset;

if (ret < 0) {

if (errno == ESTALE) {

/* log file was deleted in NFS server, fail silently */

return 0;

}

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "pread()");

return -1;

}

return 1;

}

static bool

mail_transaction_log_file_need_nfs_flush(struct mail_transaction_log_file *file)

{

const struct mail_index_header *hdr = &file->log->index->map->hdr;

uoff_t max_offset = file->last_size;

if (file->next != NULL &&

file->hdr.file_seq == file->next->hdr.prev_file_seq &&

file->next->hdr.prev_file_offset != max_offset) {

/* we already have a newer log file which says that we haven't

return TRUE;

}

if (file->hdr.file_seq == hdr->log_file_seq &&

max_offset < hdr->log_file_head_offset)

return TRUE;

return FALSE;

}

static int

mail_transaction_log_file_read(struct mail_transaction_log_file *file,

uoff_t start_offset, bool nfs_flush)

{

int ret;

i_assert(file->mmap_base == NULL);

/* NFS: if file isn't locked, we're optimistic that we can read enough

if (file->log->index->nfs_flush && nfs_flush) {

if (!file->locked)

nfs_flush_attr_cache_unlocked(file->filepath);

else {

nfs_flush_attr_cache_fd_locked(file->filepath,

file->fd);

}

}

if (file->buffer != NULL && file->buffer_offset > start_offset) {

/* we have to insert missing data to beginning of buffer */

ret = mail_transaction_log_file_insert_read(file, start_offset);

if (ret <= 0)

return ret;

}

if (file->buffer == NULL) {

file->buffer =

buffer_create_dynamic(default_pool, LOG_PREFETCH);

file->buffer_offset = start_offset;

}

if ((ret = mail_transaction_log_file_read_more(file)) <= 0)

return ret;

if (file->log->index->nfs_flush && !nfs_flush &&

mail_transaction_log_file_need_nfs_flush(file)) {

/* we didn't read enough data. flush and try again. */

return mail_transaction_log_file_read(file, start_offset, TRUE);

}

if ((ret = mail_transaction_log_file_sync(file)) <= 0) {

i_assert(ret != 0); /* happens only with mmap */

return -1;

}

i_assert(file->sync_offset >= file->buffer_offset);

buffer_set_used_size(file->buffer,

file->sync_offset - file->buffer_offset);

return 1;

}

static int

log_file_map_check_offsets(struct mail_transaction_log_file *file,

uoff_t start_offset, uoff_t end_offset)

{

if (start_offset > file->sync_offset) {

/* broken start offset */

mail_index_set_error(file->log->index,

"%s: start_offset (%"PRIuUOFF_T") > "

"current sync_offset (%"PRIuUOFF_T")",

file->filepath, start_offset, file->sync_offset);

return 0;

}

if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {

mail_index_set_error(file->log->index,

"%s: end_offset (%"PRIuUOFF_T") > "

"current sync_offset (%"PRIuUOFF_T")",

file->filepath, start_offset, file->sync_offset);

return 0;

}

return 1;

}

static int

mail_transaction_log_file_mmap(struct mail_transaction_log_file *file)

{

if (file->buffer != NULL) {

/* in case we just switched to mmaping */

buffer_free(&file->buffer);

}

file->mmap_size = file->last_size;

file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, MAP_SHARED,

file->fd, 0);

if (file->mmap_base == MAP_FAILED) {

file->mmap_base = NULL;

file->mmap_size = 0;

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "mmap()");

return -1;

}

if (file->mmap_size > mmap_get_page_size()) {

if (madvise(file->mmap_base, file->mmap_size,

MADV_SEQUENTIAL) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "madvise()");

}

}

file->buffer = buffer_create_const_data(default_pool,

file->mmap_base,

file->mmap_size);

file->buffer_offset = 0;

return 0;

}

static void

mail_transaction_log_file_munmap(struct mail_transaction_log_file *file)

{

if (file->mmap_base == NULL)

return;

if (munmap(file->mmap_base, file->mmap_size) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "munmap()");

}

file->mmap_base = NULL;

file->mmap_size = 0;

buffer_free(&file->buffer);

}

static int

mail_transaction_log_file_map_mmap(struct mail_transaction_log_file *file,

uoff_t start_offset)

{

struct stat st;

int ret;

/* we are going to mmap() this file, but it's not necessarily

i_assert(file->buffer_offset == 0 || file->mmap_base == NULL);

i_assert(file->mmap_size == 0 || file->mmap_base != NULL);

if (fstat(file->fd, &st) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "fstat()");

return -1;

}

file->last_size = st.st_size;

if ((uoff_t)st.st_size < file->sync_offset) {

mail_transaction_log_file_set_corrupted(file,

"file size shrank");

return 0;

}

if ((uoff_t)st.st_size == file->mmap_size) {

/* we already have the whole file mmaped */

if ((ret = mail_transaction_log_file_sync(file)) < 0)

return 0;

if (ret > 0)

return 1;

/* size changed, re-mmap */

}

do {

mail_transaction_log_file_munmap(file);

if (file->last_size - start_offset < mmap_get_page_size()) {

/* just reading the file is probably faster */

return mail_transaction_log_file_read(file,

start_offset,

FALSE);

}

if (mail_transaction_log_file_mmap(file) < 0)

return -1;

if ((ret = mail_transaction_log_file_sync(file)) < 0)

return 0;

} while (ret == 0);

return 1;

}

int mail_transaction_log_file_map(struct mail_transaction_log_file *file,

uoff_t start_offset, uoff_t end_offset)

{

struct mail_index *index = file->log->index;

size_t size;

int ret;

if (file->hdr.indexid == 0) {

/* corrupted */

return 0;

}

i_assert(start_offset >= file->hdr.hdr_size);

i_assert(start_offset <= end_offset);

if (index->log_locked && file == file->log->head &&

end_offset == (uoff_t)-1) {

/* we're not interested of going further than sync_offset */

if (log_file_map_check_offsets(file, start_offset,

end_offset) == 0)

return 0;

i_assert(start_offset <= file->sync_offset);

end_offset = file->sync_offset;

}

if (file->buffer != NULL && file->buffer_offset <= start_offset) {

/* see if we already have it */

size = buffer_get_used_size(file->buffer);

if (file->buffer_offset + size >= end_offset)

return 1;

}

if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {

if (start_offset < file->buffer_offset) {

/* we had moved the log to memory but failed to read

mail_index_set_error(index,

"%s: Beginning of the log isn't available",

file->filepath);

return 0;

}

return log_file_map_check_offsets(file, start_offset,

end_offset);

}

if (!index->mmap_disable)

ret = mail_transaction_log_file_map_mmap(file, start_offset);

else {

mail_transaction_log_file_munmap(file);

ret = mail_transaction_log_file_read(file, start_offset, FALSE);

}

return ret <= 0 ? ret :

log_file_map_check_offsets(file, start_offset, end_offset);

}

void mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file

*file)

{

buffer_t *buf;

if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))

return;

if (file->mmap_base != NULL) {

/* just copy to memory */

i_assert(file->buffer_offset == 0);

buf = buffer_create_dynamic(default_pool, file->mmap_size);

buffer_append(buf, file->mmap_base, file->mmap_size);

buffer_free(&file->buffer);

file->buffer = buf;

/* and lose the mmap */

if (munmap(file->mmap_base, file->mmap_size) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath,

"munmap()");

}

file->mmap_base = NULL;

} else if (file->buffer_offset != 0) {

/* we don't have the full log in the memory. read it. */

(void)mail_transaction_log_file_read(file, 0, FALSE);

}

if (close(file->fd) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "close()");

}

file->fd = -1;

}