mail-transaction-log-file.c revision f34227d18c5458c5a8bfe576ecf8d7bb4e75162e
e59faf65ce864fe95dc00f5d52b8323cdbd0608aTimo Sirainen/* Copyright (c) 2003-2010 Dovecot authors, see the included COPYING file */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen#define MEMORY_LOG_NAME "(in-memory transaction log file)"
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainenlog_file_set_syscall_error(struct mail_transaction_log_file *file,
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return mail_index_file_set_syscall_error(file->log->index,
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainenmail_transaction_log_mark_corrupted(struct mail_transaction_log_file *file)
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen unsigned int offset =
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen offsetof(struct mail_transaction_log_header, indexid);
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file) ||
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen /* indexid=0 marks the log file as corrupted */
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen const char *fmt, ...)
b397665e90fa0fc7c6a9156fdd6cf28b571e8e39Timo Sirainen "Corrupted transaction log file %s seq %u: %s "
b397665e90fa0fc7c6a9156fdd6cf28b571e8e39Timo Sirainen t_strdup_vprintf(fmt, va), file->sync_offset);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_alloc(struct mail_transaction_log *log,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen file = i_new(struct mail_transaction_log_file, 1);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenvoid mail_transaction_log_file_free(struct mail_transaction_log_file **_file)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_transaction_log_file *file = *_file;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainenmail_transaction_log_file_skip_to_head(struct mail_transaction_log_file *file)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen const struct mail_index_modseq_header *modseq_hdr;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen if (map == NULL || file->hdr.file_seq != map->hdr.log_file_seq ||
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* we can get a valid log offset from index file. initialize
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen sync_offset from it so we don't have to read the whole log
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file from beginning. */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen modseq_hdr = mail_index_map_get_modseq_header(map);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen "%s: log_file_head_offset too small",
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen } else if (modseq_hdr == NULL && file->hdr.initial_modseq == 0) {
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* modseqs not used yet */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* highest_modseq not synced, start from beginning */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen } else if (modseq_hdr->log_offset > head_offset) {
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen "%s: modseq_hdr.log_offset too large",
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* start from where we last stopped tracking modseqs */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file->sync_highest_modseq = modseq_hdr->highest_modseq;
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen if (file->hdr.file_seq == log->index->map->hdr.log_file_seq) {
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen file->saved_tail_sync_offset = file->saved_tail_offset;
dffa503fd4ce31334346e539496084c80a2d8d37Timo Sirainen if (file->saved_tail_offset > file->max_tail_offset)
dffa503fd4ce31334346e539496084c80a2d8d37Timo Sirainen file->max_tail_offset = file->saved_tail_offset;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainenmail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* insert it to correct position */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_init_hdr(struct mail_transaction_log *log,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen hdr->hdr_size = sizeof(struct mail_transaction_log_header);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* not creating index - make sure we have latest header */
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen /* if we got here from mapping, the .log file is
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen corrupted. use whatever values we got from index
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen hdr->prev_file_seq = index->map->hdr.log_file_seq;
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen hdr->prev_file_offset = index->map->hdr.log_file_head_offset;
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen hdr->file_seq = index->map->hdr.log_file_seq + 1;
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen mail_index_map_modseq_get_highest(index->map);
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen /* make sure the sequence always increases to avoid crashes
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen later. this catches the buggy case where two processes
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen happen to replace the same log file. */
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen for (file = log->head->next; file != NULL; file = file->next) {
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen if (hdr->file_seq <= log->head->hdr.file_seq) {
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen /* make sure the sequence grows */
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen if (hdr->initial_modseq < log->head->sync_highest_modseq) {
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen /* this should be always up-to-date */
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen hdr->initial_modseq = log->head->sync_highest_modseq;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file = mail_transaction_log_file_alloc(log, MEMORY_LOG_NAME);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen file->buffer = buffer_create_dynamic(default_pool, 4096);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen mail_transaction_log_get_dotlock_set(file->log, &dotlock_set);
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen ret = file_dotlock_create(&dotlock_set, file->filepath, 0,
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen log_file_set_syscall_error(file, "file_dotlock_create()");
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen "Timeout (%us) while waiting for "
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "dotlock for transaction log file %s",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ret = file_dotlock_delete(&file->log->dotlock);
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen log_file_set_syscall_error(file, "file_dotlock_delete()");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "Dotlock was lost for transaction log file %s",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenint mail_transaction_log_file_lock(struct mail_transaction_log_file *file)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return mail_transaction_log_file_dotlock(file);
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen lock_timeout_secs = I_MIN(MAIL_TRANSCATION_LOG_LOCK_TIMEOUT,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen log_file_set_syscall_error(file, "mail_index_wait_lock_fd()");
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen "Timeout (%us) while waiting for lock for "
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen "transaction log file %s",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenvoid mail_transaction_log_file_unlock(struct mail_transaction_log_file *file)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen if (lock_time >= MAIL_TRANSCATION_LOG_LOCK_TIMEOUT) {
d9fda7e3a0fa5551547ac3e3054b837fc77f4bfbTimo Sirainen i_warning("Transaction log file %s was locked for %u seconds",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainenmail_transaction_log_file_read_header(struct mail_transaction_log_file *file)
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen i_assert(file->buffer == NULL && file->mmap_base == NULL);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen if (file->last_size < mmap_get_page_size() && file->last_size > 0) {
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen /* just read the entire transaction log to memory */
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen file->buffer = buffer_create_dynamic(default_pool, 4096);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen dest = buffer_append_space_unsafe(file->buffer, dest_size);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen /* read only the header */
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen /* it's not necessarily an error to read less than wanted header size,
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen since older versions of the log format used smaller headers. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen ret = mail_transaction_log_file_read_header(file);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen /* incompatible version - fix silently */
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen if (ret < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "unexpected end of file while reading header");
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen if (file->hdr.minor_version >= 2 || file->hdr.major_version > 1) {
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen /* we have compatibility flags */
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen enum mail_index_header_compat_flags compat_flags = 0;
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen /* architecture change */
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen "Rebuilding index file %s: "
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen "CPU architecture changed",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "Header size too small");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* @UNSAFE: smaller than we expected - zero out the fields we
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen shouldn't have filled */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* corrupted */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "Transaction log file %s: marked corrupted",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->hdr.indexid != file->log->index->indexid) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* index file was probably just rebuilt and we don't
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen know about it yet */
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen "indexid changed %u -> %u",
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen file->log->index->indexid, file->hdr.indexid);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* creating index file. since transaction log is created
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen first, use the indexid in it to create the main index
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen to avoid races. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen file->log->index->indexid = file->hdr.indexid;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* make sure we already don't have a file with the same sequence
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen opened. it shouldn't happen unless the old log file was
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen corrupted. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (f = file->log->files; f != NULL; f = f->next) {
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* mark the old file corrupted. we can't safely remove
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen it from the list however, so return failure. */
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen if (strcmp(f->filepath, f->log->head->filepath) != 0) {
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen /* only mark .2 corrupted, just to make sure
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen we don't lose any changes from .log in case
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen we're somehow wrong */
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen "Transaction log %s: "
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen "duplicate transaction log sequence (%u)",
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmail_transaction_log_file_stat(struct mail_transaction_log_file *file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmail_transaction_log_file_is_dupe(struct mail_transaction_log_file *file)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (tmp = file->log->files; tmp != NULL; tmp = tmp->next) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_create2(struct mail_transaction_log_file *file,
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen /* although we check also mtime and file size below, it's done
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen only to fix broken log files. we don't bother flushing
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen attribute cache just for that. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* log creation is locked now - see if someone already created it.
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen note that if we're rotating, we need to keep the log locked until
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen the file has been rewritten. and because fcntl() locks are stupid,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if we go and open()+close() the file and we had it already opened,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen its locks are lost. so we use stat() to check if the file has been
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen recreated, although it almost never is. */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen else if (nfs_safe_stat(file->filepath, &st) < 0) {
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "stat()");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* inode/dev checks are enough when we're rotating the file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen but not when we're replacing a broken log file */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* no-one else recreated the file */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* recreated. use the file if its header is ok */
347acd14d8da653ce3757b3e29981326502bed6bTimo Sirainen mail_transaction_log_file_stat(file, FALSE) == 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* yes, it was ok */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mail_transaction_log_init_hdr(file->log, &file->hdr) < 0)
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen /* don't reset modseqs. if we're reseting due to rebuilding
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen indexes we'll probably want to keep uidvalidity and in such
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen cases we really don't want to shrink modseqs. */
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen if (write_full(new_fd, &file->hdr, sizeof(file->hdr)) < 0)
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "write_full()");
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen if (file->log->index->fsync_mode == FSYNC_MODE_ALWAYS) {
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen /* the header isn't important, so don't bother calling
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen fdatasync() unless it's required */
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "fdatasync()");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_transaction_log_file_stat(file, FALSE);
629600d9a85e8025c15a5eaeb80329e116e022c9Timo Sirainen if (file->log->head != NULL && file->log->head->locked) {
629600d9a85e8025c15a5eaeb80329e116e022c9Timo Sirainen /* we'll need to preserve the lock */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* if we return -1 the dotlock deletion code closes the fd */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* keep two log files */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* rename() would be nice and easy way to do this, except then
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen there's a race condition between the rename and
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen file_dotlock_replace(). during that time the log file
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen doesn't exist, which could cause problems. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen path2 = t_strconcat(file->filepath, ".2", NULL);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen mail_index_set_error(index, "unlink(%s) failed: %m",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* try to link() anyway */
9716b2665ee3938d3dfe64bda44d7c3ae3b55d30Timo Sirainen if (nfs_safe_link(file->filepath, path2, FALSE) < 0 &&
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen mail_index_set_error(index, "link(%s, %s) failed: %m",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* ignore the error. we don't care that much about the
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen second log file and we're going to overwrite this
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen first one. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* success */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainenint mail_transaction_log_file_create(struct mail_transaction_log_file *file,
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen "Can't create log file %s: Index is read-only",
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen mail_transaction_log_get_dotlock_set(file->log, &new_dotlock_set);
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen new_dotlock_set.lock_suffix = LOG_NEW_DOTLOCK_SUFFIX;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* With dotlocking we might already have path.lock created, so this
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen filename has to be different. */
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen fd = file_dotlock_open(&new_dotlock_set, file->filepath, 0, &dotlock);
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "file_dotlock_open()");
1f80b32fc28f7a723ff07c1694230a090808b506Timo Sirainen mail_index_fchown(index, fd, file_dotlock_get_lock_path(dotlock));
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* either fd gets used or the dotlock gets deleted and returned fd
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen is for the existing file */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen if (mail_transaction_log_file_create2(file, fd, reset, &dotlock) < 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenint mail_transaction_log_file_open(struct mail_transaction_log_file *file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int i;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0;; i++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file->fd = nfs_safe_open(file->filepath, O_RDWR);
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "open()");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mail_transaction_log_file_stat(file, ignore_estale) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_transaction_log_file_read_hdr(file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* success */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* corrupted */
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen /* don't delete */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "unlink(%s) failed: %m",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* syscall error */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* ESTALE - try again */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenlog_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_transaction_header_update *u = data;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const unsigned int offset_pos =
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen offsetof(struct mail_index_header, log_file_tail_offset);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (size < sizeof(*u) || size < sizeof(*u) + u->size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "header update extends beyond record size");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen u->offset + u->size >= offset_pos + offset_size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),
88b8aea03a24ef7a9efc30399080487b7eb03537Timo Sirainen if (file->sync_offset < file->saved_tail_sync_offset) {
88b8aea03a24ef7a9efc30399080487b7eb03537Timo Sirainen /* saved_tail_offset was already set in header,
88b8aea03a24ef7a9efc30399080487b7eb03537Timo Sirainen but we still had to resync the file to find
88b8aea03a24ef7a9efc30399080487b7eb03537Timo Sirainen modseqs. ignore this record. */
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen "Transaction log file %s seq %u: "
dffa503fd4ce31334346e539496084c80a2d8d37Timo Sirainen "log_file_tail_offset update shrank it "
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainenvoid mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen trans_size = mail_index_offset_to_uint32(hdr->size);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* tracking modseqs */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen } else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* modseqs not tracked yet. see if this is a modseq
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen extension introduction. */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen const struct mail_transaction_ext_intro *intro = data;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen const unsigned int modseq_ext_len =
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* modseq tracking started */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* not tracking modseqs */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen case MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT:
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen case MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT:
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* ignore expunge requests */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* these changes increase modseq */
8bb360f9e5de1c25e4f875205bb06e8bf15dae14Timo Sirainen const struct mail_transaction_modseq_update *rec, *end;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen end = CONST_PTR_OFFSET(data, trans_size - sizeof(*hdr));
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen uint64_t modseq = ((uint64_t)rec->modseq_high32 >> 32) |
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenmodseq_cache_hit(struct mail_transaction_log_file *file, unsigned int idx)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* @UNSAFE: move it to top */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen memmove(file->modseq_cache + 1, file->modseq_cache,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenmodseq_cache_get_offset(struct mail_transaction_log_file *file, uoff_t offset)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen for (i = 0; i < N_ELEMENTS(file->modseq_cache); i++) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* exact cache hit */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenmodseq_cache_get_modseq(struct mail_transaction_log_file *file, uint64_t modseq)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen for (i = 0; i < N_ELEMENTS(file->modseq_cache); i++) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (modseq < file->modseq_cache[i].highest_modseq)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (modseq == file->modseq_cache[i].highest_modseq) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* exact cache hit */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenlog_get_synced_record(struct mail_transaction_log_file *file, uoff_t *offset,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* we've already synced this record at some point. it should
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen trans_size = mail_index_offset_to_uint32(hdr->size);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen *offset - file->buffer_offset + trans_size > file->buffer->used) {
c15f15f71b885415fdaf2395ce52805770148917Timo Sirainen "Transaction log corrupted unexpectedly at "
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenint mail_transaction_log_file_get_highest_modseq_at(
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen *highest_modseq_r = file->sync_highest_modseq;
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen cache = modseq_cache_get_offset(file, offset);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* nothing usable in cache - scan from beginning */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* exact cache hit */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* use cache to skip over some records */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen ret = mail_transaction_log_file_map(file, cur_offset, offset);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen "%s: Transaction log corrupted, can't get modseq",
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen i_assert(cur_offset + file->buffer->used >= offset);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* @UNSAFE: cache the value */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen memmove(file->modseq_cache + 1, file->modseq_cache,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen file->modseq_cache[0].highest_modseq = cur_modseq;
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenint mail_transaction_log_file_get_modseq_next_offset(
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen cache = modseq_cache_get_modseq(file, modseq);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* nothing usable in cache - scan from beginning */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* exact cache hit */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* use cache to skip over some records */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen ret = mail_transaction_log_file_map(file, cur_offset,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen "%s: Transaction log corrupted, can't get modseq",
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* if we got to sync_offset, cur_modseq should be
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen sync_highest_modseq */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen "%s: Transaction log changed unexpectedly, "
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* @UNSAFE: cache the value */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen memmove(file->modseq_cache + 1, file->modseq_cache,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen file->modseq_cache[0].highest_modseq = cur_modseq;
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenlog_file_track_sync(struct mail_transaction_log_file *file,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* external transactions: */
651fc0f1e43fef3e02e0e7b5f498973b05f641d7Timo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* see if this updates mailbox_sync_offset */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen ret = log_file_track_mailbox_sync_offset_hdr(file, data,
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen if (file->sync_offset < file->index_undeleted_offset)
651fc0f1e43fef3e02e0e7b5f498973b05f641d7Timo Sirainen file->index_deleted_offset = file->sync_offset + trans_size;
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen if (file->sync_offset < file->index_deleted_offset)
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen file->log->index->index_delete_requested = FALSE;
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen file->index_undeleted_offset = file->sync_offset + trans_size;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if (file->max_tail_offset == file->sync_offset) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* external transactions aren't synced to mailbox. we can
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen update mailbox sync offset to skip this transaction to
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen avoid re-reading it at the next sync. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_sync(struct mail_transaction_log_file *file)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen i_assert(file->sync_offset >= file->buffer_offset);
57549c98995eb312013f5d98ea6ccf25175a6f18Timo Sirainen if (file->buffer_offset + size < file->sync_offset) {
57549c98995eb312013f5d98ea6ccf25175a6f18Timo Sirainen "log file shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")",
57549c98995eb312013f5d98ea6ccf25175a6f18Timo Sirainen file->buffer_offset + (uoff_t)size, file->sync_offset);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen hdr = CONST_PTR_OFFSET(data, file->sync_offset -
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen trans_size = mail_index_offset_to_uint32(hdr->size);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* unfinished */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (file->sync_offset - file->buffer_offset + trans_size > size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* transaction has been fully written */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (log_file_track_sync(file, hdr, trans_size) < 0)
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen if (file->mmap_base != NULL && !file->locked) {
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen /* Now that all the mmaped pages have page faulted, check if
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen the file had changed while doing that. Only after the last
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen page has faulted, the size returned by fstat() can be
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen trusted. Otherwise it might point to a page boundary while
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen the next page is still being written.
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen Without this check we might see partial transactions,
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen sometimes causing "Extension record updated without intro
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen prefix" errors. */
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "fstat()");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen avail = file->sync_offset - file->buffer_offset;
8c7000574087d5702cc3830e7f80c695ff8e9221Timo Sirainen /* There's more data than we could sync at the moment. If the
8c7000574087d5702cc3830e7f80c695ff8e9221Timo Sirainen last record's size wasn't valid, we can't know if it will
b7c051d9dd7ec0526f3c2c1f09f00d3a61c6576dTimo Sirainen be updated unless we've locked the log. */
b7c051d9dd7ec0526f3c2c1f09f00d3a61c6576dTimo Sirainen /* pread()s or the above fstat() check for mmaps should
b7c051d9dd7ec0526f3c2c1f09f00d3a61c6576dTimo Sirainen have guaranteed that this doesn't happen */
b7c051d9dd7ec0526f3c2c1f09f00d3a61c6576dTimo Sirainen "Unexpected garbage at EOF");
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen /* The size field will be updated soon */
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen mail_index_flush_read_cache(file->log->index, file->filepath,
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen file->hdr.file_seq == file->next->hdr.prev_file_seq &&
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen file->next->hdr.prev_file_offset != file->sync_offset) {
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen "Invalid transaction log size "
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen "(%"PRIuUOFF_T" vs %u): %s", file->sync_offset,
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen file->log->head->hdr.prev_file_offset, file->filepath);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainenmail_transaction_log_file_insert_read(struct mail_transaction_log_file *file,
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen data = buffer_get_space_unsafe(file->buffer, 0, size);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen ret = pread_full(file->fd, data, size, offset);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* success */
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* failure. don't leave ourself to inconsistent state */
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen buffer_set_used_size(file->buffer, file->buffer->used - size);
f87702d8d147f66d3fb6c41e5695c67f6d00612eTimo Sirainen mail_transaction_log_file_set_corrupted(file, "file shrank");
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* log file was deleted in NFS server, fail silently */
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "pread()");
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainenmail_transaction_log_file_read_more(struct mail_transaction_log_file *file)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen read_offset = file->buffer_offset + buffer_get_used_size(file->buffer);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ret = pread(file->fd, data, LOG_PREFETCH, read_offset);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen } while (ret > 0 || (ret < 0 && errno == EINTR));
8c7000574087d5702cc3830e7f80c695ff8e9221Timo Sirainen /* log file was deleted in NFS server, fail silently */
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "pread()");
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainenmail_transaction_log_file_need_nfs_flush(struct mail_transaction_log_file *file)
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen const struct mail_index_header *hdr = &file->log->index->map->hdr;
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen file->hdr.file_seq == file->next->hdr.prev_file_seq &&
d7363f0716a4ca8bf8d9af1fe277113c705739b0Timo Sirainen file->next->hdr.prev_file_offset != max_offset) {
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen /* we already have a newer log file which says that we haven't
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen synced the entire file. */
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen if (file->hdr.file_seq == hdr->log_file_seq &&
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainenmail_transaction_log_file_read(struct mail_transaction_log_file *file,
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen /* NFS: if file isn't locked, we're optimistic that we can read enough
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen data without flushing attribute cache. if after reading we notice
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen that we really should have read more, flush the cache and try again.
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen if file is locked, the attribute cache was already flushed when
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen refreshing the log. */
470365df69b5a0b84146149e3ea1adcb27b1482eTimo Sirainen nfs_flush_attr_cache_unlocked(file->filepath);
470365df69b5a0b84146149e3ea1adcb27b1482eTimo Sirainen nfs_flush_attr_cache_fd_locked(file->filepath,
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen if (file->buffer != NULL && file->buffer_offset > start_offset) {
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen /* we have to insert missing data to beginning of buffer */
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen ret = mail_transaction_log_file_insert_read(file, start_offset);
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen buffer_create_dynamic(default_pool, LOG_PREFETCH);
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen if ((ret = mail_transaction_log_file_read_more(file)) <= 0)
d7363f0716a4ca8bf8d9af1fe277113c705739b0Timo Sirainen mail_transaction_log_file_need_nfs_flush(file)) {
d7363f0716a4ca8bf8d9af1fe277113c705739b0Timo Sirainen /* we didn't read enough data. flush and try again. */
d7363f0716a4ca8bf8d9af1fe277113c705739b0Timo Sirainen return mail_transaction_log_file_read(file, start_offset, TRUE);
c131bf703b200890867f4c3839597ffdc7eba18dTimo Sirainen if ((ret = mail_transaction_log_file_sync(file)) <= 0) {
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen i_assert(ret != 0); /* happens only with mmap */
8c7000574087d5702cc3830e7f80c695ff8e9221Timo Sirainen i_assert(file->sync_offset >= file->buffer_offset);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainenlog_file_map_check_offsets(struct mail_transaction_log_file *file,
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* broken start offset */
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen file->filepath, start_offset, file->sync_offset);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen file->filepath, start_offset, file->sync_offset);
c131bf703b200890867f4c3839597ffdc7eba18dTimo Sirainenmail_transaction_log_file_mmap(struct mail_transaction_log_file *file)
c131bf703b200890867f4c3839597ffdc7eba18dTimo Sirainen /* in case we just switched to mmaping */
c131bf703b200890867f4c3839597ffdc7eba18dTimo Sirainen file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, MAP_SHARED,
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "mmap()");
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen log_file_set_syscall_error(file, "madvise()");
c131bf703b200890867f4c3839597ffdc7eba18dTimo Sirainenmail_transaction_log_file_munmap(struct mail_transaction_log_file *file)
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0)
b251b66e48ed682fa511b9dabc979807fc18f71bTimo Sirainenmail_transaction_log_file_map_mmap(struct mail_transaction_log_file *file,
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen /* we are going to mmap() this file, but it's not necessarily
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen mmaped currently. */
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen i_assert(file->buffer_offset == 0 || file->mmap_base == NULL);
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen i_assert(file->mmap_size == 0 || file->mmap_base != NULL);
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen return log_file_set_syscall_error(file, "fstat()");
a32ac1ad1ccc36d29f30b017197bb2a57feacd1aTimo Sirainen "file size shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")",
7184b2f499d5b8977671e92310148da72170edcaTimo Sirainen if (file->buffer != NULL && file->buffer_offset <= start_offset &&
c026384095b555cc86d032b043d107cc371aacecTimo Sirainen (uoff_t)st.st_size == file->buffer_offset + file->buffer->used) {
c026384095b555cc86d032b043d107cc371aacecTimo Sirainen /* we already have the whole file mapped */
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen if ((ret = mail_transaction_log_file_sync(file)) < 0)
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen /* size changed, re-mmap */
b251b66e48ed682fa511b9dabc979807fc18f71bTimo Sirainen if (file->last_size - start_offset < mmap_get_page_size()) {
b251b66e48ed682fa511b9dabc979807fc18f71bTimo Sirainen /* just reading the file is probably faster */
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen if ((ret = mail_transaction_log_file_sync(file)) < 0)
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen } while (ret == 0);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenint mail_transaction_log_file_map(struct mail_transaction_log_file *file,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* corrupted */
07e80e04c8876b6bf3f95266f48b41e1a681e445Timo Sirainen if (file->locked_sync_offset_updated && file == file->log->head &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* we're not interested of going further than sync_offset */
6f5d19886f36fb20777618ef9362be39d3dc4182Timo Sirainen if (log_file_map_check_offsets(file, start_offset,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->buffer != NULL && file->buffer_offset <= start_offset) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* see if we already have it */
f34227d18c5458c5a8bfe576ecf8d7bb4e75162eTimo Sirainen /* set this only when we've synced to end of file while locked
f34227d18c5458c5a8bfe576ecf8d7bb4e75162eTimo Sirainen (either end_offset=(uoff_t)-1 or we had to read anyway) */
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* we had moved the log to memory but failed to read
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen the beginning of the log file */
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen "%s: Beginning of the log isn't available",
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen return log_file_map_check_offsets(file, start_offset,
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* although we could just skip over the unwanted data, we have
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen to sync everything so that modseqs are calculated
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen if ((file->log->index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0)
b251b66e48ed682fa511b9dabc979807fc18f71bTimo Sirainen ret = mail_transaction_log_file_map_mmap(file, start_offset);
d7363f0716a4ca8bf8d9af1fe277113c705739b0Timo Sirainen ret = mail_transaction_log_file_read(file, start_offset, FALSE);
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen log_file_map_check_offsets(file, start_offset, end_offset);
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainenvoid mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen /* just copy to memory */
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen buf = buffer_create_dynamic(default_pool, file->mmap_size);
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen buffer_append(buf, file->mmap_base, file->mmap_size);
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen /* and lose the mmap */
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0)
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen /* we don't have the full log in the memory. read it. */
d7363f0716a4ca8bf8d9af1fe277113c705739b0Timo Sirainen (void)mail_transaction_log_file_read(file, 0, FALSE);