bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen#define MEMORY_LOG_NAME "(in-memory transaction log file)"
f9f30736bc6e4cf535085824e7e124506fe2dbbbTimo Sirainenmail_transaction_log_file_sync(struct mail_transaction_log_file *file,
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainenlog_file_set_syscall_error(struct mail_transaction_log_file *file,
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen mail_index_file_set_syscall_error(file->log->index,
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainenmail_transaction_log_mark_corrupted(struct mail_transaction_log_file *file)
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen offsetof(struct mail_transaction_log_header, indexid);
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file) ||
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen /* indexid=0 marks the log file as corrupted. we opened the file with
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen O_APPEND, and now we need to drop it for pwrite() to work (at least
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen mail_index_file_set_syscall_error(file->log->index,
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen if (fcntl(file->fd, F_SETFL, flags & ~O_APPEND) < 0) {
e925f007930955f10fa8728509a44f7a53d2b13eTimo Sirainen mail_index_file_set_syscall_error(file->log->index,
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen const char *fmt, ...)
b397665e90fa0fc7c6a9156fdd6cf28b571e8e39Timo Sirainen "Corrupted transaction log file %s seq %u: %s "
b397665e90fa0fc7c6a9156fdd6cf28b571e8e39Timo Sirainen t_strdup_vprintf(fmt, va), file->sync_offset);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_alloc(struct mail_transaction_log *log,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen file = i_new(struct mail_transaction_log_file, 1);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenvoid mail_transaction_log_file_free(struct mail_transaction_log_file **_file)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_transaction_log_file *file = *_file;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainenmail_transaction_log_file_skip_to_head(struct mail_transaction_log_file *file)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen const struct mail_index_modseq_header *modseq_hdr;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen if (map == NULL || file->hdr.file_seq != map->hdr.log_file_seq ||
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* we can get a valid log offset from index file. initialize
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen sync_offset from it so we don't have to read the whole log
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file from beginning. */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen modseq_hdr = mail_index_map_get_modseq_header(map);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen "%s: log_file_head_offset too small",
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen } else if (modseq_hdr == NULL && file->hdr.initial_modseq == 0) {
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* modseqs not used yet */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* highest_modseq not synced, start from beginning */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen } else if (modseq_hdr->log_offset > head_offset) {
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen "%s: modseq_hdr.log_offset too large",
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* start from where we last stopped tracking modseqs */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file->sync_highest_modseq = modseq_hdr->highest_modseq;
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen if (file->hdr.file_seq == log->index->map->hdr.log_file_seq) {
f153a2cec0319f549388d28f8cfd4d50229d1132Timo Sirainen file->saved_tail_sync_offset = file->saved_tail_offset;
dffa503fd4ce31334346e539496084c80a2d8d37Timo Sirainen if (file->saved_tail_offset > file->max_tail_offset)
dffa503fd4ce31334346e539496084c80a2d8d37Timo Sirainen file->max_tail_offset = file->saved_tail_offset;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainenmail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* insert it to correct position */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen /* if we read any unfinished data, make sure the buffer gets
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen truncated. */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen (void)mail_transaction_log_file_sync(file, &retry, &reason);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_init_hdr(struct mail_transaction_log *log,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen hdr->hdr_size = sizeof(struct mail_transaction_log_header);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* not creating index - make sure we have latest header */
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen /* if we got here from mapping, the .log file is
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen corrupted. use whatever values we got from index
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen hdr->prev_file_seq = index->map->hdr.log_file_seq;
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen hdr->prev_file_offset = index->map->hdr.log_file_head_offset;
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen hdr->file_seq = index->map->hdr.log_file_seq + 1;
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen mail_index_map_modseq_get_highest(index->map);
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainen /* modseq tracking in log files is required for many reasons
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainen nowadays, even if per-message modseqs aren't enabled in
1ff03d2244dd08effadfecec365bec28793a1aa0Timo Sirainen dovecot.index. */
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen /* make sure the sequence always increases to avoid crashes
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen later. this catches the buggy case where two processes
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen happen to replace the same log file. */
b5b3b4c9159f506cdfdce7399faaeeffdf73faf7Timo Sirainen for (file = log->head->next; file != NULL; file = file->next) {
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen if (hdr->file_seq <= log->head->hdr.file_seq) {
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen /* make sure the sequence grows */
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen if (hdr->initial_modseq < log->head->sync_highest_modseq) {
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen /* this should be always up-to-date */
e64d7b6f388fecd0c83a4f2acb54e30d5ac98c6cTimo Sirainen hdr->initial_modseq = log->head->sync_highest_modseq;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file = mail_transaction_log_file_alloc(log, MEMORY_LOG_NAME);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen file->buffer = buffer_create_dynamic(default_pool, 4096);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen mail_transaction_log_get_dotlock_set(file->log, &dotlock_set);
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen ret = file_dotlock_create(&dotlock_set, file->filepath, 0,
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen log_file_set_syscall_error(file, "file_dotlock_create()");
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen "Timeout (%us) while waiting for "
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "dotlock for transaction log file %s",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)
b66d803de86bfb411165b3465b0d9ef64ecfe2a1Timo Sirainen ret = file_dotlock_delete(&file->log->dotlock);
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen log_file_set_syscall_error(file, "file_dotlock_delete()");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "Dotlock was lost for transaction log file %s",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenint mail_transaction_log_file_lock(struct mail_transaction_log_file *file)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen return mail_transaction_log_file_dotlock(file);
1deffbbcd7affd2ec89284f3c644cc73db0a2b90Timo Sirainen "Index is read-only, can't write-lock %s",
97cb20eb77d486ef67eac50567e3080faca025c1Timo Sirainen lock_timeout_secs = I_MIN(MAIL_TRANSACTION_LOG_LOCK_TIMEOUT,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen log_file_set_syscall_error(file, "mail_index_wait_lock_fd()");
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen "Timeout (%us) while waiting for lock for "
94cb0544ba0b6e249f20b76ad481bad27a633632Timo Sirainen "transaction log file %s%s",
94cb0544ba0b6e249f20b76ad481bad27a633632Timo Sirainen file_lock_find(file->fd, file->log->index->lock_method, F_WRLCK));
2f8da04d700cc23fcd6630226a4866e828b761bdTimo Sirainenvoid mail_transaction_log_file_unlock(struct mail_transaction_log_file *file,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
4eccf5310af7bed72cd6f7a559a93165c516e514Timo Sirainen if (lock_time >= MAIL_TRANSACTION_LOG_LOCK_WARN_SECS && lock_reason != NULL) {
2f8da04d700cc23fcd6630226a4866e828b761bdTimo Sirainen i_warning("Transaction log file %s was locked for %u seconds (%s)",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)mail_transaction_log_file_undotlock(file);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainenmail_transaction_log_file_read_header(struct mail_transaction_log_file *file)
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen i_assert(file->buffer == NULL && file->mmap_base == NULL);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen if (file->last_size < mmap_get_page_size() && file->last_size > 0) {
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen /* just read the entire transaction log to memory.
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen note that if some of the data hasn't been fully committed
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen yet (hdr.size=0), the buffer must be truncated later */
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen file->buffer = buffer_create_dynamic(default_pool, 4096);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen dest = buffer_append_space_unsafe(file->buffer, dest_size);
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen /* read only the header */
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen /* it's not necessarily an error to read less than wanted header size,
9061a2a9a7f8da780a5b50af3603f828167c6b13Timo Sirainen since older versions of the log format used smaller headers. */
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainenmail_transaction_log_file_fail_dupe(struct mail_transaction_log_file *file)
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen /* mark the old file corrupted. we can't safely remove
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen it from the list however, so return failure. */
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen if (strcmp(file->filepath, file->log->head->filepath) != 0) {
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen /* only mark .2 corrupted, just to make sure we don't lose any
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen changes from .log in case we're somehow wrong */
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen "Transaction log %s: "
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen "duplicate transaction log sequence (%u)",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen ret = mail_transaction_log_file_read_header(file);
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen /* incompatible version - fix silently */
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen if (ret < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "unexpected end of file while reading header");
e0f4fbf127b192667c62af7875c4f2ca294b6c7aTimo Sirainen if (MAIL_TRANSACTION_LOG_VERSION_HAVE(hdr_version, COMPAT_FLAGS)) {
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen /* we have compatibility flags */
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen enum mail_index_header_compat_flags compat_flags = 0;
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen /* architecture change */
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen "Rebuilding index file %s: "
aef92409cf369afdd2ecd81a4f80083cd4082f46Timo Sirainen "CPU architecture changed",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "Header size too small");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* @UNSAFE: smaller than we expected - zero out the fields we
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen shouldn't have filled */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* corrupted */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen "Transaction log file %s: marked corrupted",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->hdr.indexid != file->log->index->indexid) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* index file was probably just rebuilt and we don't
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen know about it yet */
e653e88f7058237047a98313140d67d40cb56b74Aki Tuomi "indexid changed: %u -> %u",
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen file->log->index->indexid, file->hdr.indexid);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* creating index file. since transaction log is created
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen first, use the indexid in it to create the main index
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen to avoid races. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen file->log->index->indexid = file->hdr.indexid;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* make sure we already don't have a file with the same sequence
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen opened. it shouldn't happen unless the old log file was
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen corrupted. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (f = file->log->files; f != NULL; f = f->next) {
90c23747727c85f80e4e8eed7968f0edbeac7ac5Timo Sirainen if (strcmp(f->filepath, f->log->head->filepath) != 0) {
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen /* old "f" is the .log.2 */
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen return mail_transaction_log_file_fail_dupe(f);
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen /* new "file" is probably the .log.2 */
4fcee755029b42c1f31227211290fa5047c00075Timo Sirainen return mail_transaction_log_file_fail_dupe(file);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen file->sync_highest_modseq = file->hdr.initial_modseq;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmail_transaction_log_file_stat(struct mail_transaction_log_file *file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmail_transaction_log_file_is_dupe(struct mail_transaction_log_file *file)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (tmp = file->log->files; tmp != NULL; tmp = tmp->next) {
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainenstatic void log_write_ext_hdr_init_data(struct mail_index *index, buffer_t *buf)
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen struct mail_transaction_ext_hdr_update *ext_hdr;
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen rext = array_idx(&index->extensions, index->ext_hdr_init_id);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen /* introduce the extension */
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen hdr = buffer_append_space_unsafe(buf, sizeof(*hdr));
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen intro = buffer_append_space_unsafe(buf, sizeof(*intro));
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen buffer_append(buf, rext->name, intro->name_size);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr));
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen /* add the extension header data */
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen hdr = buffer_append_space_unsafe(buf, sizeof(*hdr));
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen ext_hdr = buffer_append_space_unsafe(buf, sizeof(*ext_hdr));
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen buffer_append(buf, index->ext_hdr_init_data, rext->hdr_size);
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr));
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenmail_transaction_log_file_create2(struct mail_transaction_log_file *file,
1bea995196e46157e495a78b8f93780c576b3ef8Timo Sirainen need_lock = file->log->head != NULL && file->log->head->locked;
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen log_file_set_syscall_error(file, "fcntl(O_APPEND)");
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen /* although we check also mtime and file size below, it's done
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen only to fix broken log files. we don't bother flushing
04b8a90af181cc4c7959266855e8ed50a22ed413Timo Sirainen attribute cache just for that. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* log creation is locked now - see if someone already created it.
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen note that if we're rotating, we need to keep the log locked until
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen the file has been rewritten. and because fcntl() locks are stupid,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if we go and open()+close() the file and we had it already opened,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen its locks are lost. so we use stat() to check if the file has been
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen recreated, although it almost never is. */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen else if (nfs_safe_stat(file->filepath, &st) < 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* inode/dev checks are enough when we're rotating the file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen but not when we're replacing a broken log file */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* no-one else recreated the file */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* recreated. use the file if its header is ok */
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen fd = nfs_safe_open(file->filepath, O_RDWR | O_APPEND);
347acd14d8da653ce3757b3e29981326502bed6bTimo Sirainen mail_transaction_log_file_stat(file, FALSE) == 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* yes, it was ok */
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen /* creating the initial index */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mail_transaction_log_init_hdr(file->log, &file->hdr) < 0)
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen /* don't reset modseqs. if we're reseting due to rebuilding
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen indexes we'll probably want to keep uidvalidity and in such
bf8f4f90cb5e5f32c2611ba3425557964b9c47fcTimo Sirainen cases we really don't want to shrink modseqs. */
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen buffer_append(writebuf, &file->hdr, sizeof(file->hdr));
47e0598840ecffa364ebed523e06939e22738f06Timo Sirainen if (index->ext_hdr_init_data != NULL && reset)
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen if (write_full(new_fd, writebuf->data, writebuf->used) < 0) {
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen log_file_set_syscall_error(file, "write_full()");
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen if (file->log->index->fsync_mode == FSYNC_MODE_ALWAYS) {
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen /* the header isn't important, so don't bother calling
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen fdatasync() unless it's required */
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen log_file_set_syscall_error(file, "fdatasync()");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_transaction_log_file_stat(file, FALSE);
629600d9a85e8025c15a5eaeb80329e116e022c9Timo Sirainen /* we'll need to preserve the lock */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* if we return -1 the dotlock deletion code closes the fd */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* keep two log files */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* rename() would be nice and easy way to do this, except then
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen there's a race condition between the rename and
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen file_dotlock_replace(). during that time the log file
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen doesn't exist, which could cause problems. */
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen path2 = t_strconcat(file->filepath, ".2", NULL);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* try to link() anyway */
9716b2665ee3938d3dfe64bda44d7c3ae3b55d30Timo Sirainen if (nfs_safe_link(file->filepath, path2, FALSE) < 0 &&
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen mail_index_set_error(index, "link(%s, %s) failed: %m",
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* ignore the error. we don't care that much about the
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen second log file and we're going to overwrite this
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen first one. */
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen /* NOTE: here's a race condition where both .log and .log.2
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen point to the same file. our reading code should ignore that
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen though by comparing the inodes. */
ce2312763f6b5fd2da8bfb20847b87f29b063fa8Timo Sirainen /* need to unlock to avoid assert-crash in
ce2312763f6b5fd2da8bfb20847b87f29b063fa8Timo Sirainen mail_transaction_log_file_free() */
ce2312763f6b5fd2da8bfb20847b87f29b063fa8Timo Sirainen mail_transaction_log_file_unlock(file, "creation failed");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* success */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainenint mail_transaction_log_file_create(struct mail_transaction_log_file *file,
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen "Can't create log file %s: Index is read-only",
57dd2621d4230bc2e06c69a4b9f8cf8f0ff2bc93Timo Sirainen "Can't create log file %s: Index is marked corrupted",
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen mail_transaction_log_get_dotlock_set(file->log, &new_dotlock_set);
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen new_dotlock_set.lock_suffix = LOG_NEW_DOTLOCK_SUFFIX;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* With dotlocking we might already have path.lock created, so this
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen filename has to be different. */
bdd7a96c363346f7c38f389791be1487ca08775bTimo Sirainen fd = file_dotlock_open(&new_dotlock_set, file->filepath, 0, &dotlock);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen log_file_set_syscall_error(file, "file_dotlock_open()");
1f80b32fc28f7a723ff07c1694230a090808b506Timo Sirainen mail_index_fchown(index, fd, file_dotlock_get_lock_path(dotlock));
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* either fd gets used or the dotlock gets deleted and returned fd
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen is for the existing file */
e1ad2d13f5f154ee975eed5c8e7f1a7823082685Timo Sirainen ret = mail_transaction_log_file_create2(file, fd, reset, &dotlock);
c0d1bfc45e224251cb549de8d8804861e8acb517Timo Sirainenint mail_transaction_log_file_open(struct mail_transaction_log_file *file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int i;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0;; i++) {
bd503f12eb667df389a99162f567bd8785798f55Timo Sirainen file->fd = nfs_safe_open(file->filepath, O_RDONLY);
94d5a3fe15ff7096f3a2c7edcf790b5d3b8d1acbTimo Sirainen file->fd = nfs_safe_open(file->filepath, O_RDONLY);
c0d1bfc45e224251cb549de8d8804861e8acb517Timo Sirainen *reason_r = t_strdup_printf("open() failed: %m");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mail_transaction_log_file_stat(file, ignore_estale) < 0)
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen else if (mail_transaction_log_file_is_dupe(file)) {
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen /* probably our already opened .log file has been
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen renamed to .log.2 and we're trying to reopen it.
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen also possible that hit a race condition where .log
b222354c9553cd60b7dd418885e10c0473f73985Timo Sirainen and .log.2 are linked. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_transaction_log_file_read_hdr(file,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* success */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* corrupted */
6bc0f424bcdb9119d8159874cf98adfa53eefd9aTimo Sirainen /* don't delete */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* syscall error */
c0d1bfc45e224251cb549de8d8804861e8acb517Timo Sirainen *reason_r = t_strdup_printf("fstat() failed: %m");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* ESTALE - try again */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenlog_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen const char **error_r)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_transaction_header_update *u = data;
537598efa668f2277b429bfe35a342a7036e04d0Timo Sirainen const unsigned int size = trans_size - sizeof(struct mail_transaction_header);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen offsetof(struct mail_index_header, log_file_tail_offset);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (size < sizeof(*u) || size < sizeof(*u) + u->size) {
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen *error_r = "header update extends beyond record size";
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen mail_transaction_log_file_set_corrupted(file, "%s", *error_r);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen u->offset + u->size >= offset_pos + offset_size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),
082e82792b8ac33ad42beac510441b37a3c50737Timo Sirainen /* ignore shrinking tail offsets */
537598efa668f2277b429bfe35a342a7036e04d0Timo Sirainen } else if (tail_offset > file->sync_offset + trans_size) {
537598efa668f2277b429bfe35a342a7036e04d0Timo Sirainen "log_file_tail_offset %u goes past sync offset %"PRIuUOFF_T,
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainenflag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen /* Hide internal flags from modseqs if the log file's version
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen is new enough. This allows upgrading without the modseqs suddenly
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen shrinking. */
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(version, HIDE_INTERNAL_MODSEQS))
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen for (unsigned int i = 0; i < count; i++) {
ace341ac567376f37ded043c4c0f2c46b9aaecb1Timo Sirainen if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(&u[i]))
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainenvoid mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen unsigned int version)
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen trans_size = mail_index_offset_to_uint32(hdr->size);
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* tracking modseqs */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen } else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* modseqs not tracked yet. see if this is a modseq
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen extension introduction. */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen const struct mail_transaction_ext_intro *intro = data;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* modseq tracking started */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* not tracking modseqs */
9e6d83a3ef6abb393eeebca423cfd0d8cb08d430Timo Sirainen /* NOTE: keep in sync with mail_index_transaction_get_highest_modseq() */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen case MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT:
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen case MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT:
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* ignore expunge requests */
f784d5bb8edbec88829524135cfa100129f5384dTimo Sirainen /* fall through */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* these changes increase modseq */
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen const struct mail_transaction_flag_update *rec = data;
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen count = (trans_size - sizeof(*hdr)) / sizeof(*rec);
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen if (flag_updates_have_non_internal(rec, count, version))
8bb360f9e5de1c25e4f875205bb06e8bf15dae14Timo Sirainen const struct mail_transaction_modseq_update *rec, *end;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen end = CONST_PTR_OFFSET(data, trans_size - sizeof(*hdr));
76959d3d6fed45d5f5e1397fcdcf09a5adb87f24Timo Sirainen uint64_t modseq = ((uint64_t)rec->modseq_high32 << 32) |
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenmodseq_cache_hit(struct mail_transaction_log_file *file, unsigned int idx)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* @UNSAFE: move it to top */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen memmove(file->modseq_cache + 1, file->modseq_cache,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenmodseq_cache_get_offset(struct mail_transaction_log_file *file, uoff_t offset)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen for (i = 0; i < N_ELEMENTS(file->modseq_cache); i++) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* exact cache hit */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenmodseq_cache_get_modseq(struct mail_transaction_log_file *file, uint64_t modseq)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen for (i = 0; i < N_ELEMENTS(file->modseq_cache); i++) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (modseq < file->modseq_cache[i].highest_modseq)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (modseq == file->modseq_cache[i].highest_modseq) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* exact cache hit */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenlog_get_synced_record(struct mail_transaction_log_file *file, uoff_t *offset,
a0c8af555ec481ab12c2a99518cf7b20debd1627Timo Sirainen const char **error_r)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* we've already synced this record at some point. it should
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen trans_size = mail_index_offset_to_uint32(hdr->size);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen *offset - file->buffer_offset + trans_size > file->buffer->used) {
c15f15f71b885415fdaf2395ce52805770148917Timo Sirainen "Transaction log corrupted unexpectedly at "
a0c8af555ec481ab12c2a99518cf7b20debd1627Timo Sirainen mail_transaction_log_file_set_corrupted(file, "%s", *error_r);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenint mail_transaction_log_file_get_highest_modseq_at(
a0c8af555ec481ab12c2a99518cf7b20debd1627Timo Sirainen const char **error_r)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen *highest_modseq_r = file->sync_highest_modseq;
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen cache = modseq_cache_get_offset(file, offset);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* nothing usable in cache - scan from beginning */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* exact cache hit */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* use cache to skip over some records */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen ret = mail_transaction_log_file_map(file, cur_offset, offset, &reason);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen "Failed to map transaction log %s for getting modseq "
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen "at offset=%"PRIuUOFF_T" with start_offset=%"PRIuUOFF_T": %s",
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen i_assert(cur_offset + file->buffer->used >= offset);
a0c8af555ec481ab12c2a99518cf7b20debd1627Timo Sirainen if (log_get_synced_record(file, &cur_offset, &hdr, error_r) < 0)
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* @UNSAFE: cache the value */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen memmove(file->modseq_cache + 1, file->modseq_cache,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen file->modseq_cache[0].highest_modseq = cur_modseq;
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainenget_modseq_next_offset_at(struct mail_transaction_log_file *file,
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainen /* make sure we've read until end of file. this is especially important
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainen with non-head logs which might only have been opened without being
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainen ret = mail_transaction_log_file_map(file, *cur_offset, (uoff_t)-1, &reason);
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainen "Failed to map transaction log %s for getting offset "
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi "for modseq=%"PRIu64" with start_offset=%"PRIuUOFF_T": %s",
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi file->filepath, modseq, *cur_offset, reason);
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainen /* check sync_highest_modseq again in case sync_offset was updated */
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen if (modseq >= file->sync_highest_modseq && use_highest) {
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainen if (log_get_synced_record(file, cur_offset, &hdr, &reason) < 0) {
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainen mail_transaction_update_modseq(hdr, hdr + 1, cur_modseq,
7e78f1cfc10cc0540134b5507e08524a0fdd5c93Timo Sirainen MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenint mail_transaction_log_file_get_modseq_next_offset(
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen cache = modseq_cache_get_modseq(file, modseq);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* nothing usable in cache - scan from beginning */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* exact cache hit */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* use cache to skip over some records */
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen if ((ret = get_modseq_next_offset_at(file, modseq, TRUE, &cur_offset,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* if we got to sync_offset, cur_modseq should be
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen sync_highest_modseq */
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen "%s: Transaction log modseq tracking is corrupted - fixing",
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen /* retry getting the offset by reading from the beginning
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen of the file */
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen ret = get_modseq_next_offset_at(file, modseq, FALSE,
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen /* get it fixed on the next sync */
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen /* clear cache, since it's unreliable */
5e03576913a0030de6f8d22c79e353a968a6a35fTimo Sirainen memset(file->modseq_cache, 0, sizeof(file->modseq_cache));
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* @UNSAFE: cache the value */
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen memmove(file->modseq_cache + 1, file->modseq_cache,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen file->modseq_cache[0].highest_modseq = cur_modseq;
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenlog_file_track_sync(struct mail_transaction_log_file *file,
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen unsigned int trans_size, const char **error_r)
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen mail_transaction_update_modseq(hdr, hdr + 1, &file->sync_highest_modseq,
92dab926b2f2270057b40a907a00cf8eb2309ed6Timo Sirainen MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen /* external transactions: */
651fc0f1e43fef3e02e0e7b5f498973b05f641d7Timo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* see if this updates mailbox_sync_offset */
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen ret = log_file_track_mailbox_sync_offset_hdr(file, data,
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen if (file->sync_offset < file->index_undeleted_offset)
f90cbe597c41d5cc91debd371f8648bd8e6ffbc2Timo Sirainen file->log->index->index_delete_requested = FALSE;
651fc0f1e43fef3e02e0e7b5f498973b05f641d7Timo Sirainen file->index_deleted_offset = file->sync_offset + trans_size;
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen if (file->sync_offset < file->index_deleted_offset)
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen file->log->index->index_delete_requested = FALSE;
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen file->index_undeleted_offset = file->sync_offset + trans_size;
e188bab0b830136d04a1dd8b55e9afefae20d930Timo Sirainen const struct mail_transaction_boundary *boundary =
e188bab0b830136d04a1dd8b55e9afefae20d930Timo Sirainen wanted_buffer_size = file->sync_offset - file->buffer_offset +
e188bab0b830136d04a1dd8b55e9afefae20d930Timo Sirainen if (wanted_buffer_size > file->buffer->used) {
e188bab0b830136d04a1dd8b55e9afefae20d930Timo Sirainen /* the full transaction hasn't been written yet */
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if (file->max_tail_offset == file->sync_offset) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* external transactions aren't synced to mailbox. we can
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen update mailbox sync offset to skip this transaction to
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen avoid re-reading it at the next sync. */
f9f30736bc6e4cf535085824e7e124506fe2dbbbTimo Sirainenmail_transaction_log_file_sync(struct mail_transaction_log_file *file,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen i_assert(file->sync_offset >= file->buffer_offset);
57549c98995eb312013f5d98ea6ccf25175a6f18Timo Sirainen if (file->buffer_offset + size < file->sync_offset) {
57549c98995eb312013f5d98ea6ccf25175a6f18Timo Sirainen "log file shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")",
57549c98995eb312013f5d98ea6ccf25175a6f18Timo Sirainen file->buffer_offset + (uoff_t)size, file->sync_offset);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
9018d05c381d859f4b856a5ea0cad3538092ba1cTimo Sirainen /* fix the sync_offset to avoid crashes later on */
9018d05c381d859f4b856a5ea0cad3538092ba1cTimo Sirainen file->sync_offset = file->buffer_offset + size;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen hdr = CONST_PTR_OFFSET(data, file->sync_offset -
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen trans_size = mail_index_offset_to_uint32(hdr->size);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* unfinished */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (file->sync_offset - file->buffer_offset + trans_size > size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* transaction has been fully written */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen if ((ret = log_file_track_sync(file, hdr, trans_size, reason_r)) <= 0) {
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen if (file->mmap_base != NULL && !file->locked) {
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen /* Now that all the mmaped pages have page faulted, check if
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen the file had changed while doing that. Only after the last
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen page has faulted, the size returned by fstat() can be
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen trusted. Otherwise it might point to a page boundary while
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen the next page is still being written.
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen Without this check we might see partial transactions,
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen sometimes causing "Extension record updated without intro
e777a43ec49dd968b9b6064030fb40ce9e14097eTimo Sirainen prefix" errors. */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen *reason_r = t_strdup_printf("fstat() failed: %m");
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen avail = file->sync_offset - file->buffer_offset;
8c7000574087d5702cc3830e7f80c695ff8e9221Timo Sirainen /* There's more data than we could sync at the moment. If the
8c7000574087d5702cc3830e7f80c695ff8e9221Timo Sirainen last record's size wasn't valid, we can't know if it will
b7c051d9dd7ec0526f3c2c1f09f00d3a61c6576dTimo Sirainen be updated unless we've locked the log. */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen /* The size field will be updated soon */
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen mail_index_flush_read_cache(file->log->index, file->filepath,
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen file->hdr.file_seq == file->next->hdr.prev_file_seq &&
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen file->next->hdr.prev_file_offset != file->sync_offset) {
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen "Invalid transaction log size "
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen "(%"PRIuUOFF_T" vs %u): %s", file->sync_offset,
de14556e542b2a75a3a2118f76241f3c85313ebdTimo Sirainen file->log->head->hdr.prev_file_offset, file->filepath);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainenmail_transaction_log_file_insert_read(struct mail_transaction_log_file *file,
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen data = buffer_get_space_unsafe(file->buffer, 0, size);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen ret = pread_full(file->fd, data, size, offset);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* success */
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* failure. don't leave ourself to inconsistent state */
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen buffer_set_used_size(file->buffer, file->buffer->used - size);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* log file was deleted in NFS server, fail silently */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen *reason_r = t_strdup_printf("read() failed: %m");
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen *reason_r = t_strdup_printf("read() failed: %m");
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainenmail_transaction_log_file_read_more(struct mail_transaction_log_file *file,
2d5d7890bd9b282eb1092fc788a2432a6c79a7adTimo Sirainen read_offset = file->buffer_offset + file->buffer->used;
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen ret = pread(file->fd, data, LOG_PREFETCH, read_offset);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen } while (ret > 0 || (ret < 0 && errno == EINTR));
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen *reason_r = t_strdup_printf("pread() failed: %m");
8c7000574087d5702cc3830e7f80c695ff8e9221Timo Sirainen /* log file was deleted in NFS server, fail silently */
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainenmail_transaction_log_file_need_nfs_flush(struct mail_transaction_log_file *file)
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen const struct mail_index_header *hdr = &file->log->index->map->hdr;
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen file->hdr.file_seq == file->next->hdr.prev_file_seq &&
d7363f0716a4ca8bf8d9af1fe277113c705739b0Timo Sirainen file->next->hdr.prev_file_offset != max_offset) {
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen /* we already have a newer log file which says that we haven't
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen synced the entire file. */
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen if (file->hdr.file_seq == hdr->log_file_seq &&
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainenmail_transaction_log_file_read(struct mail_transaction_log_file *file,
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen /* NFS: if file isn't locked, we're optimistic that we can read enough
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen data without flushing attribute cache. if after reading we notice
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen that we really should have read more, flush the cache and try again.
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen if file is locked, the attribute cache was already flushed when
837ea26bce080f166b2ec90d901faeff58beb22bTimo Sirainen refreshing the log. */
470365df69b5a0b84146149e3ea1adcb27b1482eTimo Sirainen nfs_flush_attr_cache_unlocked(file->filepath);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen nfs_flush_attr_cache_fd_locked(file->filepath, file->fd);
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen if (file->buffer != NULL && file->buffer_offset > start_offset) {
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen /* we have to insert missing data to beginning of buffer */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen ret = mail_transaction_log_file_insert_read(file, start_offset, reason_r);
7ab64f2a89e0256693a4f0f0d6c3da6daab27cdcTimo Sirainen buffer_create_dynamic(default_pool, LOG_PREFETCH);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen if ((ret = mail_transaction_log_file_read_more(file, reason_r)) <= 0)
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen else if (file->log->nfs_flush && !nfs_flush &&
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen mail_transaction_log_file_need_nfs_flush(file)) {
d7363f0716a4ca8bf8d9af1fe277113c705739b0Timo Sirainen /* we didn't read enough data. flush and try again. */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen return mail_transaction_log_file_read(file, start_offset, TRUE, reason_r);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen } else if ((ret = mail_transaction_log_file_sync(file, &retry, reason_r)) == 0) {
f9f30736bc6e4cf535085824e7e124506fe2dbbbTimo Sirainen i_assert(!retry); /* retry happens only with mmap */
f9f30736bc6e4cf535085824e7e124506fe2dbbbTimo Sirainen i_assert(file->sync_offset >= file->buffer_offset);
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainenlog_file_map_check_offsets(struct mail_transaction_log_file *file,
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* broken start offset */
6d2f4a21bdfab2901b01f292b397256328e631beTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
6d2f4a21bdfab2901b01f292b397256328e631beTimo Sirainen file->filepath, start_offset, file->sync_offset);
0a1a4e586ced13635fc1b8f2c78c94cb35ef645aTimo Sirainen "current sync_offset (%"PRIuUOFF_T"), file size=%"PRIuUOFF_T,
0a1a4e586ced13635fc1b8f2c78c94cb35ef645aTimo Sirainen file->filepath, start_offset, file->sync_offset,
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen file->filepath, start_offset, file->sync_offset);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainenmail_transaction_log_file_mmap(struct mail_transaction_log_file *file,
6307d76096764e66bddc63d4a3e5a1aa19cc528fJosef 'Jeff' Sipek /* we may have switched to mmaping */
c131bf703b200890867f4c3839597ffdc7eba18dTimo Sirainen file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, MAP_SHARED,
9c7e81a1f5e629efa2324c6c22ce02f6f6e9c792Martti Rannanjärvi file->last_mmap_error_time = ioloop_time;
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen *reason_r = t_strdup_printf("mmap(size=%"PRIuSIZE_T") failed: %m",
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen log_file_set_syscall_error(file, "madvise()");
3281669db44d09a087a203201248abbc81b3cc1aTimo Sirainen buffer_create_from_const_data(&file->mmap_buffer,
c131bf703b200890867f4c3839597ffdc7eba18dTimo Sirainenmail_transaction_log_file_munmap(struct mail_transaction_log_file *file)
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0)
b251b66e48ed682fa511b9dabc979807fc18f71bTimo Sirainenmail_transaction_log_file_map_mmap(struct mail_transaction_log_file *file,
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen /* we are going to mmap() this file, but it's not necessarily
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen mmaped currently. */
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen i_assert(file->buffer_offset == 0 || file->mmap_base == NULL);
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen i_assert(file->mmap_size == 0 || file->mmap_base != NULL);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen *reason_r = t_strdup_printf("fstat() failed: %m");
a32ac1ad1ccc36d29f30b017197bb2a57feacd1aTimo Sirainen "file size shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")",
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
7184b2f499d5b8977671e92310148da72170edcaTimo Sirainen if (file->buffer != NULL && file->buffer_offset <= start_offset &&
c026384095b555cc86d032b043d107cc371aacecTimo Sirainen (uoff_t)st.st_size == file->buffer_offset + file->buffer->used) {
c026384095b555cc86d032b043d107cc371aacecTimo Sirainen /* we already have the whole file mapped */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen if ((ret = mail_transaction_log_file_sync(file, &retry, reason_r)) != 0 ||
47571dffa47750bd6188b9a846cea8c3b066753eTimo Sirainen /* size changed, re-mmap */
b251b66e48ed682fa511b9dabc979807fc18f71bTimo Sirainen if (file->last_size - start_offset < mmap_get_page_size()) {
b251b66e48ed682fa511b9dabc979807fc18f71bTimo Sirainen /* just reading the file is probably faster */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen if (mail_transaction_log_file_mmap(file, reason_r) < 0)
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen ret = mail_transaction_log_file_sync(file, &retry, reason_r);
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainenint mail_transaction_log_file_map(struct mail_transaction_log_file *file,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* corrupted */
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen i_assert(file->buffer == NULL || file->mmap_base != NULL ||
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen file->sync_offset >= file->buffer_offset + file->buffer->used);
07e80e04c8876b6bf3f95266f48b41e1a681e445Timo Sirainen if (file->locked_sync_offset_updated && file == file->log->head &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* we're not interested of going further than sync_offset */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen if (!log_file_map_check_offsets(file, start_offset,
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen if (file->buffer != NULL && file->buffer_offset <= start_offset) {
4bc96ba6f1d67a90a75fa131bcd2cd508ea5a05aTimo Sirainen /* see if we already have it */
f34227d18c5458c5a8bfe576ecf8d7bb4e75162eTimo Sirainen /* set this only when we've synced to end of file while locked
f34227d18c5458c5a8bfe576ecf8d7bb4e75162eTimo Sirainen (either end_offset=(uoff_t)-1 or we had to read anyway) */
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
2dadd5dacee37d409fa699597bfddc03074e4853Timo Sirainen if (start_offset < file->buffer_offset || file->buffer == NULL) {
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen /* we had moved the log to memory but failed to read
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen the beginning of the log file */
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen *reason_r = "Beginning of the log isn't available";
2e99f3f3bb35715ce5e0a75a2f2a9bac3ab4224bTimo Sirainen return log_file_map_check_offsets(file, start_offset,
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen /* although we could just skip over the unwanted data, we have
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen to sync everything so that modseqs are calculated
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen if ((file->log->index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0)
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen ret = mail_transaction_log_file_map_mmap(file, map_start_offset, reason_r);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen ret = mail_transaction_log_file_read(file, map_start_offset, FALSE, reason_r);
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen i_assert(file->buffer == NULL || file->mmap_base != NULL ||
63e2edd14ae7b1dc4a08e2e659501dbf519462f9Timo Sirainen file->sync_offset >= file->buffer_offset + file->buffer->used);
ce9d23c7c1e621398d2572a1d95171136f7ef6a2Timo Sirainen return log_file_map_check_offsets(file, start_offset, end_offset,
5b809b97673fb0a73aa5b9d82122612d699f6c5bTimo Sirainenint mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file *file)
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen /* just copy to memory */
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen buf = buffer_create_dynamic(default_pool, file->mmap_size);
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen buffer_append(buf, file->mmap_base, file->mmap_size);
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen /* and lose the mmap */
029cfcdce65b284d5230adf1c920a5f526b03b5cTimo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0)
902222fb0928d1701f20a384b73f327b1d9a15ddTimo Sirainen /* we don't have the full log in the memory. read it. */
5b809b97673fb0a73aa5b9d82122612d699f6c5bTimo Sirainen ret = mail_transaction_log_file_read(file, 0, FALSE, &error);
5b809b97673fb0a73aa5b9d82122612d699f6c5bTimo Sirainen "%s: Failed to read into memory: %s", file->filepath, error);