1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "lib.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "ioloop.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "buffer.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "file-lock.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "file-dotlock.h"

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila#include "read-full.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "write-full.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "mmap-util.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "mail-index-private.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "mail-index-view-private.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "mail-transaction-log-private.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "mail-transaction-util.h"

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include "mail-index-transaction-private.h"

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include <stddef.h>

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#include <sys/stat.h>

60ba197d17023594231d9805d889817782e41859Timo Sirainen

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen#define LOG_PREFETCH 1024

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#define LOG_DOTLOCK_TIMEOUT 30

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#define LOG_DOTLOCK_STALE_TIMEOUT 0

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen#define LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT 300

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen#define LOG_NEW_DOTLOCK_SUFFIX ".newlock"

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainenstruct mail_transaction_add_ctx {

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen struct mail_transaction_log *log;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen struct mail_index_view *view;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen buffer_t *appends, *expunges;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen buffer_t *flag_updates, *cache_updates;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen};

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainenstatic struct mail_transaction_log_file *

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainenmail_transaction_log_file_open_or_create(struct mail_transaction_log *log,

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen const char *path);

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainenstatic int mail_transaction_log_rotate(struct mail_transaction_log *log,

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen int lock_type);

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainenstatic int mail_transaction_log_lock_head(struct mail_transaction_log *log);

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen const char *fmt, ...)

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen{

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen va_list va;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen file->hdr.indexid = 0;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen if (pwrite_full(file->fd, &file->hdr.indexid,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen sizeof(file->hdr.indexid), 0) < 0) {

4ba3717a04823875c2a1d60ff9dc8177ae033d12Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

4ba3717a04823875c2a1d60ff9dc8177ae033d12Timo Sirainen file->filepath, "pwrite()");

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen va_start(va, fmt);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen t_push();

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_set_error(file->log->index,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen "Corrupted transaction log file %s: %s",

4ba3717a04823875c2a1d60ff9dc8177ae033d12Timo Sirainen file->filepath, t_strdup_vprintf(fmt, va));

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen t_pop();

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen va_end(va);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen}

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainenstatic int

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)

a9b3887f4d9ed75a76fed964c1930432bf84f4f5Timo Sirainen{

a9b3887f4d9ed75a76fed964c1930432bf84f4f5Timo Sirainen int ret;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (file->log->dotlock_count > 0)

4ba3717a04823875c2a1d60ff9dc8177ae033d12Timo Sirainen ret = 1;

4ba3717a04823875c2a1d60ff9dc8177ae033d12Timo Sirainen else {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen ret = file_lock_dotlock(file->filepath, NULL, FALSE,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen LOG_DOTLOCK_TIMEOUT,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen LOG_DOTLOCK_STALE_TIMEOUT,

4ba3717a04823875c2a1d60ff9dc8177ae033d12Timo Sirainen LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen NULL, NULL, &file->log->dotlock);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (ret > 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->log->dotlock_count++;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->locked = TRUE;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return 0;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (ret < 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->filepath,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen "file_lock_dotlock()");

4c61d6b6ad6173f06563c2cee9bd813c59277dd2Timo Sirainen return -1;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_set_error(file->log->index,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen "Timeout while waiting for release of "

57dc9669d34de7d08a44be9d7d3c9f6a0c34cc87Timo Sirainen "dotlock for transaction log file %s",

57dc9669d34de7d08a44be9d7d3c9f6a0c34cc87Timo Sirainen file->filepath);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->log->index->index_lock_timeout = TRUE;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return -1;

57dc9669d34de7d08a44be9d7d3c9f6a0c34cc87Timo Sirainen}

57dc9669d34de7d08a44be9d7d3c9f6a0c34cc87Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainenstatic int

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)

5455a917515f774fcaad35558f149536b796b531Teemu Huovila{

5455a917515f774fcaad35558f149536b796b531Teemu Huovila int ret;

5455a917515f774fcaad35558f149536b796b531Teemu Huovila

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (--file->log->dotlock_count > 0)

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return 0;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen ret = file_unlock_dotlock(file->filepath, &file->log->dotlock);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (ret < 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->filepath, "file_unlock_dotlock()");

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return -1;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (ret == 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_set_error(file->log->index,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen "Dotlock was lost for transaction log file %s",

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->filepath);

4c61d6b6ad6173f06563c2cee9bd813c59277dd2Timo Sirainen return -1;

fe0dff63d6b53d22ae16ac53ab183e9355a64a32Timo Sirainen }

4c61d6b6ad6173f06563c2cee9bd813c59277dd2Timo Sirainen return 0;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen}

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

fe0dff63d6b53d22ae16ac53ab183e9355a64a32Timo Sirainenstatic int

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainenmail_transaction_log_file_lock(struct mail_transaction_log_file *file)

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen{

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen int ret;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen if (file->locked)

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return 0;

fac865bad1ba10e85d80b63dedfd3493a65510d4Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (file->log->index->fcntl_locks_disable)

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return mail_transaction_log_file_dotlock(file);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen ret = file_wait_lock_full(file->fd, F_WRLCK, DEFAULT_LOCK_TIMEOUT,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen NULL, NULL);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (ret > 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->locked = TRUE;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return 0;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (ret < 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->filepath,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen "file_wait_lock()");

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return -1;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila mail_index_set_error(file->log->index,

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila "Timeout while waiting for release of "

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen "fcntl() lock for transaction log file %s",

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila file->filepath);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila file->log->index->index_lock_timeout = TRUE;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila return -1;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila}

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila

957b0b4c9aeff7153bb9ebf91d8aea550bd07865Teemu Huovilastatic void

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainenmail_transaction_log_file_unlock(struct mail_transaction_log_file *file)

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila{

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila int ret;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen if (!file->locked)

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila return;

5455a917515f774fcaad35558f149536b796b531Teemu Huovila

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen file->locked = FALSE;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen if (file->log->index->fcntl_locks_disable) {

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen mail_transaction_log_file_undotlock(file);

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen return;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen }

5455a917515f774fcaad35558f149536b796b531Teemu Huovila

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila ret = file_wait_lock(file->fd, F_UNLCK);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila if (ret <= 0) {

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila mail_index_file_set_syscall_error(file->log->index,

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila file->filepath,

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila "file_wait_lock()");

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila }

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila}

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila#define INDEX_HAS_MISSING_LOGS(index, file) \

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila !(((file)->hdr.file_seq == (index)->hdr->log_file_seq && \

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila (index)->hdr->log_file_offset >= \

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila sizeof(struct mail_transaction_log_header)) || \

957b0b4c9aeff7153bb9ebf91d8aea550bd07865Teemu Huovila ((file)->hdr.prev_file_seq == (index)->hdr->log_file_seq && \

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen (file)->hdr.prev_file_offset == (index)->hdr->log_file_offset))

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainenstatic int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen{

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen struct mail_index *index = log->index;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen struct mail_transaction_log_file *file;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila unsigned int lock_id;

0c827d2094e80ede4c089fc00260d7ffcc764636Timo Sirainen int ret;

0c827d2094e80ede4c089fc00260d7ffcc764636Timo Sirainen

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen if (mail_transaction_log_lock_head(log) < 0)

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen return -1;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen file = log->head;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen file->refcount++;

0ebeb1831a56e020b0958ed1ced50e86ee9347ecTimo Sirainen

fac865bad1ba10e85d80b63dedfd3493a65510d4Timo Sirainen ret = mail_index_lock_shared(index, TRUE, &lock_id);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila if (ret == 0) {

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila ret = mail_index_map(index, FALSE);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila if (ret <= 0)

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila ret = -1;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila else if (INDEX_HAS_MISSING_LOGS(index, file)) {

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila /* broken - fix it by creating a new log file */

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila ret = mail_transaction_log_rotate(log, F_UNLCK);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila }

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila }

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila if (--file->refcount == 0)

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila mail_transaction_logs_clean(log);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila else

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila mail_transaction_log_file_unlock(file);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila return ret;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila}

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainenstruct mail_transaction_log *

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainenmail_transaction_log_open_or_create(struct mail_index *index)

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen{

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila struct mail_transaction_log *log;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen const char *path;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila log = i_new(struct mail_transaction_log, 1);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila log->index = index;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen path = t_strconcat(log->index->filepath,

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen MAIL_TRANSACTION_LOG_PREFIX, NULL);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila log->head = mail_transaction_log_file_open_or_create(log, path);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila if (log->head == NULL) {

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila i_free(log);

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila return NULL;

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila }

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila if (index->fd != -1 &&

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen INDEX_HAS_MISSING_LOGS(index, log->head)) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen /* head log file isn't same as head index file -

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila (void)mail_transaction_log_check_file_seq(log);

084431105a179063aeb08f45eb5f506c61ea8602Aki Tuomi }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return log;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen}

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainenvoid mail_transaction_log_close(struct mail_transaction_log *log)

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen{

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_transaction_log_views_close(log);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila log->head->refcount--;

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila mail_transaction_logs_clean(log);

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila log->index->log = NULL;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen i_free(log);

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila}

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainenstatic void

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainenmail_transaction_log_file_close(struct mail_transaction_log_file *file)

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen{

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_transaction_log_file_unlock(file);

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen if (file->buffer != NULL)

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen buffer_free(file->buffer);

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (file->mmap_base != NULL) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila file->filepath,

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila "munmap()");

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (close(file->fd) < 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila file->filepath, "close()");

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen i_free(file->filepath);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen i_free(file);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen}

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainenstatic int

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainenmail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file)

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen{

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen int ret;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen if (ret < 0) {

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen // FIXME: handle ESTALE

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen file->filepath,

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen "pread_full()");

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen return -1;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen }

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen if (ret == 0) {

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen mail_transaction_log_file_set_corrupted(file,

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen "unexpected end of file while reading header");

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen return 0;

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen }

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen

f5e35763912e29c420a6977a211ae215ece3a8abTimo Sirainen if (file->hdr.indexid == 0) {

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen /* corrupted */

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen mail_index_set_error(file->log->index,

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen "Transaction log file %s: marked corrupted",

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen file->filepath);

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen return 0;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (file->hdr.indexid != file->log->index->indexid) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (file->log->index->fd == -1) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen /* creating index file, silently rebuild

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return 0;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen /* index file was probably just rebuilt and we don't know

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_set_error(file->log->index,

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen "Transaction log file %s: invalid indexid (%u != %u)",

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen file->filepath, file->hdr.indexid,

084431105a179063aeb08f45eb5f506c61ea8602Aki Tuomi file->log->index->indexid);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return 0;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return 1;

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen}

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainenstatic int

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainenmail_transaction_log_file_create2(struct mail_transaction_log *log,

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen const char *path, int fd,

084431105a179063aeb08f45eb5f506c61ea8602Aki Tuomi dev_t dev, ino_t ino)

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen{

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen struct mail_index *index = log->index;

ebcd7cf40e53c2bbc98f7f686e206cda5c0e3111Timo Sirainen struct mail_transaction_log_header hdr;

60ba197d17023594231d9805d889817782e41859Timo Sirainen struct stat st;

60ba197d17023594231d9805d889817782e41859Timo Sirainen unsigned int lock_id;

60ba197d17023594231d9805d889817782e41859Timo Sirainen int fd2, ret;

60ba197d17023594231d9805d889817782e41859Timo Sirainen

084431105a179063aeb08f45eb5f506c61ea8602Aki Tuomi /* log creation is locked now - see if someone already created it */

60ba197d17023594231d9805d889817782e41859Timo Sirainen fd2 = open(path, O_RDWR);

60ba197d17023594231d9805d889817782e41859Timo Sirainen if (fd2 != -1) {

60ba197d17023594231d9805d889817782e41859Timo Sirainen if ((ret = fstat(fd2, &st)) < 0) {

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen mail_index_file_set_syscall_error(index, path,

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen "fstat()");

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen } else if (st.st_ino == ino && CMP_DEV_T(st.st_dev, dev)) {

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen /* same file, still broken */

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen } else {

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen (void)file_dotlock_delete(path, LOG_NEW_DOTLOCK_SUFFIX,

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen fd);

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen return fd2;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen }

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen (void)close(fd2);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen fd2 = -1;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (ret < 0)

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen return -1;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen } else if (errno != ENOENT) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_file_set_syscall_error(index, path, "open()");

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen return -1;

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen }

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen memset(&hdr, 0, sizeof(hdr));

eeb03434472130f9631f164440566fd8d09e6380Timo Sirainen hdr.indexid = index->indexid;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen if (index->fd != -1) {

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen return -1;

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen hdr.prev_file_seq = index->hdr->log_file_seq;

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen hdr.prev_file_offset = index->hdr->log_file_offset;

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen }

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen hdr.file_seq = index->hdr->log_file_seq+1;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (index->fd != -1)

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen mail_index_unlock(index, lock_id);

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (log->head != NULL && hdr.file_seq <= log->head->hdr.file_seq) {

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen /* make sure the sequence grows */

d9a7e950a9cd21f2b4a90ec7759fca9e8fcc7995Timo Sirainen hdr.file_seq = log->head->hdr.file_seq+1;

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen }

4ba3717a04823875c2a1d60ff9dc8177ae033d12Timo Sirainen

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen if (write_full(fd, &hdr, sizeof(hdr)) < 0) {

dbf26a3ea43cd79fe88f01ec99c7d9440679b996Timo Sirainen mail_index_file_set_syscall_error(index, path,

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila "write_full()");

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila return -1;

09aed882b99e865ff6d7140ae2f77a42c9e7d1a7Teemu Huovila }

4ef1f9f3293965734e6e3c38c191ceb2246a721fTeemu Huovila

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen fd2 = dup(fd);

4ba3717a04823875c2a1d60ff9dc8177ae033d12Timo Sirainen if (fd2 < 0) {

1537d20b852cbbf0d6971790b84e0cce5ca61307Timo Sirainen mail_index_file_set_syscall_error(index, path, "dup()");

eeb03434472130f9631f164440566fd8d09e6380Timo Sirainen return -1;

eeb03434472130f9631f164440566fd8d09e6380Timo Sirainen }

eeb03434472130f9631f164440566fd8d09e6380Timo Sirainen

eeb03434472130f9631f164440566fd8d09e6380Timo Sirainen if (file_dotlock_replace(path, LOG_NEW_DOTLOCK_SUFFIX, fd, FALSE) <= 0)

eeb03434472130f9631f164440566fd8d09e6380Timo Sirainen return -1;

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen /* success */

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen return fd2;

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen}

6634e45500e81cfa36932203ee69c81745efd3e6Timo Sirainen

eeb03434472130f9631f164440566fd8d09e6380Timo Sirainenstatic int

mail_transaction_log_file_create(struct mail_transaction_log *log,

const char *path, dev_t dev, ino_t ino)

{

int fd, fd2;

/* With dotlocking we might already have path.lock created, so this

fd = file_dotlock_open(path, NULL, LOG_NEW_DOTLOCK_SUFFIX,

LOG_DOTLOCK_TIMEOUT,

LOG_DOTLOCK_STALE_TIMEOUT,

LOG_DOTLOCK_IMMEDIATE_STALE_TIMEOUT, NULL, NULL);

if (fd == -1) {

mail_index_file_set_syscall_error(log->index, path,

"file_dotlock_open()");

return -1;

}

fd2 = mail_transaction_log_file_create2(log, path, fd, dev, ino);

if (fd2 < 0) {

(void)file_dotlock_delete(path, LOG_NEW_DOTLOCK_SUFFIX, fd);

return -1;

}

return fd2;

}

static struct mail_transaction_log_file *

mail_transaction_log_file_fd_open(struct mail_transaction_log *log,

const char *path, int fd)

{

struct mail_transaction_log_file **p;

struct mail_transaction_log_file *file;

struct stat st;

int ret;

if (fstat(fd, &st) < 0) {

mail_index_file_set_syscall_error(log->index, path, "fstat()");

(void)close(fd);

return NULL;

}

file = i_new(struct mail_transaction_log_file, 1);

file->refcount = 1;

file->log = log;

file->filepath = i_strdup(path);

file->fd = fd;

file->st_dev = st.st_dev;

file->st_ino = st.st_ino;

file->last_mtime = st.st_mtime;

file->sync_offset = sizeof(struct mail_transaction_log_header);

ret = mail_transaction_log_file_read_hdr(file);

if (ret == 0) {

/* corrupted header */

fd = mail_transaction_log_file_create(log, path,

st.st_dev, st.st_ino);

if (fd == -1)

ret = -1;

else if (fstat(fd, &st) < 0) {

mail_index_file_set_syscall_error(log->index, path,

"fstat()");

(void)close(fd);

fd = -1;

ret = -1;

}

if (fd != -1) {

(void)close(file->fd);

file->fd = fd;

file->st_dev = st.st_dev;

file->st_ino = st.st_ino;

file->last_mtime = st.st_mtime;

memset(&file->hdr, 0, sizeof(file->hdr));

ret = mail_transaction_log_file_read_hdr(file);

}

}

if (ret <= 0) {

mail_transaction_log_file_close(file);

return NULL;

}

for (p = &log->tail; *p != NULL; p = &(*p)->next) {

if ((*p)->hdr.file_seq >= file->hdr.file_seq) {

/* log replaced with file having same sequence as

break;

}

}

*p = file;

return file;

}

static struct mail_transaction_log_file *

mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,

const char *path)

{

int fd;

fd = open(path, O_RDWR);

if (fd == -1) {

if (errno != ENOENT) {

mail_index_file_set_syscall_error(log->index, path,

"open()");

return NULL;

}

fd = mail_transaction_log_file_create(log, path, 0, 0);

if (fd == -1)

return NULL;

}

return mail_transaction_log_file_fd_open(log, path, fd);

}

void mail_transaction_logs_clean(struct mail_transaction_log *log)

{

struct mail_transaction_log_file **p, *next;

for (p = &log->tail; *p != NULL; ) {

if ((*p)->refcount != 0)

p = &(*p)->next;

else {

next = (*p)->next;

mail_transaction_log_file_close(*p);

*p = next;

}

}

if (log->tail == NULL)

log->head = NULL;

}

static int

mail_transaction_log_rotate(struct mail_transaction_log *log, int lock)

{

struct mail_transaction_log_file *file;

struct stat st;

int fd;

if (fstat(log->head->fd, &st) < 0) {

mail_index_file_set_syscall_error(log->index,

log->head->filepath,

"fstat()");

return -1;

}

fd = mail_transaction_log_file_create(log, log->head->filepath,

st.st_dev, st.st_ino);

if (fd == -1)

return -1;

file = mail_transaction_log_file_fd_open(log, log->head->filepath, fd);

if (file == NULL)

return -1;

if (lock) {

if (mail_transaction_log_file_lock(file) < 0) {

file->refcount--;

mail_transaction_logs_clean(log);

return -1;

}

}

i_assert(file->locked == lock);

if (--log->head->refcount == 0)

mail_transaction_logs_clean(log);

else

mail_transaction_log_file_unlock(log->head);

i_assert(log->head != file);

log->head = file;

return 0;

}

static int mail_transaction_log_recreate(struct mail_transaction_log *log)

{

unsigned int lock_id;

int ret;

if (mail_index_lock_shared(log->index, TRUE, &lock_id) < 0)

return -1;

ret = mail_transaction_log_rotate(log, FALSE);

mail_index_unlock(log->index, lock_id);

return ret;

}

static int mail_transaction_log_refresh(struct mail_transaction_log *log)

{

struct mail_transaction_log_file *file;

struct stat st;

const char *path;

path = t_strconcat(log->index->filepath,

MAIL_TRANSACTION_LOG_PREFIX, NULL);

if (stat(path, &st) < 0) {

mail_index_file_set_syscall_error(log->index, path, "stat()");

if (errno == ENOENT && log->head->locked) {

/* lost? */

return mail_transaction_log_recreate(log);

}

return -1;

}

if (log->head != NULL &&

log->head->st_ino == st.st_ino &&

CMP_DEV_T(log->head->st_dev, st.st_dev)) {

/* same file */

return 0;

}

file = mail_transaction_log_file_open_or_create(log, path);

if (file == NULL)

return -1;

i_assert(!file->locked);

if (log->head != NULL) {

if (--log->head->refcount == 0)

mail_transaction_logs_clean(log);

}

log->head = file;

return 0;

}

int mail_transaction_log_file_find(struct mail_transaction_log *log,

uint32_t file_seq,

struct mail_transaction_log_file **file_r)

{

struct mail_transaction_log_file *file;

if (file_seq > log->head->hdr.file_seq) {

if (mail_transaction_log_refresh(log) < 0)

return -1;

}

for (file = log->tail; file != NULL; file = file->next) {

if (file->hdr.file_seq == file_seq) {

*file_r = file;

return 1;

}

}

return 0;

}

static int

mail_transaction_log_file_sync(struct mail_transaction_log_file *file)

{

const struct mail_transaction_header *hdr;

const void *data;

size_t size;

uint32_t hdr_size;

data = buffer_get_data(file->buffer, &size);

while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {

hdr = CONST_PTR_OFFSET(data, file->sync_offset -

file->buffer_offset);

hdr_size = mail_index_offset_to_uint32(hdr->size);

if (hdr_size == 0) {

/* unfinished */

if (file->mmap_base == NULL) {

size = file->sync_offset - file->buffer_offset;

buffer_set_used_size(file->buffer, size);

}

return 0;

}

if (hdr_size < sizeof(*hdr)) {

mail_transaction_log_file_set_corrupted(file,

"hdr.size too small (%u)", hdr_size);

return -1;

}

if (file->sync_offset - file->buffer_offset + hdr_size > size)

break;

file->sync_offset += hdr_size;

}

return 0;

}

static int

mail_transaction_log_file_read(struct mail_transaction_log_file *file,

uoff_t offset)

{

void *data;

size_t size;

uint32_t read_offset;

int ret;

i_assert(file->mmap_base == NULL);

if (file->buffer != NULL && file->buffer_offset > offset) {

/* we have to insert missing data to beginning of buffer */

size = file->buffer_offset - offset;

buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);

file->buffer_offset -= size;

data = buffer_get_space_unsafe(file->buffer, 0, size);

ret = pread_full(file->fd, data, size, offset);

if (ret == 0) {

mail_transaction_log_file_set_corrupted(file,

"Unexpected end of file");

return 0;

}

if (ret < 0) {

if (errno == ESTALE) {

/* log file was deleted in NFS server,

return 0;

}

mail_index_file_set_syscall_error(file->log->index,

file->filepath,

"pread()");

return -1;

}

}

if (file->buffer == NULL) {

file->buffer = buffer_create_dynamic(default_pool,

LOG_PREFETCH, (size_t)-1);

file->buffer_offset = offset;

}

/* read all records */

read_offset = file->buffer_offset + buffer_get_used_size(file->buffer);

do {

data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);

ret = pread(file->fd, data, LOG_PREFETCH, read_offset);

if (ret > 0)

read_offset += ret;

size = read_offset - file->buffer_offset;

buffer_set_used_size(file->buffer, size);

if (mail_transaction_log_file_sync(file) < 0)

return -1;

} while (ret > 0 || (ret < 0 && errno == EINTR));

if (ret == 0) {

/* EOF */

buffer_set_used_size(file->buffer,

file->sync_offset - file->buffer_offset);

return 1;

}

if (errno == ESTALE) {

/* log file was deleted in NFS server, fail silently */

buffer_set_used_size(file->buffer,

offset - file->buffer_offset);

return 0;

}

mail_index_file_set_syscall_error(file->log->index, file->filepath,

"pread()");

return -1;

}

int mail_transaction_log_file_map(struct mail_transaction_log_file *file,

uoff_t start_offset, uoff_t end_offset)

{

struct mail_index *index = file->log->index;

size_t size;

struct stat st;

int ret, use_mmap;

i_assert(start_offset <= end_offset);

if (file->hdr.indexid == 0) {

/* corrupted */

return 0;

}

if (start_offset < sizeof(file->hdr)) {

mail_transaction_log_file_set_corrupted(file,

"offset (%"PRIuUOFF_T") < header size (%"PRIuSIZE_T")",

start_offset, sizeof(file->hdr));

return -1;

}

/* with mmap_no_write we could alternatively just write to log with

use_mmap = !index->mmap_disable && !index->mmap_no_write;

if (file->buffer != NULL && file->buffer_offset <= start_offset) {

/* see if we already have it */

size = buffer_get_used_size(file->buffer);

if (file->buffer_offset + size >= end_offset)

return 1;

}

if (file->mmap_base != NULL || use_mmap) {

if (fstat(file->fd, &st) < 0) {

mail_index_file_set_syscall_error(index, file->filepath,

"fstat()");

return -1;

}

}

if (file->mmap_base != NULL && st.st_size == file->mmap_size &&

file->buffer_offset <= start_offset) {

/* it's all mmaped already */

i_assert(end_offset == (uoff_t)-1);

return 1;

}

if (file->buffer != NULL &&

(file->mmap_base != NULL || use_mmap)) {

buffer_free(file->buffer);

file->buffer = NULL;

}

if (file->mmap_base != NULL) {

if (munmap(file->mmap_base, file->mmap_size) < 0) {

mail_index_file_set_syscall_error(index, file->filepath,

"munmap()");

}

file->mmap_base = NULL;

}

if (!use_mmap) {

ret = mail_transaction_log_file_read(file, start_offset);

if (ret <= 0) {

/* make sure we don't leave ourself in

if (file->buffer != NULL) {

buffer_free(file->buffer);

file->buffer = NULL;

}

return ret;

}

} else {

file->mmap_size = st.st_size;

file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ,

MAP_SHARED, file->fd, 0);

if (file->mmap_base == MAP_FAILED) {

file->mmap_base = NULL;

mail_index_file_set_syscall_error(index, file->filepath,

"mmap()");

return -1;

}

if (file->mmap_size > mmap_get_page_size()) {

if (madvise(file->mmap_base, file->mmap_size,

MADV_SEQUENTIAL) < 0) {

mail_index_file_set_syscall_error(index,

file->filepath, "madvise()");

}

}

file->buffer = buffer_create_const_data(default_pool,

file->mmap_base,

file->mmap_size);

file->buffer_offset = 0;

if (mail_transaction_log_file_sync(file) < 0)

return -1;

}

if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {

mail_transaction_log_file_set_corrupted(file,

"end_offset (%"PRIuUOFF_T") > current sync_offset "

"(%"PRIuSIZE_T")", end_offset, file->sync_offset);

return -1;

}

return 1;

}

static int mail_transaction_log_lock_head(struct mail_transaction_log *log)

{

struct mail_transaction_log_file *file;

int ret = 0;

/* we want to get the head file locked. this is a bit racy,

for (;;) {

file = log->head;

if (mail_transaction_log_file_lock(file) < 0)

return -1;

file->refcount++;

ret = mail_transaction_log_refresh(log);

if (--file->refcount == 0) {

mail_transaction_logs_clean(log);

file = NULL;

}

if (ret == 0 && log->head == file) {

/* success */

break;

}

if (file != NULL)

mail_transaction_log_file_unlock(file);

if (ret < 0)

break;

/* try again */

}

return ret;

}

static void

mail_transaction_log_append_fix(struct mail_index_transaction *t,

const struct mail_transaction_header *hdr,

const void *data)

{

const struct mail_transaction_append_header *append_hdr = data;

const struct mail_index_record *old, *old_end;

struct mail_index_record *appends, *end, *rec, *dest;

uint32_t record_size = t->append_record_size;

size_t size;

int deleted = FALSE;

if (t->appends == NULL)

return;

appends = buffer_get_modifyable_data(t->appends, &size);

end = PTR_OFFSET(appends, size);

if (appends == end)

return;

/* we'll just check that none of the appends are already in

old = CONST_PTR_OFFSET(data, sizeof(*append_hdr));

old_end = CONST_PTR_OFFSET(data, hdr->size);

while (old != old_end) {

/* appends are sorted */

for (rec = appends; rec != end; ) {

if (rec->uid >= old->uid) {

if (rec->uid == old->uid) {

rec->uid = 0;

deleted = TRUE;

}

break;

}

rec = PTR_OFFSET(rec, record_size);

}

old = CONST_PTR_OFFSET(old, append_hdr->record_size);

}

if (deleted) {

/* compress deleted appends away */

for (rec = dest = appends; rec != end; ) {

if (rec->uid != 0)

dest++;

else if (rec != dest)

*rec = *dest;

rec = PTR_OFFSET(rec, record_size);

}

buffer_set_used_size(t->appends,

(char *)dest - (char *)appends);

}

}

static int mail_transaction_log_scan_pending(struct mail_transaction_log *log,

struct mail_index_transaction *t)

{

struct mail_transaction_log_view *sync_view;

const struct mail_transaction_header *hdr;

const void *data;

uint32_t max_cache_file_seq = 0;

int ret;

sync_view = mail_transaction_log_view_open(log);

ret = mail_transaction_log_view_set(sync_view, t->view->log_file_seq,

t->view->log_file_offset,

log->head->hdr.file_seq, (uoff_t)-1,

MAIL_TRANSACTION_TYPE_MASK);

while ((ret = mail_transaction_log_view_next(sync_view,

&hdr, &data, NULL)) == 1) {

switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {

case MAIL_TRANSACTION_APPEND:

mail_transaction_log_append_fix(t, hdr, data);

break;

case MAIL_TRANSACTION_CACHE_RESET: {

const struct mail_transaction_cache_reset *reset = data;

max_cache_file_seq = reset->new_file_seq;

break;

}

}

}

/* make sure we're not writing cache_offsets to old cache file */

if (t->new_cache_file_seq == 0 && max_cache_file_seq != 0 &&

max_cache_file_seq != t->last_cache_file_seq &&

t->cache_updates != NULL) {

buffer_free(t->cache_updates);

t->cache_updates = NULL;

}

mail_transaction_log_view_close(sync_view);

return ret;

}

static int log_append_buffer(struct mail_transaction_log_file *file,

const buffer_t *buf, const buffer_t *hdr_buf,

enum mail_transaction_type type, int external)

{

struct mail_transaction_header hdr;

const void *data, *hdr_data;

size_t size, hdr_data_size;

uint32_t hdr_size;

i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);

data = buffer_get_data(buf, &size);

if (size == 0)

return 0;

if (hdr_buf != NULL)

hdr_data = buffer_get_data(hdr_buf, &hdr_data_size);

else {

hdr_data = NULL;

hdr_data_size = 0;

}

memset(&hdr, 0, sizeof(hdr));

hdr.type = type;

if (type == MAIL_TRANSACTION_EXPUNGE)

hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;

if (external)

hdr.type |= MAIL_TRANSACTION_EXTERNAL;

hdr_size =

mail_index_uint32_to_offset(sizeof(hdr) + size + hdr_data_size);

if (file->first_append_size == 0) {

/* size will be written later once everything is in disk */

file->first_append_size = hdr_size;

} else {

hdr.size = hdr_size;

}

if (pwrite_full(file->fd, &hdr, sizeof(hdr), file->sync_offset) < 0)

return -1;

file->sync_offset += sizeof(hdr);

if (hdr_data_size > 0) {

if (pwrite_full(file->fd, hdr_data, hdr_data_size,

file->sync_offset) < 0)

return -1;

file->sync_offset += hdr_data_size;

}

if (pwrite_full(file->fd, data, size, file->sync_offset) < 0)

return -1;

file->sync_offset += size;

return 0;

}

static const buffer_t *get_cache_reset_buf(struct mail_index_transaction *t)

{

struct mail_transaction_cache_reset u;

buffer_t *buf;

memset(&u, 0, sizeof(u));

u.new_file_seq = t->new_cache_file_seq;

buf = buffer_create_static(pool_datastack_create(), sizeof(u));

buffer_append(buf, &u, sizeof(u));

return buf;

}

static const buffer_t *

log_get_hdr_update_buffer(struct mail_index_transaction *t)

{

buffer_t *buf;

struct mail_transaction_header_update u;

uint16_t offset;

int state = 0;

memset(&u, 0, sizeof(u));

buf = buffer_create_dynamic(pool_datastack_create(), 256, (size_t)-1);

for (offset = 0; offset <= sizeof(t->hdr_change); offset++) {

if (offset < sizeof(t->hdr_change) && t->hdr_mask[offset]) {

if (state == 0) {

u.offset = offset;

state++;

}

} else {

if (state > 0) {

u.size = offset - u.offset;

buffer_append(buf, &u, sizeof(uint16_t)*2);

buffer_append(buf, t->hdr_change + u.offset,

u.size);

state = 0;

}

}

}

return buf;

}

int mail_transaction_log_append(struct mail_index_transaction *t,

uint32_t *log_file_seq_r,

uoff_t *log_file_offset_r)

{

struct mail_transaction_extra_rec_header extra_rec_hdr;

struct mail_index_view *view = t->view;

struct mail_index *index;

struct mail_transaction_log *log;

struct mail_transaction_log_file *file;

struct mail_index_header idx_hdr;

uoff_t append_offset;

buffer_t *hdr_buf;

unsigned int i, lock_id;

int ret;

index = mail_index_view_get_index(view);

log = index->log;

if (t->updates == NULL && t->new_cache_file_seq == 0 &&

t->cache_updates == NULL && t->expunges == NULL &&

t->appends == NULL && !t->hdr_changed) {

/* nothing to append */

*log_file_seq_r = 0;

*log_file_offset_r = 0;

return 0;

}

if (log->index->log_locked) {

i_assert(view->external);

} else {

if (mail_transaction_log_lock_head(log) < 0)

return -1;

/* update sync_offset */

if (mail_transaction_log_file_map(log->head,

log->head->sync_offset,

(uoff_t)-1) < 0) {

mail_transaction_log_file_unlock(log->head);

return -1;

}

}

if (mail_index_lock_shared(log->index, TRUE, &lock_id) < 0) {

if (!log->index->log_locked)

mail_transaction_log_file_unlock(log->head);

return -1;

}

idx_hdr = *log->index->hdr;

mail_index_unlock(log->index, lock_id);

if (log->head->sync_offset > MAIL_TRANSACTION_LOG_ROTATE_SIZE &&

log->head->last_mtime <

ioloop_time - MAIL_TRANSACTION_LOG_ROTATE_MIN_TIME) {

/* we might want to rotate, but check first that head file

if (log->head->hdr.file_seq == idx_hdr.log_file_seq) {

if (mail_transaction_log_rotate(log, TRUE) < 0) {

/* that didn't work. well, try to continue

}

}

}

file = log->head;

file->first_append_size = 0;

append_offset = file->sync_offset;

if (t->cache_updates != NULL &&

t->last_cache_file_seq < idx_hdr.cache_file_seq) {

/* cache_offsets point to old file, don't allow */

buffer_free(t->cache_updates);

t->cache_updates = NULL;

}

if (t->appends != NULL ||

(t->cache_updates != NULL && t->new_cache_file_seq == 0)) {

if (mail_transaction_log_scan_pending(log, t) < 0) {

if (!log->index->log_locked)

mail_transaction_log_file_unlock(file);

return -1;

}

}

ret = 0;

if (t->appends != NULL) {

struct mail_transaction_append_header hdr;

memset(&hdr, 0, sizeof(hdr));

hdr.record_size = t->append_record_size;

hdr_buf = buffer_create_data(pool_datastack_create(),

&hdr, sizeof(hdr));

buffer_set_used_size(hdr_buf, sizeof(hdr));

ret = log_append_buffer(file, t->appends, hdr_buf,

MAIL_TRANSACTION_APPEND,

view->external);

}

if (t->updates != NULL && ret == 0) {

ret = log_append_buffer(file, t->updates, NULL,

MAIL_TRANSACTION_FLAG_UPDATE,

view->external);

}

if (t->new_cache_file_seq != 0) {

ret = log_append_buffer(file, get_cache_reset_buf(t), NULL,

MAIL_TRANSACTION_CACHE_RESET,

view->external);

}

if (t->cache_updates != NULL && ret == 0) {

ret = log_append_buffer(file, t->cache_updates, NULL,

MAIL_TRANSACTION_CACHE_UPDATE,

view->external);

}

hdr_buf = buffer_create_data(pool_datastack_create(),

&extra_rec_hdr, sizeof(extra_rec_hdr));

buffer_set_used_size(hdr_buf, sizeof(extra_rec_hdr));

for (i = 0; i < view->index->extra_records_count; i++) {

if (t->extra_rec_updates[i] == NULL || ret != 0)

continue;

/* FIXME: do data_id mapping conversion */

extra_rec_hdr.data_id = i;

ret = log_append_buffer(file, t->extra_rec_updates[i], hdr_buf,

MAIL_TRANSACTION_EXTRA_REC_UPDATE,

view->external);

}

if (t->expunges != NULL && ret == 0) {

ret = log_append_buffer(file, t->expunges, NULL,

MAIL_TRANSACTION_EXPUNGE,

view->external);

}

if (t->hdr_changed && ret == 0) {

ret = log_append_buffer(file, log_get_hdr_update_buffer(t),

NULL, MAIL_TRANSACTION_HEADER_UPDATE,

view->external);

}

if (ret < 0) {

mail_index_file_set_syscall_error(log->index, file->filepath,

"pwrite()");

}

if (ret == 0 && (t->updates != NULL || t->appends != NULL) &&

t->hide_transaction) {

mail_index_view_add_synced_transaction(view, file->hdr.file_seq,

append_offset);

}

if (ret == 0 && fsync(file->fd) < 0) {

/* we don't know how much of it got written,

mail_index_file_set_syscall_error(log->index, file->filepath,

"fsync()");

ret = -1;

}

if (ret == 0 && file->first_append_size != 0) {

/* synced - rewrite first record's header */

ret = pwrite_full(file->fd, &file->first_append_size,

sizeof(uint32_t), append_offset);

if (ret < 0) {

mail_index_file_set_syscall_error(log->index,

file->filepath,

"pwrite()");

}

}

if (ret < 0) {

file->sync_offset = append_offset;

}

*log_file_seq_r = file->hdr.file_seq;

*log_file_offset_r = file->sync_offset;

if (!log->index->log_locked)

mail_transaction_log_file_unlock(file);

return ret;

}

int mail_transaction_log_sync_lock(struct mail_transaction_log *log,

uint32_t *file_seq_r, uoff_t *file_offset_r)

{

i_assert(!log->index->log_locked);

if (mail_transaction_log_lock_head(log) < 0)

return -1;

/* update sync_offset */

if (mail_transaction_log_file_map(log->head, log->head->sync_offset,

(uoff_t)-1) < 0) {

mail_transaction_log_file_unlock(log->head);

return -1;

}

log->index->log_locked = TRUE;