cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen#include "lib.h"

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen#include "ioloop.h"

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen#include "buffer.h"

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen#include "file-dotlock.h"

a1607f6001a9949e1cf9d49eb0aa936dbb4c77ffTimo Sirainen#include "nfs-workarounds.h"

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen#include "read-full.h"

00d58fcfe8191d6ce7efa801d289a5c0fe88d1aeTimo Sirainen#include "write-full.h"

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen#include "mmap-util.h"

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen#include "mail-index-private.h"

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen#include "mail-transaction-log-private.h"

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen#define LOG_PREFETCH 1024

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen#define MEMORY_LOG_NAME "(in-memory transaction log file)"

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen const char *fmt, ...)

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen{

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen va_list va;

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen file->corrupted = TRUE;

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen file->hdr.indexid = 0;

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen /* indexid=0 marks the log file as corrupted */

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (pwrite_full(file->fd, &file->hdr.indexid,

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen sizeof(file->hdr.indexid),

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen offsetof(struct mail_transaction_log_header,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen indexid)) < 0) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen mail_index_file_set_syscall_error(file->log->index,

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen file->filepath, "pwrite()");

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen va_start(va, fmt);

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen T_BEGIN {

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen mail_index_set_error(file->log->index,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen "Corrupted transaction log file %s: %s",

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath, t_strdup_vprintf(fmt, va));

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen } T_END;

4462bd7b4c7ef3de006f060e155a90e5de7cae21Timo Sirainen va_end(va);

f6c1297c26b355c4aec2a08978f51ec3efecb351Timo Sirainen}

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstruct mail_transaction_log_file *

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenmail_transaction_log_file_alloc(struct mail_transaction_log *log,

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen const char *path)

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen{

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen struct mail_transaction_log_file *file;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file = i_new(struct mail_transaction_log_file, 1);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->log = log;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath = i_strdup(path);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->fd = -1;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen return file;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen}

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenvoid mail_transaction_log_file_free(struct mail_transaction_log_file **_file)

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen{

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen struct mail_transaction_log_file *file = *_file;

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen struct mail_transaction_log_file **p;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen int old_errno = errno;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen *_file = NULL;

cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen

73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen mail_transaction_log_file_unlock(file);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen for (p = &file->log->files; *p != NULL; p = &(*p)->next) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (*p == file) {

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen *p = file->next;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen break;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen }

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (file == file->log->head)

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->log->head = NULL;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (file->buffer != NULL)

73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen buffer_free(&file->buffer);

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen

e07bf3772a2bc075de4915ad0961beb8d083c22dTimo Sirainen if (file->mmap_base != NULL) {

e07bf3772a2bc075de4915ad0961beb8d083c22dTimo Sirainen if (munmap(file->mmap_base, file->mmap_size) < 0) {

73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen mail_index_file_set_syscall_error(file->log->index,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath,

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen "munmap()");

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

e07bf3772a2bc075de4915ad0961beb8d083c22dTimo Sirainen }

e07bf3772a2bc075de4915ad0961beb8d083c22dTimo Sirainen

e07bf3772a2bc075de4915ad0961beb8d083c22dTimo Sirainen if (file->fd != -1) {

e07bf3772a2bc075de4915ad0961beb8d083c22dTimo Sirainen if (close(file->fd) < 0) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen mail_index_file_set_syscall_error(file->log->index,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath,

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen "close()");

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen }

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen }

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen i_free(file->filepath);

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen i_free(file);

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen errno = old_errno;

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen}

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainenstatic void

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainenmail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen{

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen struct mail_transaction_log *log = file->log;

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen struct mail_transaction_log_file **p;

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen struct mail_index_map *map = log->index->map;

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen if (map != NULL && file->hdr.file_seq == map->hdr.log_file_seq &&

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen map->hdr.log_file_head_offset != 0) {

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen /* we can get a valid log offset from index file. initialize

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (map->hdr.log_file_head_offset >= file->hdr.hdr_size)

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->sync_offset = map->hdr.log_file_head_offset;

96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen else {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen mail_index_set_error(log->index,

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen "%s: log_file_head_offset too small",

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen log->index->filepath);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->sync_offset = file->hdr.hdr_size;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

d66be2bebfa96e7d3d20e2153f60e6e25dcc9a18Timo Sirainen file->saved_tail_offset = map->hdr.log_file_tail_offset;

e376e08040b5f21ff79a15ae728d2532a34207f6Timo Sirainen } else {

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen file->sync_offset = file->hdr.hdr_size;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen /* insert it to correct position */

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen for (p = &log->files; *p != NULL; p = &(*p)->next) {

73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen if ((*p)->hdr.file_seq > file->hdr.file_seq)

fd35227c47190afc832579ca5c76524792701bf7Timo Sirainen break;

fd35227c47190afc832579ca5c76524792701bf7Timo Sirainen i_assert((*p)->hdr.file_seq < file->hdr.file_seq);

fd35227c47190afc832579ca5c76524792701bf7Timo Sirainen }

fd35227c47190afc832579ca5c76524792701bf7Timo Sirainen

fd35227c47190afc832579ca5c76524792701bf7Timo Sirainen file->next = *p;

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainen *p = file;

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainen}

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainen

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainenstatic int

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainenmail_transaction_log_init_hdr(struct mail_transaction_log *log,

73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen struct mail_transaction_log_header *hdr)

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen{

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen struct mail_index *index = log->index;

96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen memset(hdr, 0, sizeof(*hdr));

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;

d66be2bebfa96e7d3d20e2153f60e6e25dcc9a18Timo Sirainen hdr->hdr_size = sizeof(struct mail_transaction_log_header);

d66be2bebfa96e7d3d20e2153f60e6e25dcc9a18Timo Sirainen hdr->indexid = log->index->indexid;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen hdr->create_stamp = ioloop_time;

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen if (index->fd != -1) {

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen /* not creating index - make sure we have latest header */

783278c012551bc2f25f065a8d3ec1a3cfc0d296Timo Sirainen if (!index->mapping) {

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (mail_index_map(index,

5d49cbcf87354f0ddf3b71bc5f0cefdc02b14f68Timo Sirainen MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)

ce3faaaaab3e2d45b023396287e02f88e5c76e74Timo Sirainen return -1;

fd35227c47190afc832579ca5c76524792701bf7Timo Sirainen } else {

ce3faaaaab3e2d45b023396287e02f88e5c76e74Timo Sirainen /* if we got here from mapping, the .log file is

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen }

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen }

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen if (index->map != NULL) {

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainen hdr->prev_file_seq = index->map->hdr.log_file_seq;

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainen hdr->prev_file_offset = index->map->hdr.log_file_head_offset;

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainen hdr->file_seq = index->map->hdr.log_file_seq + 1;

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainen } else {

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainen hdr->file_seq = 1;

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainen }

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainen

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainen if (log->head != NULL && hdr->file_seq <= log->head->hdr.file_seq) {

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen /* make sure the sequence grows */

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen hdr->file_seq = log->head->hdr.file_seq+1;

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen }

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen return 0;

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen}

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainenstruct mail_transaction_log_file *

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainenmail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen{

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen struct mail_transaction_log_file *file;

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen file = mail_transaction_log_file_alloc(log, MEMORY_LOG_NAME);

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainen if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainen i_free(file);

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainen return NULL;

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainen }

2e263a9d901483a902720a30c474761bd3324fe8Timo Sirainen

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen file->buffer = buffer_create_dynamic(default_pool, 4096);

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen file->buffer_offset = sizeof(file->hdr);

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen mail_transaction_log_file_add_to_list(file);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen return file;

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen}

bbd4c4cf902539c25c471157eb9849459734759cTimo Sirainen

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainenstatic int

3e7565a7b39694bcdf448d8eb2a7f0774733297bTimo Sirainenmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen{

19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen int ret;

eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (file->log->dotlock_count > 0)

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen ret = 1;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen else {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen ret = file_dotlock_create(&file->log->dotlock_settings,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath, 0,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen &file->log->dotlock);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (ret > 0) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->log->dotlock_count++;

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen file->locked = TRUE;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen return 0;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (ret < 0) {

73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen mail_index_file_set_syscall_error(file->log->index,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen "file_dotlock_create()");

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return -1;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen mail_index_set_error(file->log->index,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen "Timeout while waiting for "

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen "dotlock for transaction log file %s",

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->log->index->index_lock_timeout = TRUE;

73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen return -1;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen}

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainenstatic int

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainenmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen{

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen int ret;

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (--file->log->dotlock_count > 0)

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen return 0;

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen ret = file_dotlock_delete(&file->log->dotlock);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (ret < 0) {

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen file->filepath, "file_dotlock_delete()");

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen return -1;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (ret == 0) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen mail_index_set_error(file->log->index,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen "Dotlock was lost for transaction log file %s",

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen file->filepath);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen return -1;

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen }

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen return 0;

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen}

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainenint mail_transaction_log_file_lock(struct mail_transaction_log_file *file)

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen{

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen int ret;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (file->locked)

bd4d0a1a7c0626452b8d82f37e3ec07267ac9896Timo Sirainen return 0;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

bd4d0a1a7c0626452b8d82f37e3ec07267ac9896Timo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {

bd4d0a1a7c0626452b8d82f37e3ec07267ac9896Timo Sirainen file->locked = TRUE;

bd4d0a1a7c0626452b8d82f37e3ec07267ac9896Timo Sirainen return 0;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen }

cbc61fcb33b370d049c16a3c44568b4deb4e2b33Timo Sirainen

bd4d0a1a7c0626452b8d82f37e3ec07267ac9896Timo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen return mail_transaction_log_file_dotlock(file);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen i_assert(file->file_lock == NULL);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen F_WRLCK, MAIL_TRANSCATION_LOG_LOCK_TIMEOUT,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen &file->file_lock);

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (ret > 0) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->locked = TRUE;

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen return 0;

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen }

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen if (ret < 0) {

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen file->filepath,

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen "mail_index_wait_lock_fd()");

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen return -1;

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen }

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen mail_index_set_error(file->log->index,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen "Timeout while waiting for lock for transaction log file %s",

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath);

56963ffad65b860c827553dfaf09fb766cb7e20eTimo Sirainen file->log->index->index_lock_timeout = TRUE;

56963ffad65b860c827553dfaf09fb766cb7e20eTimo Sirainen return -1;

56963ffad65b860c827553dfaf09fb766cb7e20eTimo Sirainen}

56963ffad65b860c827553dfaf09fb766cb7e20eTimo Sirainen

56963ffad65b860c827553dfaf09fb766cb7e20eTimo Sirainenvoid mail_transaction_log_file_unlock(struct mail_transaction_log_file *file)

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen{

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen if (!file->locked)

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen return;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen file->locked = FALSE;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen return;

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen mail_transaction_log_file_undotlock(file);

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen return;

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen }

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen file_unlock(&file->file_lock);

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen}

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainenstatic int

e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainenmail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen bool ignore_estale)

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen{

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen struct mail_transaction_log_file *f;

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen int ret;

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen

e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen if (file->corrupted)

e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen return 0;

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen if (ret < 0) {

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen if (errno != ESTALE || !ignore_estale) {

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen mail_index_file_set_syscall_error(file->log->index,

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen file->filepath,

9514dbd41faad453406fc3a10aca791417a9ce75Timo Sirainen "pread_full()");

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen }

9514dbd41faad453406fc3a10aca791417a9ce75Timo Sirainen return -1;

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen }

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen if (ret == 0) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen mail_transaction_log_file_set_corrupted(file,

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen "unexpected end of file while reading header");

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen return 0;

b87daa509bf5b306189282a9df795d094a6d7150Timo Sirainen }

b87daa509bf5b306189282a9df795d094a6d7150Timo Sirainen

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen /* incompatible version - fix silently */

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen return 0;

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen }

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen mail_transaction_log_file_set_corrupted(file,

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen "Header size too small");

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen return 0;

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen }

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (file->hdr.hdr_size < sizeof(file->hdr)) {

f6d57a2c182f63cd52819f0abb3c3d9f828afe19Timo Sirainen /* @UNSAFE: smaller than we expected - zero out the fields we

783278c012551bc2f25f065a8d3ec1a3cfc0d296Timo Sirainen memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,

4073f0dbf3277f981a8fcee3b89ea15aaf380a7fTimo Sirainen sizeof(file->hdr) - file->hdr.hdr_size);

4073f0dbf3277f981a8fcee3b89ea15aaf380a7fTimo Sirainen }

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (file->hdr.indexid == 0) {

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen /* corrupted */

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen file->corrupted = TRUE;

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen mail_index_set_error(file->log->index,

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen "Transaction log file %s: marked corrupted",

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen file->filepath);

5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen return 0;

5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen }

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen if (file->hdr.indexid != file->log->index->indexid) {

539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen if (file->log->index->indexid != 0 &&

5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen !file->log->index->initial_create) {

cf7164ece50797a67fc4bfb5889022ac93a36a8aTimo Sirainen /* index file was probably just rebuilt and we don't

5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen mail_transaction_log_file_set_corrupted(file,

5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen "indexid changed %u -> %u",

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen file->log->index->indexid, file->hdr.indexid);

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen return 0;

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen }

0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen

5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen /* creating index file. since transaction log is created

file->log->index->indexid = file->hdr.indexid;

}

/* make sure we already don't have a file with the same sequence

for (f = file->log->files; f != NULL; f = f->next) {

if (f->hdr.file_seq == file->hdr.file_seq) {

/* mark the old file corrupted. we can't safely remove

mail_transaction_log_file_set_corrupted(f,

"duplicate transaction log sequence (%u)",

f->hdr.file_seq);

return 0;

}

}

return 1;

}

static int

mail_transaction_log_file_stat(struct mail_transaction_log_file *file,

bool ignore_estale)

{

struct stat st;

if (fstat(file->fd, &st) < 0) {

if (errno != ESTALE || !ignore_estale) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "fstat()");

}

return -1;

}

file->st_dev = st.st_dev;

file->st_ino = st.st_ino;

file->last_mtime = st.st_mtime;

file->last_size = st.st_size;

return 0;

}

static bool

mail_transaction_log_file_is_dupe(struct mail_transaction_log_file *file)

{

struct mail_transaction_log_file *tmp;

for (tmp = file->log->files; tmp != NULL; tmp = tmp->next) {

if (tmp->st_ino == file->st_ino &&

CMP_DEV_T(tmp->st_dev, file->st_dev))

return TRUE;

}

return FALSE;

}

static int

mail_transaction_log_file_create2(struct mail_transaction_log_file *file,

int new_fd, bool reset,

struct dotlock **dotlock)

{

struct mail_index *index = file->log->index;

struct stat st;

const char *path2;

int fd, ret;

bool rename_existing;

if (index->nfs_flush) {

/* although we check also mtime and file size below, it's done

nfs_flush_file_handle_cache(file->filepath);

}

/* log creation is locked now - see if someone already created it.

if (reset)

rename_existing = FALSE;

else if (nfs_safe_stat(file->filepath, &st) < 0) {

if (errno != ENOENT) {

mail_index_file_set_syscall_error(index, file->filepath,

"stat()");

return -1;

}

rename_existing = FALSE;

} else if (st.st_ino == file->st_ino &&

CMP_DEV_T(st.st_dev, file->st_dev) &&

/* inode/dev checks are enough when we're rotating the file,

st.st_mtime == file->last_mtime &&

(uoff_t)st.st_size == file->last_size) {

/* no-one else recreated the file */

rename_existing = TRUE;

} else {

/* recreated. use the file if its header is ok */

fd = nfs_safe_open(file->filepath, O_RDWR);

if (fd == -1) {

if (errno != ENOENT) {

mail_index_file_set_syscall_error(index,

file->filepath, "open()");

return -1;

}

} else {

file->fd = fd;

if (mail_transaction_log_file_read_hdr(file,

FALSE) > 0 &&

mail_transaction_log_file_stat(file, FALSE) == 0) {

/* yes, it was ok */

(void)file_dotlock_delete(dotlock);

mail_transaction_log_file_add_to_list(file);

return 0;

}

file->fd = -1;

if (close(fd) < 0) {

mail_index_file_set_syscall_error(index,

file->filepath, "close()");

}

}

rename_existing = FALSE;

}

if (mail_transaction_log_init_hdr(file->log, &file->hdr) < 0)

return -1;

if (reset) {

file->hdr.prev_file_seq = 0;

file->hdr.prev_file_offset = 0;

}

if (write_full(new_fd, &file->hdr, sizeof(file->hdr)) < 0) {

mail_index_file_set_syscall_error(index, file->filepath,

"write_full()");

return -1;

}

if (index->nfs_flush) {

/* the header isn't important, so don't bother calling

if (fdatasync(new_fd) < 0) {

mail_index_file_set_syscall_error(index, file->filepath,

"fdatasync()");

return -1;

}

}

file->fd = new_fd;

ret = mail_transaction_log_file_stat(file, FALSE);

if (file->log->head != NULL && file->log->head->locked) {

/* we'll need to preserve the lock */

if (mail_transaction_log_file_lock(file) < 0)

ret = -1;

}

/* if we return -1 the dotlock deletion code closes the fd */

file->fd = -1;

if (ret < 0)

return -1;

/* keep two log files */

if (rename_existing) {

/* rename() would be nice and easy way to do this, except then

path2 = t_strconcat(file->filepath, ".2", NULL);

if (unlink(path2) < 0 && errno != ENOENT) {

mail_index_set_error(index, "unlink(%s) failed: %m",

path2);

/* try to link() anyway */

}

if (nfs_safe_link(file->filepath, path2, FALSE) < 0 &&

errno != ENOENT && errno != EEXIST) {

mail_index_set_error(index, "link(%s, %s) failed: %m",

file->filepath, path2);

/* ignore the error. we don't care that much about the

}

}

if (file_dotlock_replace(dotlock,

DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) <= 0)

return -1;

/* success */

file->fd = new_fd;

mail_transaction_log_file_add_to_list(file);

return 0;

}

int mail_transaction_log_file_create(struct mail_transaction_log_file *file,

bool reset)

{

struct mail_index *index = file->log->index;

struct dotlock *dotlock;

mode_t old_mask;

int fd;

i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));

/* With dotlocking we might already have path.lock created, so this

old_mask = umask(index->mode ^ 0666);

fd = file_dotlock_open(&file->log->new_dotlock_settings,

file->filepath, 0, &dotlock);

umask(old_mask);

if (fd == -1) {

mail_index_file_set_syscall_error(index, file->filepath,

"file_dotlock_open()");

return -1;

}

if (index->gid != (gid_t)-1 &&

fchown(fd, (uid_t)-1, index->gid) < 0) {

mail_index_file_set_syscall_error(index, file->filepath,

"fchown()");

(void)file_dotlock_delete(&dotlock);

return -1;

}

/* either fd gets used or the dotlock gets deleted and returned fd

if (mail_transaction_log_file_create2(file, fd, reset, &dotlock) < 0) {

if (dotlock != NULL)

(void)file_dotlock_delete(&dotlock);

return -1;

}

return 0;

}

int mail_transaction_log_file_open(struct mail_transaction_log_file *file,

bool check_existing)

{

unsigned int i;

bool ignore_estale;

int ret;

for (i = 0;; i++) {

file->fd = nfs_safe_open(file->filepath, O_RDWR);

if (file->fd == -1) {

if (errno == ENOENT)

return 0;

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "open()");

return -1;

}

ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT;

if (mail_transaction_log_file_stat(file, ignore_estale) < 0)

ret = -1;

else if (check_existing &&

mail_transaction_log_file_is_dupe(file))

return 0;

else {

ret = mail_transaction_log_file_read_hdr(file,

ignore_estale);

}

if (ret > 0) {

/* success */

break;

}

if (ret == 0) {

/* corrupted */

if (unlink(file->filepath) < 0 && errno != ENOENT) {

mail_index_set_error(file->log->index,

"unlink(%s) failed: %m",

file->filepath);

}

return 0;

}

if (errno != ESTALE ||

i == MAIL_INDEX_ESTALE_RETRY_COUNT) {

/* syscall error */

return -1;

}

/* ESTALE - try again */

}

mail_transaction_log_file_add_to_list(file);

return 1;

}

static int

log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,

const void *data, unsigned int size)

{

const struct mail_transaction_header_update *u = data;

const struct mail_index_header *ihdr;

const unsigned int offset_pos =

offsetof(struct mail_index_header, log_file_tail_offset);

const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);

uint32_t sync_offset;

i_assert(offset_size == sizeof(sync_offset));

if (size < sizeof(*u) || size < sizeof(*u) + u->size) {

mail_transaction_log_file_set_corrupted(file,

"header update extends beyond record size");

return -1;

}

if (u->offset <= offset_pos &&

u->offset + u->size >= offset_pos + offset_size) {

memcpy(&sync_offset,

CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),

sizeof(sync_offset));

if (sync_offset < file->saved_tail_offset) {

mail_transaction_log_file_set_corrupted(file,

"log_file_tail_offset shrank");

return -1;

}

file->saved_tail_offset = sync_offset;

if (sync_offset > file->max_tail_offset)

file->max_tail_offset = sync_offset;

return 1;

}

return 0;

}

static int

log_file_track_mailbox_sync_offset(struct mail_transaction_log_file *file,

const struct mail_transaction_header *hdr,

unsigned int trans_size)

{

int ret;

i_assert((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0);

if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==

MAIL_TRANSACTION_HEADER_UPDATE) {

/* see if this updates mailbox_sync_offset */

ret = log_file_track_mailbox_sync_offset_hdr(file, hdr + 1,

trans_size -

sizeof(*hdr));

if (ret != 0)

return ret < 0 ? -1 : 0;

}

if (file->max_tail_offset == file->sync_offset) {

/* external transactions aren't synced to mailbox. we can

file->max_tail_offset += trans_size;

}

return 0;

}

static int

mail_transaction_log_file_sync(struct mail_transaction_log_file *file)

{

const struct mail_transaction_header *hdr;

const void *data;

struct stat st;

size_t size, avail;

uint32_t trans_size = 0;

data = buffer_get_data(file->buffer, &size);

if (file->sync_offset < file->buffer_offset)

file->sync_offset = file->buffer_offset;

while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {

hdr = CONST_PTR_OFFSET(data, file->sync_offset -

file->buffer_offset);

trans_size = mail_index_offset_to_uint32(hdr->size);

if (trans_size == 0) {

/* unfinished */

return 1;

}

if (trans_size < sizeof(*hdr)) {

mail_transaction_log_file_set_corrupted(file,

"hdr.size too small (%u)", trans_size);

return -1;

}

if (file->sync_offset - file->buffer_offset + trans_size > size)

break;

/* transaction has been fully written */

if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {

if (log_file_track_mailbox_sync_offset(file, hdr,

trans_size) < 0)

return -1;

}

file->sync_offset += trans_size;

trans_size = 0;

}

if (file->mmap_base != NULL && !file->locked) {

/* Now that all the mmaped pages have page faulted, check if

if (fstat(file->fd, &st) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath,

"fstat()");

return -1;

}

if ((uoff_t)st.st_size != file->last_size) {

file->last_size = st.st_size;

return 0;

}

}

avail = file->sync_offset - file->buffer_offset;

if (avail != size) {

/* There's more data than we could sync at the moment. If the

if (trans_size != 0) {

/* pread()s or the above fstat() check for mmaps should

mail_transaction_log_file_set_corrupted(file,

"hdr.size too large (%u)", trans_size);

return -1;

} else if (file->locked) {

mail_transaction_log_file_set_corrupted(file,

"Unexpected garbage at EOF");

return -1;

}

/* The size field will be updated soon */

mail_index_flush_read_cache(file->log->index, file->filepath,

file->fd, file->locked);

}

if (file->next != NULL &&

file->hdr.file_seq == file->next->hdr.prev_file_seq &&

file->next->hdr.prev_file_offset != file->sync_offset) {

mail_index_set_error(file->log->index,

"Invalid transaction log size "

"(%"PRIuUOFF_T" vs %u): %s", file->sync_offset,

file->log->head->hdr.prev_file_offset, file->filepath);

return -1;

}

return 1;

}

static int

mail_transaction_log_file_insert_read(struct mail_transaction_log_file *file,

uoff_t offset)

{

void *data;

size_t size;

ssize_t ret;

size = file->buffer_offset - offset;

buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);

data = buffer_get_space_unsafe(file->buffer, 0, size);

ret = pread_full(file->fd, data, size, offset);

if (ret > 0) {

/* success */

file->buffer_offset -= size;

return 1;

}

/* failure. don't leave ourself to inconsistent state */

buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1);

buffer_set_used_size(file->buffer, file->buffer->used - size);

if (ret == 0) {

mail_transaction_log_file_set_corrupted(file, "file shrank");

return 0;

} else if (errno == ESTALE) {

/* log file was deleted in NFS server, fail silently */

return 0;

} else {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "pread()");

return -1;

}

}

static int

mail_transaction_log_file_read_more(struct mail_transaction_log_file *file)

{

void *data;

size_t size;

uint32_t read_offset;

ssize_t ret;

read_offset = file->buffer_offset + buffer_get_used_size(file->buffer);

do {

data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);

ret = pread(file->fd, data, LOG_PREFETCH, read_offset);

if (ret > 0)

read_offset += ret;

size = read_offset - file->buffer_offset;

buffer_set_used_size(file->buffer, size);

} while (ret > 0 || (ret < 0 && errno == EINTR));

file->last_size = read_offset;

if (ret < 0) {

if (errno == ESTALE) {

/* log file was deleted in NFS server, fail silently */

return 0;

}

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "pread()");

return -1;

}

return 1;

}

static bool

mail_transaction_log_file_need_nfs_flush(struct mail_transaction_log_file *file)

{

const struct mail_index_header *hdr = &file->log->index->map->hdr;

uoff_t max_offset = file->last_size;

if (file->next != NULL &&

file->hdr.file_seq == file->next->hdr.prev_file_seq &&

file->next->hdr.prev_file_offset != max_offset) {

/* we already have a newer log file which says that we haven't

return TRUE;

}

if (file->hdr.file_seq == hdr->log_file_seq &&

max_offset < hdr->log_file_head_offset)

return TRUE;

return FALSE;

}

static int

mail_transaction_log_file_read(struct mail_transaction_log_file *file,

uoff_t start_offset, bool nfs_flush)

{

int ret;

i_assert(file->mmap_base == NULL);

/* NFS: if file isn't locked, we're optimistic that we can read enough

if (file->log->index->nfs_flush && nfs_flush) {

if (!file->locked)

nfs_flush_attr_cache_unlocked(file->filepath);

else {

nfs_flush_attr_cache_fd_locked(file->filepath,

file->fd);

}

}

if (file->buffer != NULL && file->buffer_offset > start_offset) {

/* we have to insert missing data to beginning of buffer */

ret = mail_transaction_log_file_insert_read(file, start_offset);

if (ret <= 0)

return ret;

}

if (file->buffer == NULL) {

file->buffer =

buffer_create_dynamic(default_pool, LOG_PREFETCH);

file->buffer_offset = start_offset;

}

if ((ret = mail_transaction_log_file_read_more(file)) <= 0)

return ret;

if (file->log->index->nfs_flush && !nfs_flush &&

mail_transaction_log_file_need_nfs_flush(file)) {

/* we didn't read enough data. flush and try again. */

return mail_transaction_log_file_read(file, start_offset, TRUE);

}

if ((ret = mail_transaction_log_file_sync(file)) <= 0) {

i_assert(ret != 0); /* happens only with mmap */

return -1;

}

i_assert(file->sync_offset >= file->buffer_offset);

buffer_set_used_size(file->buffer,

file->sync_offset - file->buffer_offset);

return 1;

}

static int

log_file_map_check_offsets(struct mail_transaction_log_file *file,

uoff_t start_offset, uoff_t end_offset)

{

if (start_offset > file->sync_offset) {

/* broken start offset */

mail_index_set_error(file->log->index,

"%s: start_offset (%"PRIuUOFF_T") > "

"current sync_offset (%"PRIuUOFF_T")",

file->filepath, start_offset, file->sync_offset);

return 0;

}

if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {

mail_index_set_error(file->log->index,

"%s: end_offset (%"PRIuUOFF_T") > "

"current sync_offset (%"PRIuUOFF_T")",

file->filepath, start_offset, file->sync_offset);

return 0;

}

return 1;

}

static int

mail_transaction_log_file_mmap(struct mail_transaction_log_file *file)

{

if (file->buffer != NULL) {

/* in case we just switched to mmaping */

buffer_free(&file->buffer);

}

file->mmap_size = file->last_size;

file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, MAP_SHARED,

file->fd, 0);

if (file->mmap_base == MAP_FAILED) {

file->mmap_base = NULL;

file->mmap_size = 0;

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "mmap()");

return -1;

}

if (file->mmap_size > mmap_get_page_size()) {

if (madvise(file->mmap_base, file->mmap_size,

MADV_SEQUENTIAL) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "madvise()");

}

}

file->buffer = buffer_create_const_data(default_pool,

file->mmap_base,

file->mmap_size);

file->buffer_offset = 0;

return 0;

}

static void

mail_transaction_log_file_munmap(struct mail_transaction_log_file *file)

{

if (file->mmap_base == NULL)

return;

if (munmap(file->mmap_base, file->mmap_size) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "munmap()");

}

file->mmap_base = NULL;

file->mmap_size = 0;

buffer_free(&file->buffer);

}

static int

mail_transaction_log_file_map_mmap(struct mail_transaction_log_file *file,

uoff_t start_offset)

{

struct stat st;

int ret;

/* we are going to mmap() this file, but it's not necessarily

i_assert(file->buffer_offset == 0 || file->mmap_base == NULL);

i_assert(file->mmap_size == 0 || file->mmap_base != NULL);

if (fstat(file->fd, &st) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "fstat()");

return -1;

}

file->last_size = st.st_size;

if ((uoff_t)st.st_size < file->sync_offset) {

mail_transaction_log_file_set_corrupted(file,

"file size shrank");

return 0;

}

if (file->buffer != NULL && file->buffer_offset <= start_offset &&

(uoff_t)st.st_size == file->buffer_offset + file->buffer->used) {

/* we already have the whole file mapped */

if ((ret = mail_transaction_log_file_sync(file)) < 0)

return 0;

if (ret > 0)

return 1;

/* size changed, re-mmap */

}

do {

mail_transaction_log_file_munmap(file);

if (file->last_size - start_offset < mmap_get_page_size()) {

/* just reading the file is probably faster */

return mail_transaction_log_file_read(file,

start_offset,

FALSE);

}

if (mail_transaction_log_file_mmap(file) < 0)

return -1;

if ((ret = mail_transaction_log_file_sync(file)) < 0)

return 0;

} while (ret == 0);

return 1;

}

int mail_transaction_log_file_map(struct mail_transaction_log_file *file,

uoff_t start_offset, uoff_t end_offset)

{

struct mail_index *index = file->log->index;

size_t size;

int ret;

if (file->hdr.indexid == 0) {

/* corrupted */

return 0;

}

i_assert(start_offset >= file->hdr.hdr_size);

i_assert(start_offset <= end_offset);

if (index->log_locked && file == file->log->head &&

end_offset == (uoff_t)-1) {

/* we're not interested of going further than sync_offset */

if (log_file_map_check_offsets(file, start_offset,

end_offset) == 0)

return 0;

i_assert(start_offset <= file->sync_offset);

end_offset = file->sync_offset;

}

if (file->buffer != NULL && file->buffer_offset <= start_offset) {

/* see if we already have it */

size = buffer_get_used_size(file->buffer);

if (file->buffer_offset + size >= end_offset)

return 1;

}

if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {

if (start_offset < file->buffer_offset) {

/* we had moved the log to memory but failed to read

mail_index_set_error(index,

"%s: Beginning of the log isn't available",

file->filepath);

return 0;

}

return log_file_map_check_offsets(file, start_offset,

end_offset);

}

if (!index->mmap_disable)

ret = mail_transaction_log_file_map_mmap(file, start_offset);

else {

mail_transaction_log_file_munmap(file);

ret = mail_transaction_log_file_read(file, start_offset, FALSE);

}

return ret <= 0 ? ret :

log_file_map_check_offsets(file, start_offset, end_offset);

}

void mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file

*file)

{

buffer_t *buf;

if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))

return;

if (file->mmap_base != NULL) {

/* just copy to memory */

i_assert(file->buffer_offset == 0);

buf = buffer_create_dynamic(default_pool, file->mmap_size);

buffer_append(buf, file->mmap_base, file->mmap_size);

buffer_free(&file->buffer);

file->buffer = buf;

/* and lose the mmap */

if (munmap(file->mmap_base, file->mmap_size) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath,

"munmap()");

}

file->mmap_base = NULL;

} else if (file->buffer_offset != 0) {

/* we don't have the full log in the memory. read it. */

(void)mail_transaction_log_file_read(file, 0, FALSE);

}

if (close(file->fd) < 0) {

mail_index_file_set_syscall_error(file->log->index,

file->filepath, "close()");

}

file->fd = -1;

}