mail-transaction-log-file.c revision 0a1a4e586ced13635fc1b8f2c78c94cb35ef645a
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher/* Copyright (c) 2003-2017 Dovecot authors, see the included COPYING file */
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
ee359fe1384507fed6c2274e7bfe81d288de4542Stephen Gallagher#include "lib.h"
33396dc46ea52c18f47db1b5d590880806521005Sumit Bose#include "array.h"
ee359fe1384507fed6c2274e7bfe81d288de4542Stephen Gallagher#include "ioloop.h"
33396dc46ea52c18f47db1b5d590880806521005Sumit Bose#include "file-dotlock.h"
703dc1eb5b050b24235a6640f271d34ea008cf98Jan Engelhardt#include "nfs-workarounds.h"
703dc1eb5b050b24235a6640f271d34ea008cf98Jan Engelhardt#include "read-full.h"
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher#include "write-full.h"
324fb26ba803a999bedc29e93c46c84f27abf5b7Sumit Bose#include "mmap-util.h"
324fb26ba803a999bedc29e93c46c84f27abf5b7Sumit Bose#include "mail-index-private.h"
324fb26ba803a999bedc29e93c46c84f27abf5b7Sumit Bose#include "mail-index-modseq.h"
324fb26ba803a999bedc29e93c46c84f27abf5b7Sumit Bose#include "mail-transaction-log-private.h"
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher#define LOG_PREFETCH IO_BLOCK_SIZE
84ae5edab16ad6be5e3be956cb6fa031c1428eb5Stephen Gallagher#define MEMORY_LOG_NAME "(in-memory transaction log file)"
5ebdc2391e96cfcc86ebdb8f223e159c00a0d82bLukas Slebodnik#define LOG_NEW_DOTLOCK_SUFFIX ".newlock"
5ebdc2391e96cfcc86ebdb8f223e159c00a0d82bLukas Slebodnik
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagherstatic int
002f84aea86371aa079b867c0ec39396b97109d3Lukas Slebodnikmail_transaction_log_file_sync(struct mail_transaction_log_file *file,
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher bool *retry_r, const char **reason_r);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
700d45751e997c634504a4f22facd2edf82edea7Lukas Slebodnikstatic void
87d3b47abba6a40fcf809c85a2b138bc1013d9c5Jakub Hrozeklog_file_set_syscall_error(struct mail_transaction_log_file *file,
87d3b47abba6a40fcf809c85a2b138bc1013d9c5Jakub Hrozek const char *function)
deeadf40db3a1eec64cf030e54afc4cb8612a8d5Lukas Slebodnik{
ccf340e56364851f2e5b75e52d3d63701b662954Lukas Slebodnik mail_index_file_set_syscall_error(file->log->index,
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->filepath, function);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher}
f0ea3ed816182fadf77f3e7f7ddb298b287007adLukas Slebodnik
84ae5edab16ad6be5e3be956cb6fa031c1428eb5Stephen Gallagherstatic void
cc98edd9479d4622634a1275c98058916c14059aStephen Gallaghermail_transaction_log_mark_corrupted(struct mail_transaction_log_file *file)
ee359fe1384507fed6c2274e7bfe81d288de4542Stephen Gallagher{
cc98edd9479d4622634a1275c98058916c14059aStephen Gallagher unsigned int offset =
d3da1c165cdb4c1ec126a8f4b6b544ca415b9d20Pavel Březina offsetof(struct mail_transaction_log_header, indexid);
d3da1c165cdb4c1ec126a8f4b6b544ca415b9d20Pavel Březina int flags;
d3da1c165cdb4c1ec126a8f4b6b544ca415b9d20Pavel Březina
c481179da5d5b53ce16d8784c0bd2857ffc2f061Lukas Slebodnik if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file) ||
1183d29d87c5c7439cf2364b7d7324d4a13b6e35Stephen Gallagher file->log->index->readonly)
002f84aea86371aa079b867c0ec39396b97109d3Lukas Slebodnik return;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher /* indexid=0 marks the log file as corrupted. we opened the file with
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher O_APPEND, and now we need to drop it for pwrite() to work (at least
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher in Linux) */
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher flags = fcntl(file->fd, F_GETFL, 0);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (flags < 0) {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher mail_index_file_set_syscall_error(file->log->index,
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->filepath, "fcntl(F_GETFL)");
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher return;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (fcntl(file->fd, F_SETFL, flags & ~O_APPEND) < 0) {
c89589fa349f38214c9cb8d9389c0fd557e5dca2Simo Sorce mail_index_file_set_syscall_error(file->log->index,
c89589fa349f38214c9cb8d9389c0fd557e5dca2Simo Sorce file->filepath, "fcntl(F_SETFL)");
f775337a7d4ca1c0be8eab683d0d753cbaee49e2Lukas Slebodnik return;
f775337a7d4ca1c0be8eab683d0d753cbaee49e2Lukas Slebodnik }
f775337a7d4ca1c0be8eab683d0d753cbaee49e2Lukas Slebodnik if (pwrite_full(file->fd, &file->hdr.indexid,
86b61156743b7ebdc049450a6f88452890fd9a61Jakub Hrozek sizeof(file->hdr.indexid), offset) < 0) {
86b61156743b7ebdc049450a6f88452890fd9a61Jakub Hrozek mail_index_file_set_syscall_error(file->log->index,
86b61156743b7ebdc049450a6f88452890fd9a61Jakub Hrozek file->filepath, "pwrite()");
c89589fa349f38214c9cb8d9389c0fd557e5dca2Simo Sorce }
c89589fa349f38214c9cb8d9389c0fd557e5dca2Simo Sorce}
86b61156743b7ebdc049450a6f88452890fd9a61Jakub Hrozek
86b61156743b7ebdc049450a6f88452890fd9a61Jakub Hrozekvoid
86b61156743b7ebdc049450a6f88452890fd9a61Jakub Hrozekmail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik const char *fmt, ...)
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik{
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik va_list va;
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik file->corrupted = TRUE;
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik file->hdr.indexid = 0;
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik mail_transaction_log_mark_corrupted(file);
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik va_start(va, fmt);
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik T_BEGIN {
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik mail_index_set_error(file->log->index,
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik "Corrupted transaction log file %s seq %u: %s "
6d11fdcd8ef05000dd20b3431f8491790f99a802Lukas Slebodnik "(sync_offset=%"PRIuUOFF_T")",
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose file->filepath, file->hdr.file_seq,
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose t_strdup_vprintf(fmt, va), file->sync_offset);
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose } T_END;
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose va_end(va);
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose}
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bosestruct mail_transaction_log_file *
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bosemail_transaction_log_file_alloc(struct mail_transaction_log *log,
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose const char *path)
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose{
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose struct mail_transaction_log_file *file;
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose
5e6622722e84d594298a8324f3685a1bda2b5868Sumit Bose file = i_new(struct mail_transaction_log_file, 1);
48130eef6c5c64a07094b9e8582ba358b2048f24Jakub Hrozek file->log = log;
48130eef6c5c64a07094b9e8582ba358b2048f24Jakub Hrozek file->filepath = i_strdup(path);
48130eef6c5c64a07094b9e8582ba358b2048f24Jakub Hrozek file->fd = -1;
48130eef6c5c64a07094b9e8582ba358b2048f24Jakub Hrozek return file;
1658c567191c35beaddffafdb079abe33248037bLukas Slebodnik}
1658c567191c35beaddffafdb079abe33248037bLukas Slebodnik
1658c567191c35beaddffafdb079abe33248037bLukas Slebodnikvoid mail_transaction_log_file_free(struct mail_transaction_log_file **_file)
faa16fc9f0c9a02b26497e7cf148a92586144c08David Disseldorp{
faa16fc9f0c9a02b26497e7cf148a92586144c08David Disseldorp struct mail_transaction_log_file *file = *_file;
faa16fc9f0c9a02b26497e7cf148a92586144c08David Disseldorp struct mail_transaction_log_file **p;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher int old_errno = errno;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher *_file = NULL;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher i_assert(!file->locked);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
d921c1eba437662437847279f251a0a5d8f70127Maxim if (*p == file) {
d921c1eba437662437847279f251a0a5d8f70127Maxim *p = file->next;
d921c1eba437662437847279f251a0a5d8f70127Maxim break;
d921c1eba437662437847279f251a0a5d8f70127Maxim }
d921c1eba437662437847279f251a0a5d8f70127Maxim }
d921c1eba437662437847279f251a0a5d8f70127Maxim
d921c1eba437662437847279f251a0a5d8f70127Maxim if (file == file->log->head)
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer file->log->head = NULL;
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer if (file->buffer != NULL)
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer buffer_free(&file->buffer);
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer if (file->mmap_base != NULL) {
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer if (munmap(file->mmap_base, file->mmap_size) < 0)
327127bb7fcc07f882209f029e14026de1b23c94Maxim log_file_set_syscall_error(file, "munmap()");
327127bb7fcc07f882209f029e14026de1b23c94Maxim }
327127bb7fcc07f882209f029e14026de1b23c94Maxim
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (file->fd != -1) {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (close(file->fd) < 0)
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher log_file_set_syscall_error(file, "close()");
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher i_free(file->filepath);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher i_free(file);
eb2e21b764d03544d8161e9956d7f70b07b75f77Simo Sorce
bc9235cfb80bd64a3bfa959e8d26d5ad1be0bdf4Jakub Hrozek errno = old_errno;
bc9235cfb80bd64a3bfa959e8d26d5ad1be0bdf4Jakub Hrozek}
07d82f79d2970a08628ebf71343441ec55faa6faPavel Březina
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagherstatic void
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallaghermail_transaction_log_file_skip_to_head(struct mail_transaction_log_file *file)
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher{
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher struct mail_transaction_log *log = file->log;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher struct mail_index_map *map = log->index->map;
4b6a0d0b3d42e5fdb457f47d9adfa5e66b160256Stephen Gallagher const struct mail_index_modseq_header *modseq_hdr;
90fd1bbd6035cdab46faa3a695a2fb2be6508b17Sumit Bose uoff_t head_offset;
03713859dffacc7142393e53c73d8d4cf7dee8d5Pavel Březina
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik if (map == NULL || file->hdr.file_seq != map->hdr.log_file_seq ||
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik map->hdr.log_file_head_offset == 0)
af4ffe1001adcc0a96897e426d26444f07af9aa1Benjamin Franzke return;
1d1a0a019d8d4d9ab0f51ada03604cd2cada287eSumit Bose
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher /* we can get a valid log offset from index file. initialize
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher sync_offset from it so we don't have to read the whole log
64ea4127f463798410a2c20e0261c6b15f60257fJakub Hrozek file from beginning. */
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher head_offset = map->hdr.log_file_head_offset;
772464c842968d6e544118ae1aa7c49a7cda2ad6Stephen Gallagher
32381402a4a9afc003782c9e2301fc59c9bda2a9Yassir Elley modseq_hdr = mail_index_map_get_modseq_header(map);
068dbee9ca7bf5b37330eff91c94ae10f288d09fJakub Hrozek if (head_offset < file->hdr.hdr_size) {
d802eba25e7c1304e5036684261bcf41540532d8Nikolai Kondrashov mail_index_set_error(log->index,
98ce3c3e85a4bb2e1822bf8ab2a1c2ab9e3dd61dJakub Hrozek "%s: log_file_head_offset too small",
be65f065fef1d387281096ef095a2acef39ecc12Jakub Hrozek log->index->filepath);
e124844907ed6973915e4d56f5442ecd07535a12Jakub Hrozek file->sync_offset = file->hdr.hdr_size;
f36078af138f052cd9a30360867b0ebd0805af5eJakub Hrozek file->sync_highest_modseq = file->hdr.initial_modseq;
34c78b745eb349eef2b0f13ef2b722632aebe619Jan Cholasta } else if (modseq_hdr == NULL && file->hdr.initial_modseq == 0) {
cb4d5b588e704114b7090678752d33512baa718eJakub Hrozek /* modseqs not used yet */
e07a94a66985b674c5df11ca466792902164c4e2George McCollister file->sync_offset = head_offset;
77c0d1f6074059dafd2293f9c42ea0f9d60f8aadJakub Hrozek file->sync_highest_modseq = 0;
a2e417f38c57ed87c956ddcecf4dafca93842b65Lukas Slebodnik } else if (modseq_hdr == NULL ||
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer modseq_hdr->log_seq != file->hdr.file_seq) {
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer /* highest_modseq not synced, start from beginning */
a9c287bda3fc2a1e12cef2135ade96945f11ad01Sumit Bose file->sync_offset = file->hdr.hdr_size;
bc13c352ba9c2877f1e9bc62e55ad60fc000a55dJakub Hrozek file->sync_highest_modseq = file->hdr.initial_modseq;
b1ce544568eff89f2263ae180e323f263f1cff3aSimo Sorce } else if (modseq_hdr->log_offset > head_offset) {
b1ce544568eff89f2263ae180e323f263f1cff3aSimo Sorce mail_index_set_error(log->index,
b1ce544568eff89f2263ae180e323f263f1cff3aSimo Sorce "%s: modseq_hdr.log_offset too large",
b9c563c29243291f40489bb0dcbf3946fca72d58Jakub Hrozek log->index->filepath);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->sync_offset = file->hdr.hdr_size;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->sync_highest_modseq = file->hdr.initial_modseq;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher } else {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher /* start from where we last stopped tracking modseqs */
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->sync_offset = modseq_hdr->log_offset;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->sync_highest_modseq = modseq_hdr->highest_modseq;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (file->hdr.file_seq == log->index->map->hdr.log_file_seq) {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->saved_tail_offset =
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher log->index->map->hdr.log_file_tail_offset;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->saved_tail_sync_offset = file->saved_tail_offset;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (file->saved_tail_offset > file->max_tail_offset)
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->max_tail_offset = file->saved_tail_offset;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher}
a5077712fc8c24e8cad08207b7b5a6603bde6a7cJakub Hrozek
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagherstatic void
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallaghermail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher{
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher struct mail_transaction_log_file **p;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher const char *reason;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher bool retry;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->sync_offset = file->hdr.hdr_size;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->sync_highest_modseq = file->hdr.initial_modseq;
2a5790216f57e9bdfb2930d52860bb5300366536Jakub Hrozek mail_transaction_log_file_skip_to_head(file);
b9e5bd09a5ff7009537a18914dbebcf10498f592Sumit Bose
af4ffe1001adcc0a96897e426d26444f07af9aa1Benjamin Franzke /* insert it to correct position */
6b0a7c72bb841d6885a620c68bd51d55109b66c7Jakub Hrozek for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
9917c138d9a270deb5820915384fbde751190c2aLukas Slebodnik if ((*p)->hdr.file_seq > file->hdr.file_seq)
0e1dcef53d9d8465ce97d31ad11be4445a6e7eb8Lukas Slebodnik break;
c3889e5a101a075defe533d81f5296d5e680f639Lukas Slebodnik i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
b9c8ce2bdd4045782c243605a1b999098bedcffcNoam Meltzer }
40b2be4f4312470044cdef460b02b66003f5c85fJakub Hrozek
bf54fbed126ec3d459af40ea370ffadacd31c76dJakub Hrozek file->next = *p;
9d453f1e8b28983b363b44c49b7cd701a994fd97Nikolai Kondrashov *p = file;
29c5542feb4c45865ea61be97e0e84a1d1f04918Jakub Hrozek
0c1b38d1a86460a638fa0d97099a6eba10cfccf0Jakub Hrozek if (file->buffer != NULL) {
3fc158e59eebbc2f538fe0076a03928d0d4eab9fPavel Březina /* if we read any unfinished data, make sure the buffer gets
f0beb4e313970ffd075cd711ed6cfbac03ad5af6Christian Heimes truncated. */
f0beb4e313970ffd075cd711ed6cfbac03ad5af6Christian Heimes (void)mail_transaction_log_file_sync(file, &retry, &reason);
f0beb4e313970ffd075cd711ed6cfbac03ad5af6Christian Heimes buffer_set_used_size(file->buffer,
f0beb4e313970ffd075cd711ed6cfbac03ad5af6Christian Heimes file->sync_offset - file->buffer_offset);
b9c563c29243291f40489bb0dcbf3946fca72d58Jakub Hrozek }
bea0dc79faf609de8603cb42f190adae544bc8fbJakub Hrozek}
b9c563c29243291f40489bb0dcbf3946fca72d58Jakub Hrozek
793f2573b2beaf8b48eab850429482acf68ec2b1Pavel Březinastatic int
793f2573b2beaf8b48eab850429482acf68ec2b1Pavel Březinamail_transaction_log_init_hdr(struct mail_transaction_log *log,
793f2573b2beaf8b48eab850429482acf68ec2b1Pavel Březina struct mail_transaction_log_header *hdr)
8ccc9b7c317cf5ee8f295b38bfc4c2b7d551f8f1Lukas Slebodnik{
793f2573b2beaf8b48eab850429482acf68ec2b1Pavel Březina struct mail_index *index = log->index;
793f2573b2beaf8b48eab850429482acf68ec2b1Pavel Březina struct mail_transaction_log_file *file;
b9c563c29243291f40489bb0dcbf3946fca72d58Jakub Hrozek
b9c563c29243291f40489bb0dcbf3946fca72d58Jakub Hrozek i_assert(index->indexid != 0);
b9c563c29243291f40489bb0dcbf3946fca72d58Jakub Hrozek
b9c563c29243291f40489bb0dcbf3946fca72d58Jakub Hrozek i_zero(hdr);
b9c563c29243291f40489bb0dcbf3946fca72d58Jakub Hrozek hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION;
b32159300fea63222d8dd9200ed634087704ea74Stephen Gallagher hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION;
b32159300fea63222d8dd9200ed634087704ea74Stephen Gallagher hdr->hdr_size = sizeof(struct mail_transaction_log_header);
9dbdf62243f01f6aee41c2b5f2976c56da47f25dLukas Slebodnik hdr->indexid = log->index->indexid;
9dbdf62243f01f6aee41c2b5f2976c56da47f25dLukas Slebodnik hdr->create_stamp = ioloop_time;
9dbdf62243f01f6aee41c2b5f2976c56da47f25dLukas Slebodnik#if !WORDS_BIGENDIAN
b32159300fea63222d8dd9200ed634087704ea74Stephen Gallagher hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
9dbdf62243f01f6aee41c2b5f2976c56da47f25dLukas Slebodnik#endif
9dbdf62243f01f6aee41c2b5f2976c56da47f25dLukas Slebodnik
9dbdf62243f01f6aee41c2b5f2976c56da47f25dLukas Slebodnik if (index->fd != -1) {
b32159300fea63222d8dd9200ed634087704ea74Stephen Gallagher /* not creating index - make sure we have latest header */
9dbdf62243f01f6aee41c2b5f2976c56da47f25dLukas Slebodnik if (!index->mapping) {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (mail_index_map(index,
539b1be3507abdf8ac235b06eeed5011b0b5cde2Ondrej Kos MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
539b1be3507abdf8ac235b06eeed5011b0b5cde2Ondrej Kos return -1;
574a1c20f114851071ae74112b34488c3d1aeeb3Ondrej Kos } else {
574a1c20f114851071ae74112b34488c3d1aeeb3Ondrej Kos /* if we got here from mapping, the .log file is
574a1c20f114851071ae74112b34488c3d1aeeb3Ondrej Kos corrupted. use whatever values we got from index
574a1c20f114851071ae74112b34488c3d1aeeb3Ondrej Kos file */
2a5790216f57e9bdfb2930d52860bb5300366536Jakub Hrozek }
e6e26182d58c05d896f72f2925426658a6dc70b5Jakub Hrozek }
e6e26182d58c05d896f72f2925426658a6dc70b5Jakub Hrozek if (index->map != NULL) {
9542512d7be40f2000298c86d3d2b728f4f0f65aStephen Gallagher hdr->prev_file_seq = index->map->hdr.log_file_seq;
e157b9f6cb370e1b94bcac2044d26ad66d640fbaPavel Březina hdr->prev_file_offset = index->map->hdr.log_file_head_offset;
0c1b38d1a86460a638fa0d97099a6eba10cfccf0Jakub Hrozek hdr->file_seq = index->map->hdr.log_file_seq + 1;
e6e26182d58c05d896f72f2925426658a6dc70b5Jakub Hrozek hdr->initial_modseq =
2a5790216f57e9bdfb2930d52860bb5300366536Jakub Hrozek mail_index_map_modseq_get_highest(index->map);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher } else {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher hdr->file_seq = 1;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (hdr->initial_modseq == 0) {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher /* modseq tracking in log files is required for many reasons
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher nowadays, even if per-message modseqs aren't enabled in
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher dovecot.index. */
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher hdr->initial_modseq = 1;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek if (log->head != NULL) {
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek /* make sure the sequence always increases to avoid crashes
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek later. this catches the buggy case where two processes
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher happen to replace the same log file. */
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher for (file = log->head->next; file != NULL; file = file->next) {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (hdr->file_seq <= file->hdr.file_seq)
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek hdr->file_seq = file->hdr.file_seq + 1;
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek }
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek if (hdr->file_seq <= log->head->hdr.file_seq) {
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek /* make sure the sequence grows */
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher hdr->file_seq = log->head->hdr.file_seq+1;
6b01dae732eedee808f32a9cdd4b5656a9f839c4Jakub Hrozek }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (hdr->initial_modseq < log->head->sync_highest_modseq) {
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher /* this should be always up-to-date */
b50dffea929ee5cd0c59ba3c4822337cc162ff92Kamil Dudka hdr->initial_modseq = log->head->sync_highest_modseq;
b50dffea929ee5cd0c59ba3c4822337cc162ff92Kamil Dudka }
b50dffea929ee5cd0c59ba3c4822337cc162ff92Kamil Dudka }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher return 0;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher}
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozekstruct mail_transaction_log_file *
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallaghermail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log)
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek{
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek struct mail_transaction_log_file *file;
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek file = mail_transaction_log_file_alloc(log, MEMORY_LOG_NAME);
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) {
d9378e64499642e86989158f274372187314d5b2Lukas Slebodnik i_free(file);
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek return NULL;
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek }
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek file->buffer = buffer_create_dynamic(default_pool, 4096);
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek file->buffer_offset = sizeof(file->hdr);
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek mail_transaction_log_file_add_to_list(file);
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek return file;
6f51c802311fd81a409a26763ed45b28a3234d0dJakub Hrozek}
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozek
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagherstatic int
4d81fe27ced3d2e96866aeaf61661a925cb8edf1Jakub Hrozekmail_transaction_log_file_dotlock(struct mail_transaction_log_file *file)
8b1f525acd20f36c836e827de3c251088961c5d9Stephen Gallagher{
f5b6f977d4144c28e9c66f3f1c9d634d595d1117Marko Myllynen struct dotlock_settings dotlock_set;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher int ret;
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik if (file->log->dotlock_count > 0)
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik ret = 1;
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik else {
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik mail_transaction_log_get_dotlock_set(file->log, &dotlock_set);
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik ret = file_dotlock_create(&dotlock_set, file->filepath, 0,
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik &file->log->dotlock);
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik }
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik if (ret > 0) {
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik file->log->dotlock_count++;
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik file->locked = TRUE;
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik file->lock_created = time(NULL);
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik return 0;
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik }
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik if (ret < 0) {
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik log_file_set_syscall_error(file, "file_dotlock_create()");
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik return -1;
dc4c30bae512c0b45ff925d9e998337f8fe97e94Lukas Slebodnik }
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher mail_index_set_error(file->log->index,
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik "Timeout (%us) while waiting for "
72e60fd4eabcfbcdbfe01e8c38b94052bc6c2067Jakub Hrozek "dotlock for transaction log file %s",
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik dotlock_set.timeout, file->filepath);
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik file->log->index->index_lock_timeout = TRUE;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik return -1;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik}
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnikstatic int
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnikmail_transaction_log_file_undotlock(struct mail_transaction_log_file *file)
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik{
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik int ret;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik if (--file->log->dotlock_count > 0)
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher return 0;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik ret = file_dotlock_delete(&file->log->dotlock);
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik if (ret < 0) {
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik log_file_set_syscall_error(file, "file_dotlock_delete()");
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik return -1;
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik }
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik if (ret == 0) {
827dd342494de18099dddd0272c1a85f10703556Lukas Slebodnik mail_index_set_error(file->log->index,
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik "Dotlock was lost for transaction log file %s",
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik file->filepath);
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik return -1;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik }
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik return 0;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik}
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnikint mail_transaction_log_file_lock(struct mail_transaction_log_file *file)
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik{
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik unsigned int lock_timeout_secs;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik int ret;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik if (file->locked)
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik return 0;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik file->locked = TRUE;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik return 0;
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik }
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik return mail_transaction_log_file_dotlock(file);
4a5a18f489f4d19aa0571528a7f0c7a8d35ac83fLukas Slebodnik
9d453f1e8b28983b363b44c49b7cd701a994fd97Nikolai Kondrashov if (file->log->index->readonly) {
9d453f1e8b28983b363b44c49b7cd701a994fd97Nikolai Kondrashov mail_index_set_error(file->log->index,
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher "Index is read-only, can't write-lock %s",
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->filepath);
3d038d2e0dc7af04ec2f7c85ae325accb39f6237Jakub Hrozek return -1;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher i_assert(file->file_lock == NULL);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher lock_timeout_secs = I_MIN(MAIL_TRANSACTION_LOG_LOCK_TIMEOUT,
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->log->index->max_lock_timeout_secs);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
77c0d1f6074059dafd2293f9c42ea0f9d60f8aadJakub Hrozek F_WRLCK, lock_timeout_secs,
77c0d1f6074059dafd2293f9c42ea0f9d60f8aadJakub Hrozek &file->file_lock);
77c0d1f6074059dafd2293f9c42ea0f9d60f8aadJakub Hrozek if (ret > 0) {
77c0d1f6074059dafd2293f9c42ea0f9d60f8aadJakub Hrozek file->locked = TRUE;
77c0d1f6074059dafd2293f9c42ea0f9d60f8aadJakub Hrozek file->lock_created = time(NULL);
e07a94a66985b674c5df11ca466792902164c4e2George McCollister return 0;
e07a94a66985b674c5df11ca466792902164c4e2George McCollister }
e07a94a66985b674c5df11ca466792902164c4e2George McCollister if (ret < 0) {
bf01e8179cbb2be476805340636098deda7e1366Sumit Bose log_file_set_syscall_error(file, "mail_index_wait_lock_fd()");
0d5bb38364a6976e9c85d6349aa13a04d181a090Sumit Bose return -1;
0d5bb38364a6976e9c85d6349aa13a04d181a090Sumit Bose }
0d5bb38364a6976e9c85d6349aa13a04d181a090Sumit Bose
172c07013d1ea99447a780fd36f49d5c3a76981bJakub Hrozek mail_index_set_error(file->log->index,
9917c138d9a270deb5820915384fbde751190c2aLukas Slebodnik "Timeout (%us) while waiting for lock for "
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher "transaction log file %s%s",
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce lock_timeout_secs, file->filepath,
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce file_lock_find(file->fd, file->log->index->lock_method, F_WRLCK));
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce file->log->index->index_lock_timeout = TRUE;
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce return -1;
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce}
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce
336879aabae137f9a81304f147fb0d43001654b0Simo Sorcevoid mail_transaction_log_file_unlock(struct mail_transaction_log_file *file,
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce const char *lock_reason)
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce{
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce unsigned int lock_time;
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce
336879aabae137f9a81304f147fb0d43001654b0Simo Sorce if (!file->locked)
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik return;
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik file->locked = FALSE;
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik file->locked_sync_offset_updated = FALSE;
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik return;
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik lock_time = time(NULL) - file->lock_created;
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik if (lock_time >= MAIL_TRANSACTION_LOG_LOCK_WARN_SECS && lock_reason != NULL) {
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik i_warning("Transaction log file %s was locked for %u seconds (%s)",
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik file->filepath, lock_time, lock_reason);
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik }
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik (void)mail_transaction_log_file_undotlock(file);
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik return;
aac071824f6c98003f30d49ab440c15b4b53692cLukas Slebodnik }
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik file_unlock(&file->file_lock);
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik}
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnikstatic ssize_t
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnikmail_transaction_log_file_read_header(struct mail_transaction_log_file *file)
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik{
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik void *dest;
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik size_t pos, dest_size;
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik ssize_t ret;
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik i_assert(file->buffer == NULL && file->mmap_base == NULL);
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik i_zero(&file->hdr);
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik if (file->last_size < mmap_get_page_size() && file->last_size > 0) {
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik /* just read the entire transaction log to memory.
356eef72675cde4dc5627c1e2f1a01846ec6eb1dLukas Slebodnik note that if some of the data hasn't been fully committed
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik yet (hdr.size=0), the buffer must be truncated later */
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik file->buffer = buffer_create_dynamic(default_pool, 4096);
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik file->buffer_offset = 0;
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik dest_size = file->last_size;
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik dest = buffer_append_space_unsafe(file->buffer, dest_size);
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik } else {
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik /* read only the header */
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik dest = &file->hdr;
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik dest_size = sizeof(file->hdr);
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik }
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik /* it's not necessarily an error to read less than wanted header size,
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik since older versions of the log format used smaller headers. */
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik pos = 0;
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik do {
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik ret = pread(file->fd, PTR_OFFSET(dest, pos),
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik dest_size - pos, pos);
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik if (ret > 0)
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik pos += ret;
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik } while (ret > 0 && pos < dest_size);
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik if (file->buffer != NULL) {
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik buffer_set_used_size(file->buffer, pos);
2e505786d6d9d537f5b6631099862f6b93e2e687Lukas Slebodnik memcpy(&file->hdr, file->buffer->data,
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher I_MIN(pos, sizeof(file->hdr)));
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher return ret < 0 ? -1 : (ssize_t)pos;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher}
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagherstatic int
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallaghermail_transaction_log_file_fail_dupe(struct mail_transaction_log_file *file)
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher{
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher int ret;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher
1c7f25390572025baa6783ede14523e22fc73043Lukas Slebodnik /* mark the old file corrupted. we can't safely remove
40b2be4f4312470044cdef460b02b66003f5c85fJakub Hrozek it from the list however, so return failure. */
40b2be4f4312470044cdef460b02b66003f5c85fJakub Hrozek file->hdr.indexid = 0;
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher if (strcmp(file->filepath, file->log->head->filepath) != 0) {
29c5542feb4c45865ea61be97e0e84a1d1f04918Jakub Hrozek /* only mark .2 corrupted, just to make sure we don't lose any
29c5542feb4c45865ea61be97e0e84a1d1f04918Jakub Hrozek changes from .log in case we're somehow wrong */
29c5542feb4c45865ea61be97e0e84a1d1f04918Jakub Hrozek mail_transaction_log_mark_corrupted(file);
9d453f1e8b28983b363b44c49b7cd701a994fd97Nikolai Kondrashov ret = 0;
9d453f1e8b28983b363b44c49b7cd701a994fd97Nikolai Kondrashov } else {
8c294c1cd4d721818a59684cf7f2b36123f79163Stephen Gallagher ret = -1;
8c294c1cd4d721818a59684cf7f2b36123f79163Stephen Gallagher }
5484044ea7bb632b915f706685fce509f6eacc48Jakub Hrozek if (!file->corrupted) {
5484044ea7bb632b915f706685fce509f6eacc48Jakub Hrozek file->corrupted = TRUE;
5484044ea7bb632b915f706685fce509f6eacc48Jakub Hrozek mail_index_set_error(file->log->index,
5484044ea7bb632b915f706685fce509f6eacc48Jakub Hrozek "Transaction log %s: "
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher "duplicate transaction log sequence (%u)",
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher file->filepath, file->hdr.file_seq);
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher }
551aa6c36797ed720487f5974dcadabf19e6ff9fStephen Gallagher return ret;
a8d887323f83984679a7d9b827a70146656bb7b2Sumit Bose}
a8d887323f83984679a7d9b827a70146656bb7b2Sumit Bose
a8d887323f83984679a7d9b827a70146656bb7b2Sumit Bosestatic int
96c73559adfbdac96720008fc022cb1d540b53c3Jakub Hrozekmail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
3be9e26dcd169d44ae105f1b8a0674464c700b77Sumit Bose bool ignore_estale)
9542512d7be40f2000298c86d3d2b728f4f0f65aStephen Gallagher{
428db8a58c0c149d5efccc6d788f70916c1d34d7Jakub Hrozek struct mail_transaction_log_file *f;
9d453f1e8b28983b363b44c49b7cd701a994fd97Nikolai Kondrashov int ret;
e0c86d21388bffe2e3919e780780c40d96186abbJakub Hrozek
a6098862048d4bb469130b9ff21be3020d6f2c54Sumit Bose i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
db36dca3d45e6eefbb30042ee65876566f1a6014Sumit Bose
2a9af1f71887f02935e2fb6ad5023afba5b6d43eSumit Bose if (file->corrupted)
2a9af1f71887f02935e2fb6ad5023afba5b6d43eSumit Bose return 0;
f3c85d900c4663854cc7bbae7d9f77867ed1f69bSumit Bose
a7e27c11866a48742bb70564b88e15bf15e9367dPavel Březina ret = mail_transaction_log_file_read_header(file);
a7e27c11866a48742bb70564b88e15bf15e9367dPavel Březina if (ret < 0) {
f1ce53a3b5656361557f80f61dfd42a371230c65Stephen Gallagher if (errno != ESTALE || !ignore_estale)
6dcbfe52d5e64205c0d922f3e89add066b42c496Jakub Hrozek log_file_set_syscall_error(file, "pread()");
f1ce53a3b5656361557f80f61dfd42a371230c65Stephen Gallagher return -1;
84ae5edab16ad6be5e3be956cb6fa031c1428eb5Stephen Gallagher }
if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {
/* incompatible version - fix silently */
return 0;
}
if (ret < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
mail_transaction_log_file_set_corrupted(file,
"unexpected end of file while reading header");
return 0;
}
const unsigned int hdr_version =
MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr);
if (MAIL_TRANSACTION_LOG_VERSION_HAVE(hdr_version, COMPAT_FLAGS)) {
/* we have compatibility flags */
enum mail_index_header_compat_flags compat_flags = 0;
#if !WORDS_BIGENDIAN
compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
if (file->hdr.compat_flags != compat_flags) {
/* architecture change */
mail_index_set_error(file->log->index,
"Rebuilding index file %s: "
"CPU architecture changed",
file->log->index->filepath);
return 0;
}
}
if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
mail_transaction_log_file_set_corrupted(file,
"Header size too small");
return 0;
}
if (file->hdr.hdr_size < sizeof(file->hdr)) {
/* @UNSAFE: smaller than we expected - zero out the fields we
shouldn't have filled */
memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
sizeof(file->hdr) - file->hdr.hdr_size);
}
if (file->hdr.indexid == 0) {
/* corrupted */
file->corrupted = TRUE;
mail_index_set_error(file->log->index,
"Transaction log file %s: marked corrupted",
file->filepath);
return 0;
}
if (file->hdr.indexid != file->log->index->indexid) {
if (file->log->index->indexid != 0 &&
!file->log->index->initial_create) {
/* index file was probably just rebuilt and we don't
know about it yet */
mail_transaction_log_file_set_corrupted(file,
"indexid changed %u -> %u",
file->log->index->indexid, file->hdr.indexid);
return 0;
}
/* creating index file. since transaction log is created
first, use the indexid in it to create the main index
to avoid races. */
file->log->index->indexid = file->hdr.indexid;
}
/* make sure we already don't have a file with the same sequence
opened. it shouldn't happen unless the old log file was
corrupted. */
for (f = file->log->files; f != NULL; f = f->next) {
if (f->hdr.file_seq == file->hdr.file_seq) {
if (strcmp(f->filepath, f->log->head->filepath) != 0) {
/* old "f" is the .log.2 */
return mail_transaction_log_file_fail_dupe(f);
} else {
/* new "file" is probably the .log.2 */
return mail_transaction_log_file_fail_dupe(file);
}
}
}
file->sync_highest_modseq = file->hdr.initial_modseq;
return 1;
}
static int
mail_transaction_log_file_stat(struct mail_transaction_log_file *file,
bool ignore_estale)
{
struct stat st;
if (fstat(file->fd, &st) < 0) {
if (!ESTALE_FSTAT(errno) || !ignore_estale)
log_file_set_syscall_error(file, "fstat()");
return -1;
}
file->st_dev = st.st_dev;
file->st_ino = st.st_ino;
file->last_mtime = st.st_mtime;
file->last_size = st.st_size;
return 0;
}
static bool
mail_transaction_log_file_is_dupe(struct mail_transaction_log_file *file)
{
struct mail_transaction_log_file *tmp;
for (tmp = file->log->files; tmp != NULL; tmp = tmp->next) {
if (tmp->st_ino == file->st_ino &&
CMP_DEV_T(tmp->st_dev, file->st_dev))
return TRUE;
}
return FALSE;
}
static void log_write_ext_hdr_init_data(struct mail_index *index, buffer_t *buf)
{
const struct mail_index_registered_ext *rext;
struct mail_transaction_header *hdr;
struct mail_transaction_ext_intro *intro;
struct mail_transaction_ext_hdr_update *ext_hdr;
unsigned int hdr_offset;
rext = array_idx(&index->extensions, index->ext_hdr_init_id);
/* introduce the extension */
hdr_offset = buf->used;
hdr = buffer_append_space_unsafe(buf, sizeof(*hdr));
hdr->type = MAIL_TRANSACTION_EXT_INTRO;
intro = buffer_append_space_unsafe(buf, sizeof(*intro));
intro->ext_id = (uint32_t)-1;
intro->hdr_size = rext->hdr_size;
intro->record_size = rext->record_size;
intro->record_align = rext->record_align;
intro->name_size = strlen(rext->name);
buffer_append(buf, rext->name, intro->name_size);
if (buf->used % 4 != 0)
buffer_append_zero(buf, 4 - buf->used % 4);
hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr));
hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset);
/* add the extension header data */
hdr_offset = buf->used;
hdr = buffer_append_space_unsafe(buf, sizeof(*hdr));
hdr->type = MAIL_TRANSACTION_EXT_HDR_UPDATE;
ext_hdr = buffer_append_space_unsafe(buf, sizeof(*ext_hdr));
ext_hdr->size = rext->hdr_size;
buffer_append(buf, index->ext_hdr_init_data, rext->hdr_size);
hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr));
hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset);
}
static int
mail_transaction_log_file_create2(struct mail_transaction_log_file *file,
int new_fd, bool reset,
struct dotlock **dotlock)
{
struct mail_index *index = file->log->index;
struct stat st;
const char *path2;
buffer_t *writebuf;
int fd, ret;
bool rename_existing, need_lock;
need_lock = file->log->head != NULL && file->log->head->locked;
if (fcntl(new_fd, F_SETFL, O_APPEND) < 0) {
log_file_set_syscall_error(file, "fcntl(O_APPEND)");
return -1;
}
if (file->log->nfs_flush) {
/* although we check also mtime and file size below, it's done
only to fix broken log files. we don't bother flushing
attribute cache just for that. */
nfs_flush_file_handle_cache(file->filepath);
}
/* log creation is locked now - see if someone already created it.
note that if we're rotating, we need to keep the log locked until
the file has been rewritten. and because fcntl() locks are stupid,
if we go and open()+close() the file and we had it already opened,
its locks are lost. so we use stat() to check if the file has been
recreated, although it almost never is. */
if (reset)
rename_existing = FALSE;
else if (nfs_safe_stat(file->filepath, &st) < 0) {
if (errno != ENOENT) {
log_file_set_syscall_error(file, "stat()");
return -1;
}
rename_existing = FALSE;
} else if (st.st_ino == file->st_ino &&
CMP_DEV_T(st.st_dev, file->st_dev) &&
/* inode/dev checks are enough when we're rotating the file,
but not when we're replacing a broken log file */
st.st_mtime == file->last_mtime &&
(uoff_t)st.st_size == file->last_size) {
/* no-one else recreated the file */
rename_existing = TRUE;
} else {
/* recreated. use the file if its header is ok */
fd = nfs_safe_open(file->filepath, O_RDWR | O_APPEND);
if (fd == -1) {
if (errno != ENOENT) {
log_file_set_syscall_error(file, "open()");
return -1;
}
} else {
file->fd = fd;
file->last_size = 0;
if (mail_transaction_log_file_read_hdr(file,
FALSE) > 0 &&
mail_transaction_log_file_stat(file, FALSE) == 0) {
/* yes, it was ok */
file_dotlock_delete(dotlock);
mail_transaction_log_file_add_to_list(file);
return 0;
}
file->fd = -1;
if (close(fd) < 0)
log_file_set_syscall_error(file, "close()");
}
rename_existing = FALSE;
}
if (index->fd == -1 && !rename_existing) {
/* creating the initial index */
reset = TRUE;
}
if (mail_transaction_log_init_hdr(file->log, &file->hdr) < 0)
return -1;
if (reset) {
/* don't reset modseqs. if we're reseting due to rebuilding
indexes we'll probably want to keep uidvalidity and in such
cases we really don't want to shrink modseqs. */
file->hdr.prev_file_seq = 0;
file->hdr.prev_file_offset = 0;
}
writebuf = buffer_create_dynamic(pool_datastack_create(), 128);
buffer_append(writebuf, &file->hdr, sizeof(file->hdr));
if (index->ext_hdr_init_data != NULL && reset)
log_write_ext_hdr_init_data(index, writebuf);
if (write_full(new_fd, writebuf->data, writebuf->used) < 0) {
log_file_set_syscall_error(file, "write_full()");
return -1;
}
if (file->log->index->fsync_mode == FSYNC_MODE_ALWAYS) {
/* the header isn't important, so don't bother calling
fdatasync() unless it's required */
if (fdatasync(new_fd) < 0) {
log_file_set_syscall_error(file, "fdatasync()");
return -1;
}
}
file->fd = new_fd;
ret = mail_transaction_log_file_stat(file, FALSE);
if (need_lock && ret == 0) {
/* we'll need to preserve the lock */
if (mail_transaction_log_file_lock(file) < 0)
ret = -1;
}
/* if we return -1 the dotlock deletion code closes the fd */
file->fd = -1;
if (ret < 0)
return -1;
/* keep two log files */
if (rename_existing) {
/* rename() would be nice and easy way to do this, except then
there's a race condition between the rename and
file_dotlock_replace(). during that time the log file
doesn't exist, which could cause problems. */
path2 = t_strconcat(file->filepath, ".2", NULL);
if (i_unlink_if_exists(path2) < 0) {
/* try to link() anyway */
}
if (nfs_safe_link(file->filepath, path2, FALSE) < 0 &&
errno != ENOENT && errno != EEXIST) {
mail_index_set_error(index, "link(%s, %s) failed: %m",
file->filepath, path2);
/* ignore the error. we don't care that much about the
second log file and we're going to overwrite this
first one. */
}
/* NOTE: here's a race condition where both .log and .log.2
point to the same file. our reading code should ignore that
though by comparing the inodes. */
}
if (file_dotlock_replace(dotlock,
DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) <= 0) {
/* need to unlock to avoid assert-crash in
mail_transaction_log_file_free() */
mail_transaction_log_file_unlock(file, "creation failed");
return -1;
}
/* success */
file->fd = new_fd;
mail_transaction_log_file_add_to_list(file);
i_assert(!need_lock || file->locked);
return 1;
}
int mail_transaction_log_file_create(struct mail_transaction_log_file *file,
bool reset)
{
struct mail_index *index = file->log->index;
struct dotlock_settings new_dotlock_set;
struct dotlock *dotlock;
mode_t old_mask;
int fd, ret;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
if (file->log->index->readonly) {
mail_index_set_error(index,
"Can't create log file %s: Index is read-only",
file->filepath);
return -1;
}
if (index->indexid == 0) {
mail_index_set_error(index,
"Can't create log file %s: Index is marked corrupted",
file->filepath);
return -1;
}
mail_transaction_log_get_dotlock_set(file->log, &new_dotlock_set);
new_dotlock_set.lock_suffix = LOG_NEW_DOTLOCK_SUFFIX;
/* With dotlocking we might already have path.lock created, so this
filename has to be different. */
old_mask = umask(index->mode ^ 0666);
fd = file_dotlock_open(&new_dotlock_set, file->filepath, 0, &dotlock);
umask(old_mask);
if (fd == -1) {
log_file_set_syscall_error(file, "file_dotlock_open()");
return -1;
}
mail_index_fchown(index, fd, file_dotlock_get_lock_path(dotlock));
/* either fd gets used or the dotlock gets deleted and returned fd
is for the existing file */
ret = mail_transaction_log_file_create2(file, fd, reset, &dotlock);
if (ret < 0) {
if (dotlock != NULL)
file_dotlock_delete(&dotlock);
return -1;
}
return ret;
}
int mail_transaction_log_file_open(struct mail_transaction_log_file *file,
const char **reason_r)
{
struct mail_index *index = file->log->index;
unsigned int i;
bool ignore_estale;
int ret;
for (i = 0;; i++) {
if (!index->readonly) {
file->fd = nfs_safe_open(file->filepath,
O_RDWR | O_APPEND);
} else {
file->fd = nfs_safe_open(file->filepath, O_RDONLY);
}
if (file->fd == -1 && errno == EACCES) {
file->fd = nfs_safe_open(file->filepath, O_RDONLY);
index->readonly = TRUE;
}
if (file->fd == -1) {
if (errno == ENOENT) {
*reason_r = "File doesn't exist";
return 0;
}
log_file_set_syscall_error(file, "open()");
*reason_r = t_strdup_printf("open() failed: %m");
return -1;
}
ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
if (mail_transaction_log_file_stat(file, ignore_estale) < 0)
ret = -1;
else if (mail_transaction_log_file_is_dupe(file)) {
/* probably our already opened .log file has been
renamed to .log.2 and we're trying to reopen it.
also possible that hit a race condition where .log
and .log.2 are linked. */
*reason_r = "File is already open";
return 0;
} else {
ret = mail_transaction_log_file_read_hdr(file,
ignore_estale);
}
if (ret > 0) {
/* success */
break;
}
if (ret == 0) {
/* corrupted */
if (index->readonly) {
/* don't delete */
} else {
i_unlink_if_exists(file->filepath);
}
*reason_r = "File is corrupted";
return 0;
}
if (errno != ESTALE ||
i == MAIL_INDEX_ESTALE_RETRY_COUNT) {
/* syscall error */
*reason_r = t_strdup_printf("fstat() failed: %m");
return -1;
}
/* ESTALE - try again */
if (file->buffer != NULL)
buffer_free(&file->buffer);
}
mail_transaction_log_file_add_to_list(file);
return 1;
}
static int
log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
const void *data, unsigned int trans_size,
const char **error_r)
{
const struct mail_transaction_header_update *u = data;
const struct mail_index_header *ihdr;
const unsigned int size = trans_size - sizeof(struct mail_transaction_header);
const unsigned int offset_pos =
offsetof(struct mail_index_header, log_file_tail_offset);
const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);
uint32_t tail_offset;
i_assert(offset_size == sizeof(tail_offset));
if (size < sizeof(*u) || size < sizeof(*u) + u->size) {
*error_r = "header update extends beyond record size";
mail_transaction_log_file_set_corrupted(file, "%s", *error_r);
return -1;
}
if (u->offset <= offset_pos &&
u->offset + u->size >= offset_pos + offset_size) {
memcpy(&tail_offset,
CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),
sizeof(tail_offset));
if (tail_offset < file->saved_tail_offset) {
/* ignore shrinking tail offsets */
return 1;
} else if (tail_offset > file->sync_offset + trans_size) {
mail_transaction_log_file_set_corrupted(file,
"log_file_tail_offset %u goes past sync offset %"PRIuUOFF_T,
tail_offset, file->sync_offset + trans_size);
} else {
file->saved_tail_offset = tail_offset;
if (tail_offset > file->max_tail_offset)
file->max_tail_offset = tail_offset;
return 1;
}
}
return 0;
}
static bool
flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
unsigned int count, unsigned int version)
{
const uint8_t internal_flags =
MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
/* Hide internal flags from modseqs if the log file's version
is new enough. This allows upgrading without the modseqs suddenly
shrinking. */
if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(version, HIDE_INTERNAL_MODSEQS))
return TRUE;
for (unsigned int i = 0; i < count; i++) {
uint8_t changed_flags = u->add_flags | u->remove_flags;
if ((changed_flags & ~internal_flags) != 0)
return TRUE;
}
return FALSE;
}
void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
const void *data, uint64_t *cur_modseq,
unsigned int version)
{
uint32_t trans_size;
trans_size = mail_index_offset_to_uint32(hdr->size);
i_assert(trans_size != 0);
if (*cur_modseq != 0) {
/* tracking modseqs */
} else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
MAIL_TRANSACTION_EXT_INTRO) {
/* modseqs not tracked yet. see if this is a modseq
extension introduction. */
const struct mail_transaction_ext_intro *intro = data;
const unsigned int modseq_ext_len =
strlen(MAIL_INDEX_MODSEQ_EXT_NAME);
if (intro->name_size == modseq_ext_len &&
memcmp(intro + 1, MAIL_INDEX_MODSEQ_EXT_NAME,
modseq_ext_len) == 0) {
/* modseq tracking started */
*cur_modseq += 1;
}
return;
} else {
/* not tracking modseqs */
return;
}
/* NOTE: keep in sync with mail_index_transaction_get_highest_modseq() */
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT:
case MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT:
if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
/* ignore expunge requests */
break;
}
case MAIL_TRANSACTION_APPEND:
case MAIL_TRANSACTION_KEYWORD_UPDATE:
case MAIL_TRANSACTION_KEYWORD_RESET:
case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
/* these changes increase modseq */
*cur_modseq += 1;
break;
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *rec = data;
unsigned int count;
count = (trans_size - sizeof(*hdr)) / sizeof(*rec);
if (flag_updates_have_non_internal(rec, count, version))
*cur_modseq += 1;
break;
}
case MAIL_TRANSACTION_MODSEQ_UPDATE: {
const struct mail_transaction_modseq_update *rec, *end;
end = CONST_PTR_OFFSET(data, trans_size - sizeof(*hdr));
for (rec = data; rec < end; rec++) {
uint64_t modseq = ((uint64_t)rec->modseq_high32 << 32) |
rec->modseq_low32;
if (*cur_modseq < modseq)
*cur_modseq = modseq;
}
}
}
}
static struct modseq_cache *
modseq_cache_hit(struct mail_transaction_log_file *file, unsigned int idx)
{
struct modseq_cache cache;
if (idx > 0) {
/* @UNSAFE: move it to top */
cache = file->modseq_cache[idx];
memmove(file->modseq_cache + 1, file->modseq_cache,
sizeof(*file->modseq_cache) * idx);
file->modseq_cache[0] = cache;
}
return &file->modseq_cache[0];
}
static struct modseq_cache *
modseq_cache_get_offset(struct mail_transaction_log_file *file, uoff_t offset)
{
unsigned int i, best = UINT_MAX;
for (i = 0; i < N_ELEMENTS(file->modseq_cache); i++) {
if (offset < file->modseq_cache[i].offset)
continue;
if (file->modseq_cache[i].offset == 0)
return NULL;
if (offset == file->modseq_cache[i].offset) {
/* exact cache hit */
return modseq_cache_hit(file, i);
}
if (best == UINT_MAX ||
file->modseq_cache[i].offset <
file->modseq_cache[best].offset)
best = i;
}
if (best == UINT_MAX)
return NULL;
return &file->modseq_cache[best];
}
static struct modseq_cache *
modseq_cache_get_modseq(struct mail_transaction_log_file *file, uint64_t modseq)
{
unsigned int i, best = UINT_MAX;
for (i = 0; i < N_ELEMENTS(file->modseq_cache); i++) {
if (modseq < file->modseq_cache[i].highest_modseq)
continue;
if (file->modseq_cache[i].offset == 0)
return NULL;
if (modseq == file->modseq_cache[i].highest_modseq) {
/* exact cache hit */
return modseq_cache_hit(file, i);
}
if (best == UINT_MAX ||
file->modseq_cache[i].highest_modseq <
file->modseq_cache[best].highest_modseq)
best = i;
}
if (best == UINT_MAX)
return NULL;
return &file->modseq_cache[best];
}
static int
log_get_synced_record(struct mail_transaction_log_file *file, uoff_t *offset,
const struct mail_transaction_header **hdr_r,
const char **error_r)
{
const struct mail_transaction_header *hdr;
uint32_t trans_size;
hdr = CONST_PTR_OFFSET(file->buffer->data,
*offset - file->buffer_offset);
/* we've already synced this record at some point. it should
be valid. */
trans_size = mail_index_offset_to_uint32(hdr->size);
if (trans_size < sizeof(*hdr) ||
*offset - file->buffer_offset + trans_size > file->buffer->used) {
*error_r = t_strdup_printf(
"Transaction log corrupted unexpectedly at "
"%"PRIuUOFF_T": Invalid size %u (type=%x)",
*offset, trans_size, hdr->type);
mail_transaction_log_file_set_corrupted(file, "%s", *error_r);
return -1;
}
*offset += trans_size;
*hdr_r = hdr;
return 0;
}
int mail_transaction_log_file_get_highest_modseq_at(
struct mail_transaction_log_file *file,
uoff_t offset, uint64_t *highest_modseq_r,
const char **error_r)
{
const struct mail_transaction_header *hdr;
struct modseq_cache *cache;
uoff_t cur_offset;
uint64_t cur_modseq;
const char *reason;
int ret;
i_assert(offset <= file->sync_offset);
if (offset == file->sync_offset) {
*highest_modseq_r = file->sync_highest_modseq;
return 0;
}
cache = modseq_cache_get_offset(file, offset);
if (cache == NULL) {
/* nothing usable in cache - scan from beginning */
cur_offset = file->hdr.hdr_size;
cur_modseq = file->hdr.initial_modseq;
} else if (cache->offset == offset) {
/* exact cache hit */
*highest_modseq_r = cache->highest_modseq;
return 0;
} else {
/* use cache to skip over some records */
cur_offset = cache->offset;
cur_modseq = cache->highest_modseq;
}
ret = mail_transaction_log_file_map(file, cur_offset, offset, &reason);
if (ret <= 0) {
*error_r = t_strdup_printf(
"Failed to map transaction log %s for getting modseq "
"at offset=%"PRIuUOFF_T" with start_offset=%"PRIuUOFF_T": %s",
file->filepath, offset, cur_offset, reason);
return -1;
}
i_assert(cur_offset >= file->buffer_offset);
i_assert(cur_offset + file->buffer->used >= offset);
while (cur_offset < offset) {
if (log_get_synced_record(file, &cur_offset, &hdr, error_r) < 0)
return- 1;
mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
}
/* @UNSAFE: cache the value */
memmove(file->modseq_cache + 1, file->modseq_cache,
sizeof(*file->modseq_cache) *
(N_ELEMENTS(file->modseq_cache) - 1));
file->modseq_cache[0].offset = cur_offset;
file->modseq_cache[0].highest_modseq = cur_modseq;
*highest_modseq_r = cur_modseq;
return 0;
}
int mail_transaction_log_file_get_modseq_next_offset(
struct mail_transaction_log_file *file,
uint64_t modseq, uoff_t *next_offset_r)
{
const struct mail_transaction_header *hdr;
struct modseq_cache *cache;
uoff_t cur_offset;
uint64_t cur_modseq;
const char *reason;
int ret;
if (modseq == file->sync_highest_modseq) {
*next_offset_r = file->sync_offset;
return 0;
}
if (modseq == file->hdr.initial_modseq) {
*next_offset_r = file->hdr.hdr_size;
return 0;
}
cache = modseq_cache_get_modseq(file, modseq);
if (cache == NULL) {
/* nothing usable in cache - scan from beginning */
cur_offset = file->hdr.hdr_size;
cur_modseq = file->hdr.initial_modseq;
} else if (cache->highest_modseq == modseq) {
/* exact cache hit */
*next_offset_r = cache->offset;
return 0;
} else {
/* use cache to skip over some records */
cur_offset = cache->offset;
cur_modseq = cache->highest_modseq;
}
/* make sure we've read until end of file. this is especially important
with non-head logs which might only have been opened without being
synced. */
ret = mail_transaction_log_file_map(file, cur_offset, (uoff_t)-1, &reason);
if (ret <= 0) {
mail_index_set_error(file->log->index,
"Failed to map transaction log %s for getting offset "
"for modseq=%llu with start_offset=%"PRIuUOFF_T": %s",
file->filepath, (unsigned long long)modseq,
cur_offset, reason);
return -1;
}
/* check sync_highest_modseq again in case sync_offset was updated */
if (modseq >= file->sync_highest_modseq) {
*next_offset_r = file->sync_offset;
return 0;
}
i_assert(cur_offset >= file->buffer_offset);
while (cur_offset < file->sync_offset) {
if (log_get_synced_record(file, &cur_offset, &hdr, &reason) < 0) {
mail_index_set_error(file->log->index,
"%s: %s", file->filepath, reason);
return -1;
}
mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
if (cur_modseq >= modseq)
break;
}
if (cur_offset == file->sync_offset) {
/* if we got to sync_offset, cur_modseq should be
sync_highest_modseq */
mail_index_set_error(file->log->index,
"%s: Transaction log changed unexpectedly, "
"can't get modseq", file->filepath);
return -1;
}
/* @UNSAFE: cache the value */
memmove(file->modseq_cache + 1, file->modseq_cache,
sizeof(*file->modseq_cache) *
(N_ELEMENTS(file->modseq_cache) - 1));
file->modseq_cache[0].offset = cur_offset;
file->modseq_cache[0].highest_modseq = cur_modseq;
*next_offset_r = cur_offset;
return 0;
}
static int
log_file_track_sync(struct mail_transaction_log_file *file,
const struct mail_transaction_header *hdr,
unsigned int trans_size, const char **error_r)
{
const void *data = hdr + 1;
int ret;
mail_transaction_update_modseq(hdr, hdr + 1, &file->sync_highest_modseq,
MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
return 1;
/* external transactions: */
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_HEADER_UPDATE:
/* see if this updates mailbox_sync_offset */
ret = log_file_track_mailbox_sync_offset_hdr(file, data,
trans_size, error_r);
if (ret != 0)
return ret < 0 ? -1 : 1;
break;
case MAIL_TRANSACTION_INDEX_DELETED:
if (file->sync_offset < file->index_undeleted_offset)
break;
file->log->index->index_deleted = TRUE;
file->log->index->index_delete_requested = FALSE;
file->index_deleted_offset = file->sync_offset + trans_size;
break;
case MAIL_TRANSACTION_INDEX_UNDELETED:
if (file->sync_offset < file->index_deleted_offset)
break;
file->log->index->index_deleted = FALSE;
file->log->index->index_delete_requested = FALSE;
file->index_undeleted_offset = file->sync_offset + trans_size;
break;
case MAIL_TRANSACTION_BOUNDARY: {
const struct mail_transaction_boundary *boundary =
(const void *)(hdr + 1);
size_t wanted_buffer_size;
wanted_buffer_size = file->sync_offset - file->buffer_offset +
boundary->size;
if (wanted_buffer_size > file->buffer->used) {
/* the full transaction hasn't been written yet */
return 0;
}
break;
}
}
if (file->max_tail_offset == file->sync_offset) {
/* external transactions aren't synced to mailbox. we can
update mailbox sync offset to skip this transaction to
avoid re-reading it at the next sync. */
file->max_tail_offset += trans_size;
}
return 1;
}
static int
mail_transaction_log_file_sync(struct mail_transaction_log_file *file,
bool *retry_r, const char **reason_r)
{
const struct mail_transaction_header *hdr;
const void *data;
struct stat st;
size_t size, avail;
uint32_t trans_size = 0;
int ret;
i_assert(file->sync_offset >= file->buffer_offset);
*retry_r = FALSE;
data = buffer_get_data(file->buffer, &size);
if (file->buffer_offset + size < file->sync_offset) {
*reason_r = t_strdup_printf(
"log file shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")",
file->buffer_offset + (uoff_t)size, file->sync_offset);
mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
/* fix the sync_offset to avoid crashes later on */
file->sync_offset = file->buffer_offset + size;
return 0;
}
while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {
hdr = CONST_PTR_OFFSET(data, file->sync_offset -
file->buffer_offset);
trans_size = mail_index_offset_to_uint32(hdr->size);
if (trans_size == 0) {
/* unfinished */
return 1;
}
if (trans_size < sizeof(*hdr)) {
*reason_r = t_strdup_printf(
"hdr.size too small (%u)", trans_size);
mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
return 0;
}
if (file->sync_offset - file->buffer_offset + trans_size > size)
break;
/* transaction has been fully written */
if ((ret = log_file_track_sync(file, hdr, trans_size, reason_r)) <= 0) {
if (ret < 0)
return 0;
break;
}
file->sync_offset += trans_size;
}
if (file->mmap_base != NULL && !file->locked) {
/* Now that all the mmaped pages have page faulted, check if
the file had changed while doing that. Only after the last
page has faulted, the size returned by fstat() can be
trusted. Otherwise it might point to a page boundary while
the next page is still being written.
Without this check we might see partial transactions,
sometimes causing "Extension record updated without intro
prefix" errors. */
if (fstat(file->fd, &st) < 0) {
log_file_set_syscall_error(file, "fstat()");
*reason_r = t_strdup_printf("fstat() failed: %m");
return -1;
}
if ((uoff_t)st.st_size != file->last_size) {
file->last_size = st.st_size;
*retry_r = TRUE;
*reason_r = "File size changed - retrying";
return 0;
}
}
avail = file->sync_offset - file->buffer_offset;
if (avail != size) {
/* There's more data than we could sync at the moment. If the
last record's size wasn't valid, we can't know if it will
be updated unless we've locked the log. */
if (file->locked) {
*reason_r = "Unexpected garbage at EOF";
mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
return 0;
}
/* The size field will be updated soon */
mail_index_flush_read_cache(file->log->index, file->filepath,
file->fd, file->locked);
}
if (file->next != NULL &&
file->hdr.file_seq == file->next->hdr.prev_file_seq &&
file->next->hdr.prev_file_offset != file->sync_offset) {
*reason_r = t_strdup_printf(
"Invalid transaction log size "
"(%"PRIuUOFF_T" vs %u): %s", file->sync_offset,
file->log->head->hdr.prev_file_offset, file->filepath);
mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
return 0;
}
return 1;
}
static int
mail_transaction_log_file_insert_read(struct mail_transaction_log_file *file,
uoff_t offset, const char **reason_r)
{
void *data;
size_t size;
ssize_t ret;
size = file->buffer_offset - offset;
buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
data = buffer_get_space_unsafe(file->buffer, 0, size);
ret = pread_full(file->fd, data, size, offset);
if (ret > 0) {
/* success */
file->buffer_offset -= size;
return 1;
}
/* failure. don't leave ourself to inconsistent state */
buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1);
buffer_set_used_size(file->buffer, file->buffer->used - size);
if (ret == 0) {
*reason_r = "file shrank unexpectedly";
mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
return 0;
} else if (errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
*reason_r = t_strdup_printf("read() failed: %m");
return 0;
} else {
log_file_set_syscall_error(file, "pread()");
*reason_r = t_strdup_printf("read() failed: %m");
return -1;
}
}
static int
mail_transaction_log_file_read_more(struct mail_transaction_log_file *file,
const char **reason_r)
{
void *data;
size_t size;
uint32_t read_offset;
ssize_t ret;
read_offset = file->buffer_offset + file->buffer->used;
do {
data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH);
ret = pread(file->fd, data, LOG_PREFETCH, read_offset);
if (ret > 0)
read_offset += ret;
size = read_offset - file->buffer_offset;
buffer_set_used_size(file->buffer, size);
} while (ret > 0 || (ret < 0 && errno == EINTR));
file->last_size = read_offset;
if (ret < 0) {
*reason_r = t_strdup_printf("pread() failed: %m");
if (errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
return 0;
}
log_file_set_syscall_error(file, "pread()");
return -1;
}
return 1;
}
static bool
mail_transaction_log_file_need_nfs_flush(struct mail_transaction_log_file *file)
{
const struct mail_index_header *hdr = &file->log->index->map->hdr;
uoff_t max_offset = file->last_size;
if (file->next != NULL &&
file->hdr.file_seq == file->next->hdr.prev_file_seq &&
file->next->hdr.prev_file_offset != max_offset) {
/* we already have a newer log file which says that we haven't
synced the entire file. */
return TRUE;
}
if (file->hdr.file_seq == hdr->log_file_seq &&
max_offset < hdr->log_file_head_offset)
return TRUE;
return FALSE;
}
static int
mail_transaction_log_file_read(struct mail_transaction_log_file *file,
uoff_t start_offset, bool nfs_flush,
const char **reason_r)
{
bool retry;
int ret;
i_assert(file->mmap_base == NULL);
/* NFS: if file isn't locked, we're optimistic that we can read enough
data without flushing attribute cache. if after reading we notice
that we really should have read more, flush the cache and try again.
if file is locked, the attribute cache was already flushed when
refreshing the log. */
if (file->log->nfs_flush && nfs_flush) {
if (!file->locked)
nfs_flush_attr_cache_unlocked(file->filepath);
else
nfs_flush_attr_cache_fd_locked(file->filepath, file->fd);
}
if (file->buffer != NULL && file->buffer_offset > start_offset) {
/* we have to insert missing data to beginning of buffer */
ret = mail_transaction_log_file_insert_read(file, start_offset, reason_r);
if (ret <= 0)
return ret;
}
if (file->buffer == NULL) {
file->buffer =
buffer_create_dynamic(default_pool, LOG_PREFETCH);
file->buffer_offset = start_offset;
}
if ((ret = mail_transaction_log_file_read_more(file, reason_r)) <= 0)
;
else if (file->log->nfs_flush && !nfs_flush &&
mail_transaction_log_file_need_nfs_flush(file)) {
/* we didn't read enough data. flush and try again. */
return mail_transaction_log_file_read(file, start_offset, TRUE, reason_r);
} else if ((ret = mail_transaction_log_file_sync(file, &retry, reason_r)) == 0) {
i_assert(!retry); /* retry happens only with mmap */
}
i_assert(file->sync_offset >= file->buffer_offset);
buffer_set_used_size(file->buffer,
file->sync_offset - file->buffer_offset);
return ret;
}
static bool
log_file_map_check_offsets(struct mail_transaction_log_file *file,
uoff_t start_offset, uoff_t end_offset,
const char **reason_r)
{
struct stat st, st2;
if (start_offset > file->sync_offset) {
/* broken start offset */
if (fstat(file->fd, &st) < 0) {
log_file_set_syscall_error(file, "fstat()");
st.st_size = -1;
}
*reason_r = t_strdup_printf(
"%s: start_offset (%"PRIuUOFF_T") > "
"current sync_offset (%"PRIuUOFF_T"), file size=%"PRIuUOFF_T,
file->filepath, start_offset, file->sync_offset,
st.st_size);
if (stat(file->filepath, &st2) == 0) {
if (st.st_ino != st2.st_ino) {
*reason_r = t_strdup_printf(
"%s, file unexpectedly replaced", *reason_r);
}
} else if (errno == ENOENT) {
*reason_r = t_strdup_printf(
"%s, file unexpectedly deleted", *reason_r);
} else {
log_file_set_syscall_error(file, "stat()");
}
return FALSE;
}
if (end_offset != (uoff_t)-1 && end_offset > file->sync_offset) {
*reason_r = t_strdup_printf(
"%s: end_offset (%"PRIuUOFF_T") > "
"current sync_offset (%"PRIuUOFF_T")",
file->filepath, start_offset, file->sync_offset);
return FALSE;
}
return TRUE;
}
static int
mail_transaction_log_file_mmap(struct mail_transaction_log_file *file,
const char **reason_r)
{
if (file->buffer != NULL) {
/* in case we just switched to mmaping */
buffer_free(&file->buffer);
}
file->mmap_size = file->last_size;
file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, MAP_SHARED,
file->fd, 0);
if (file->mmap_base == MAP_FAILED) {
file->mmap_base = NULL;
if (ioloop_time != file->last_mmap_error_time) {
file->last_mmap_error_time = ioloop_time;
log_file_set_syscall_error(file, t_strdup_printf(
"mmap(size=%"PRIuSIZE_T")", file->mmap_size));
}
*reason_r = t_strdup_printf("mmap(size=%"PRIuSIZE_T") failed: %m",
file->mmap_size);
file->mmap_size = 0;
return -1;
}
if (file->mmap_size > mmap_get_page_size()) {
if (madvise(file->mmap_base, file->mmap_size,
MADV_SEQUENTIAL) < 0)
log_file_set_syscall_error(file, "madvise()");
}
buffer_create_from_const_data(&file->mmap_buffer,
file->mmap_base, file->mmap_size);
file->buffer = &file->mmap_buffer;
file->buffer_offset = 0;
return 0;
}
static void
mail_transaction_log_file_munmap(struct mail_transaction_log_file *file)
{
if (file->mmap_base == NULL)
return;
i_assert(file->buffer != NULL);
if (munmap(file->mmap_base, file->mmap_size) < 0)
log_file_set_syscall_error(file, "munmap()");
file->mmap_base = NULL;
file->mmap_size = 0;
buffer_free(&file->buffer);
}
static int
mail_transaction_log_file_map_mmap(struct mail_transaction_log_file *file,
uoff_t start_offset, const char **reason_r)
{
struct stat st;
bool retry;
int ret;
/* we are going to mmap() this file, but it's not necessarily
mmaped currently. */
i_assert(file->buffer_offset == 0 || file->mmap_base == NULL);
i_assert(file->mmap_size == 0 || file->mmap_base != NULL);
if (fstat(file->fd, &st) < 0) {
log_file_set_syscall_error(file, "fstat()");
*reason_r = t_strdup_printf("fstat() failed: %m");
return -1;
}
file->last_size = st.st_size;
if ((uoff_t)st.st_size < file->sync_offset) {
*reason_r = t_strdup_printf(
"file size shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")",
(uoff_t)st.st_size, file->sync_offset);
mail_transaction_log_file_set_corrupted(file, "%s", *reason_r);
return 0;
}
if (file->buffer != NULL && file->buffer_offset <= start_offset &&
(uoff_t)st.st_size == file->buffer_offset + file->buffer->used) {
/* we already have the whole file mapped */
if ((ret = mail_transaction_log_file_sync(file, &retry, reason_r)) != 0 ||
!retry)
return ret;
/* size changed, re-mmap */
}
do {
mail_transaction_log_file_munmap(file);
if (file->last_size - start_offset < mmap_get_page_size()) {
/* just reading the file is probably faster */
return mail_transaction_log_file_read(file,
start_offset,
FALSE, reason_r);
}
if (mail_transaction_log_file_mmap(file, reason_r) < 0)
return -1;
ret = mail_transaction_log_file_sync(file, &retry, reason_r);
} while (retry);
return ret;
}
int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
uoff_t start_offset, uoff_t end_offset,
const char **reason_r)
{
uoff_t map_start_offset = start_offset;
size_t size;
int ret;
if (file->hdr.indexid == 0) {
/* corrupted */
*reason_r = "corrupted, indexid=0";
return 0;
}
i_assert(start_offset >= file->hdr.hdr_size);
i_assert(start_offset <= end_offset);
i_assert(file->buffer == NULL || file->mmap_base != NULL ||
file->sync_offset >= file->buffer_offset + file->buffer->used);
if (file->locked_sync_offset_updated && file == file->log->head &&
end_offset == (uoff_t)-1) {
/* we're not interested of going further than sync_offset */
if (!log_file_map_check_offsets(file, start_offset,
end_offset, reason_r))
return 0;
i_assert(start_offset <= file->sync_offset);
end_offset = file->sync_offset;
}
if (file->buffer != NULL && file->buffer_offset <= start_offset) {
/* see if we already have it */
size = file->buffer->used;
if (file->buffer_offset + size >= end_offset)
return 1;
}
if (file->locked) {
/* set this only when we've synced to end of file while locked
(either end_offset=(uoff_t)-1 or we had to read anyway) */
file->locked_sync_offset_updated = TRUE;
}
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
if (start_offset < file->buffer_offset || file->buffer == NULL) {
/* we had moved the log to memory but failed to read
the beginning of the log file */
*reason_r = "Beginning of the log isn't available";
return 0;
}
return log_file_map_check_offsets(file, start_offset,
end_offset, reason_r) ? 1 : 0;
}
if (start_offset > file->sync_offset)
mail_transaction_log_file_skip_to_head(file);
if (start_offset > file->sync_offset) {
/* although we could just skip over the unwanted data, we have
to sync everything so that modseqs are calculated
correctly */
map_start_offset = file->sync_offset;
}
if ((file->log->index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0)
ret = mail_transaction_log_file_map_mmap(file, map_start_offset, reason_r);
else {
mail_transaction_log_file_munmap(file);
ret = mail_transaction_log_file_read(file, map_start_offset, FALSE, reason_r);
}
i_assert(file->buffer == NULL || file->mmap_base != NULL ||
file->sync_offset >= file->buffer_offset + file->buffer->used);
if (ret <= 0)
return ret;
i_assert(file->buffer != NULL);
return log_file_map_check_offsets(file, start_offset, end_offset,
reason_r) ? 1 : 0;
}
void mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file
*file)
{
const char *error;
buffer_t *buf;
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
return;
if (file->mmap_base != NULL) {
/* just copy to memory */
i_assert(file->buffer_offset == 0);
buf = buffer_create_dynamic(default_pool, file->mmap_size);
buffer_append(buf, file->mmap_base, file->mmap_size);
buffer_free(&file->buffer);
file->buffer = buf;
/* and lose the mmap */
if (munmap(file->mmap_base, file->mmap_size) < 0)
log_file_set_syscall_error(file, "munmap()");
file->mmap_base = NULL;
} else if (file->buffer_offset != 0) {
/* we don't have the full log in the memory. read it. */
(void)mail_transaction_log_file_read(file, 0, FALSE, &error);
}
file->last_size = 0;
if (close(file->fd) < 0)
log_file_set_syscall_error(file, "close()");
file->fd = -1;
i_free(file->filepath);
file->filepath = i_strdup(file->log->filepath);
}