mbox-sync.c revision d0e5f8252516c4d4df2fbcdae4d37f9e5f931199
183bea41fa640dc8117f3eb45ff935cd81377a84Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen/*
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen required disk I/O. We may need to:
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen - Write missing X-UID and X-IMAPbase headers
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen - Write missing or broken Content-Length header if there's space
0536ccb51d41e3078c3a9fa33e509fb4b2420f95Timo Sirainen - Expunge specified messages
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen Here's how we do it:
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
70afae43cc78ea6ecca83f6c587072c442a15ec1Timo Sirainen - Start reading the mails from the beginning
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen of them, remember how much each message has and offset to beginning of the
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen padding
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen - If header needs to be rewritten and there's enough space, do it
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen - If we didn't have enough space, remember how much was missing
660b4d36110c44b1e4b4b45a78c22d1569ccdb54Timo Sirainen - Continue reading and counting the padding in each message. If available
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen padding is enough to rewrite all the previous messages needing it, do it
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen - When we encounter expunged message, treat all of it as padding and
283ccfe110ed62e48f36e0d84e47da8cae5106beTimo Sirainen rewrite previous messages if needed (and there's enough space).
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen Afterwards keep moving messages backwards to fill the expunged space.
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen Moving is done by rewriting each message's headers, with possibly adding
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen missing Content-Length header and padding. Message bodies are moved
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen without modifications.
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen - If we encounter end of file, grow the file and rewrite needed messages
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen - Rewriting is done by moving message body forward, rewriting message's
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen header and doing the same for previous message, until all of them are
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen rewritten.
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen*/
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "lib.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "ioloop.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "buffer.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "istream.h"
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen#include "file-set-size.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "str.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "write-full.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "istream-raw-mbox.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "mbox-storage.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "mbox-file.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "mbox-lock.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include "mbox-sync-private.h"
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#include <stddef.h>
8d6a6eccd3f2e34df967b90bb45e20755241bdbbTimo Sirainen#include <sys/stat.h>
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen#define MBOX_SYNC_SECS 1
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen{
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
306f6bb8791755257c0db900f17659402c660057Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen " from mbox file %s", from_offset,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen sync_ctx->ibox->path);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen return -1;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen }
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen return 0;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen}
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainenstatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen struct mbox_sync_mail_context *mail_ctx,
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen uoff_t grow_size)
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen{
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen uoff_t src_offset, file_size;
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen i_assert(grow_size > 0);
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen /* put the padding between last message's header and body */
1388b590dbd85245b591346f860bc1319953318aTimo Sirainen file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen if (file_set_size(sync_ctx->fd, file_size) < 0) {
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen return -1;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen }
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen src_offset = mail_ctx->body_offset;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen mail_ctx->body_offset += grow_size;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen file_size - mail_ctx->body_offset) < 0)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen return -1;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen istream_raw_mbox_flush(sync_ctx->input);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen return 0;
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen}
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainenstatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen{
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen struct mail_index_sync_rec *sync;
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen size_t size, src, dest;
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen sync = buffer_get_modifyable_data(syncs_buf, &size);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen size /= sizeof(*sync);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen for (src = dest = 0; src < size; src++) {
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen if (sync[src].uid2 >= uid) {
0a3d3ca7db7cbdc947ccabe740c40561b3f5c066Timo Sirainen if (src != dest)
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen sync[dest] = sync[src];
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen dest++;
d514e6e6ea2320c18c58e0ade9184f5aa67d491bTimo Sirainen }
d514e6e6ea2320c18c58e0ade9184f5aa67d491bTimo Sirainen }
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen
36d2b3dc8766ef336a289a51075ca2f3236ef1efTimo Sirainen buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
36d2b3dc8766ef336a289a51075ca2f3236ef1efTimo Sirainen}
5561fe7754843250d0f2701332f63467f77f71dbTimo Sirainen
5561fe7754843250d0f2701332f63467f77f71dbTimo Sirainenstatic int
5561fe7754843250d0f2701332f63467f77f71dbTimo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
c7398cd92a003144358c121bdea63fba35b884aeTimo Sirainen struct mbox_sync_mail_context *mail_ctx)
c7398cd92a003144358c121bdea63fba35b884aeTimo Sirainen{
c7398cd92a003144358c121bdea63fba35b884aeTimo Sirainen /* get EOF */
c7398cd92a003144358c121bdea63fba35b884aeTimo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
c7398cd92a003144358c121bdea63fba35b884aeTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input))
c7398cd92a003144358c121bdea63fba35b884aeTimo Sirainen return 0;
c7398cd92a003144358c121bdea63fba35b884aeTimo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
0a3d3ca7db7cbdc947ccabe740c40561b3f5c066Timo Sirainen mail_ctx->sync_ctx = sync_ctx;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen mail_ctx->seq = ++sync_ctx->seq;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen mail_ctx->header = sync_ctx->header;
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen mail_ctx->uidl = sync_ctx->uidl;
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen str_truncate(mail_ctx->uidl, 0);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen mail_ctx->mail.from_offset =
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen mail_ctx->mail.offset =
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen if (mail_ctx->seq == 1)
193f5296d2a6b847970c222d8a261b89aae46331Timo Sirainen sync_ctx->seen_first_mail = TRUE;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen if (mail_ctx->seq > 1 && sync_ctx->dest_first_mail) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen /* First message was expunged and this is the next one.
193f5296d2a6b847970c222d8a261b89aae46331Timo Sirainen Skip \n header */
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen mail_ctx->mail.from_offset++;
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen }
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen
d514e6e6ea2320c18c58e0ade9184f5aa67d491bTimo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
d514e6e6ea2320c18c58e0ade9184f5aa67d491bTimo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
d514e6e6ea2320c18c58e0ade9184f5aa67d491bTimo Sirainen sync_ctx->input->eof);
d514e6e6ea2320c18c58e0ade9184f5aa67d491bTimo Sirainen
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen mail_ctx->mail.body_size =
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen mail_ctx->content_length);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0 && !mail_ctx->pseudo) {
2a4e8f370c566ffd360922227fc73d0ee36abee7Timo Sirainen if (!sync_ctx->ibox->keep_recent) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen /* need to add 'O' flag to Status-header */
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen mail_ctx->need_rewrite = TRUE;
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen }
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen mail_ctx->recent = TRUE;
70afae43cc78ea6ecca83f6c587072c442a15ec1Timo Sirainen }
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen return 1;
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen}
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen
70afae43cc78ea6ecca83f6c587072c442a15ec1Timo Sirainenstatic int mbox_sync_buf_have_expunges(buffer_t *syncs_buf)
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen{
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen const struct mail_index_sync_rec *sync;
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen size_t size, i;
70afae43cc78ea6ecca83f6c587072c442a15ec1Timo Sirainen
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen sync = buffer_get_data(syncs_buf, &size);
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen size /= sizeof(*sync);
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen for (i = 0; i < size; i++) {
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen if (sync[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen return TRUE;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen }
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen return FALSE;
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen}
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen uint32_t uid, int *sync_expunge_r)
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen{
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen int ret;
c8b29548f94160e0fe2f02e66d899d9e6cb495b3Timo Sirainen
c8b29548f94160e0fe2f02e66d899d9e6cb495b3Timo Sirainen *sync_expunge_r = FALSE;
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen
fdb97244fa45c32c3593726c15aa69ce29bc7121Timo Sirainen if (sync_ctx->ibox->mbox_readonly || sync_ctx->index_sync_ctx == NULL)
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen return 0;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen if (uid == 0) {
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen /* nothing for this or the future ones */
d6500661eb699ff335ac570c8646b6e067e1aac6Timo Sirainen uid = (uint32_t)-1;
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen }
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen while (uid >= sync_rec->uid1) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen if (uid <= sync_rec->uid2 &&
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen buffer_append(sync_ctx->syncs, sync_rec,
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen sizeof(*sync_rec));
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
6246b93fb37890dcb2f4df9896438f3f376ab284Timo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen *sync_expunge_r = TRUE;
1964dbea138cb4a213b1bce1eeee68992b18829aTimo Sirainen }
dcc76bb1e1bb287e3e71e6a39a7ca207fab0eaa8Timo Sirainen
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen if (ret < 0) {
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
9d7451b57769988f7e3e41cd8790e65429ffc5c7Timo Sirainen return -1;
}
if (ret == 0) {
memset(sync_rec, 0, sizeof(*sync_rec));
break;
}
if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) {
if (sync_rec->uid2 >= sync_ctx->next_uid) {
sync_ctx->next_uid = sync_rec->uid2 + 1;
sync_ctx->update_base_uid_last = sync_rec->uid2;
}
memset(sync_rec, 0, sizeof(*sync_rec));
}
}
if (!*sync_expunge_r)
*sync_expunge_r = mbox_sync_buf_have_expunges(sync_ctx->syncs);
return 0;
}
static void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
keywords_mask_t keywords)
{
const struct mail_index_sync_rec *sync;
size_t size, i;
sync = buffer_get_data(syncs_buf, &size);
size /= sizeof(*sync);
for (i = 0; i < size; i++) {
if (sync[i].type != MAIL_INDEX_SYNC_TYPE_FLAGS)
continue;
mail_index_sync_flags_apply(&sync[i], flags, keywords);
}
}
static int
mbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
uint32_t uid, const struct mail_index_record **rec_r)
{
const struct mail_index_record *rec = NULL;
uint32_t messages_count;
int ret = 0;
messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
while (sync_ctx->idx_seq <= messages_count) {
ret = mail_index_lookup(sync_ctx->sync_view,
sync_ctx->idx_seq, &rec);
if (ret < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
if (uid <= rec->uid)
break;
/* externally expunged message, remove from index */
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
sync_ctx->idx_seq++;
rec = NULL;
}
if (ret == 0 && uid < sync_ctx->hdr->next_uid) {
/* this UID was already in index and it was expunged */
mail_storage_set_critical(sync_ctx->ibox->box.storage,
"mbox sync: Expunged message reappeared in mailbox %s "
"(UID %u < %u)", sync_ctx->ibox->path, uid,
sync_ctx->hdr->next_uid);
ret = 0;
} else if (rec != NULL && rec->uid != uid) {
/* new UID in the middle of the mailbox - shouldn't happen */
mail_storage_set_critical(sync_ctx->ibox->box.storage,
"mbox sync: UID inserted in the middle of mailbox %s "
"(%u > %u)", sync_ctx->ibox->path, rec->uid, uid);
ret = 0; rec = NULL;
} else {
ret = 1;
}
*rec_r = rec;
return ret;
}
static int mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
unsigned char hdr_md5_sum[],
const struct mail_index_record **rec_r)
{
const struct mail_index_record *rec = NULL;
uint32_t messages_count;
const void *data;
int ret;
messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
while (sync_ctx->idx_seq <= messages_count) {
ret = mail_index_lookup(sync_ctx->sync_view,
sync_ctx->idx_seq, &rec);
if (ret < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
if (mail_index_lookup_extra(sync_ctx->sync_view,
sync_ctx->idx_seq,
sync_ctx->ibox->md5hdr_extra_idx,
&data) < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
if (memcmp(data, hdr_md5_sum, 16) == 0)
break;
/* externally expunged message, remove from index */
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
sync_ctx->idx_seq++;
rec = NULL;
}
*rec_r = rec;
return 0;
}
static int
mbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mail,
int nocheck)
{
const void *data;
uint64_t offset;
if (!nocheck) {
/* see if from_offset needs updating */
if (mail_index_lookup_extra(sync_ctx->sync_view,
sync_ctx->idx_seq,
sync_ctx->ibox->mbox_extra_idx,
&data) < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
if (data != NULL &&
*((const uint64_t *)data) == mail->from_offset)
return 0;
}
offset = mail->from_offset;
mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->ibox->mbox_extra_idx, &offset);
return 0;
}
static int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
const struct mail_index_record *rec)
{
struct mbox_sync_mail *mail = &mail_ctx->mail;
keywords_mask_t idx_keywords;
uint8_t idx_flags, mbox_flags;
if (rec == NULL) {
/* new message */
mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
if (sync_ctx->ibox->keep_recent &&
(mail->flags & MBOX_NONRECENT) == 0)
mbox_flags |= MAIL_RECENT;
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags,
mail->keywords);
if (sync_ctx->ibox->md5hdr_extra_idx != 0) {
mail_index_update_extra_rec(sync_ctx->t,
sync_ctx->idx_seq,
sync_ctx->ibox->md5hdr_extra_idx,
mail_ctx->hdr_md5_sum);
}
if (str_len(mail_ctx->uidl) > 0) {
/*FIXME:mail_cache_add(sync_ctx->cache_trans,
MAIL_CACHE_UID_STRING,
str_data(mail_ctx->uidl),
str_len(mail_ctx->uidl));*/
}
} else {
/* see if flags changed */
idx_flags = rec->flags;
memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
mbox_sync_apply_index_syncs(sync_ctx->syncs,
&idx_flags, idx_keywords);
mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
(mail->flags & MAIL_FLAGS_MASK);
mbox_flags ^= MAIL_RECENT;
if ((idx_flags & ~MAIL_RECENT) != (mbox_flags & ~MAIL_RECENT) ||
memcmp(idx_keywords, mail->keywords,
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags,
mail->keywords);
} else if (((idx_flags ^ mbox_flags) & MAIL_RECENT) != 0) {
/* drop recent flag */
memset(idx_keywords, 0, INDEX_KEYWORDS_BYTE_COUNT);
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REMOVE, MAIL_RECENT,
idx_keywords);
}
}
if (mail_ctx->recent)
index_mailbox_set_recent(sync_ctx->ibox, sync_ctx->idx_seq);
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
return -1;
}
return 0;
}
static int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
{
struct istream *input = ctx->sync_ctx->file_input;
const unsigned char *data;
size_t size, from_line_size;
buffer_set_used_size(ctx->sync_ctx->from_line, 0);
from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
i_stream_seek(input, ctx->mail.from_offset);
for (;;) {
data = i_stream_get_data(input, &size);
if (size >= from_line_size)
size = from_line_size;
buffer_append(ctx->sync_ctx->from_line, data, size);
i_stream_skip(input, size);
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int
mbox_write_from_line(struct mbox_sync_mail_context *ctx)
{
string_t *str = ctx->sync_ctx->from_line;
if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
ctx->mail.from_offset) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
return -1;
}
istream_raw_mbox_flush(ctx->sync_ctx->input);
return 0;
}
static int update_from_offsets(struct mbox_sync_context *sync_ctx)
{
const struct mbox_sync_mail *mails;
uint32_t idx, extra_idx;
uint64_t offset;
size_t size;
extra_idx = sync_ctx->ibox->mbox_extra_idx;
mails = buffer_get_modifyable_data(sync_ctx->mails, &size);
size /= sizeof(*mails);
i_assert(sync_ctx->seq - sync_ctx->need_space_seq + 1 == size);
for (idx = 0; idx < size; idx++) {
if (mails[idx].idx_seq == 0)
continue;
offset = mails[idx].from_offset;
mail_index_update_extra_rec(sync_ctx->t, mails[idx].idx_seq,
extra_idx, &offset);
}
return 0;
}
static int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
{
mail_ctx->mail.offset = mail_ctx->mail.from_offset;
mail_ctx->mail.space =
mail_ctx->body_offset - mail_ctx->mail.from_offset +
mail_ctx->mail.body_size;
mail_ctx->mail.body_size = 0;
if (mail_ctx->sync_ctx->dest_first_mail) {
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
mail_ctx->mail.space++;
}
mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
return 0;
}
static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
off_t move_diff;
int ret;
if (sync_ctx->ibox->mbox_readonly)
return 0;
if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
/* move the header backwards to fill expunged space */
move_diff = -sync_ctx->expunged_space;
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
ret = mbox_sync_try_rewrite(mail_ctx, move_diff, FALSE);
if (ret < 0)
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
mail_ctx->mail.from_offset += move_diff;
mail_ctx->mail.offset += move_diff;
if (mbox_write_from_line(mail_ctx) < 0)
return -1;
}
} else if (mail_ctx->need_rewrite ||
buffer_get_used_size(sync_ctx->syncs) != 0 ||
(mail_ctx->seq == 1 &&
sync_ctx->update_base_uid_last != 0)) {
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
if ((ret = mbox_sync_try_rewrite(mail_ctx, 0, FALSE)) < 0)
return -1;
} else {
/* nothing to do */
return 0;
}
if (ret == 0 && sync_ctx->need_space_seq == 0) {
/* first mail with no space to write it */
sync_ctx->need_space_seq = sync_ctx->seq;
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
memset(&mail, 0, sizeof(mail));
mail.flags = MBOX_EXPUNGED;
mail.offset = mail_ctx->mail.from_offset -
sync_ctx->expunged_space;
mail.space = sync_ctx->expunged_space;
sync_ctx->need_space_seq--;
buffer_append(sync_ctx->mails, &mail, sizeof(mail));
}
}
return 0;
}
static int
mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t padding;
buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
sync_ctx->space_diff += mail_ctx->mail.space;
if (sync_ctx->space_diff < 0)
return 0;
/* we have enough space now */
padding = MBOX_HEADER_PADDING *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
if (mail_ctx->mail.uid == 0 &&
(uoff_t)sync_ctx->space_diff > padding) {
/* don't waste too much on padding */
sync_ctx->expunged_space = sync_ctx->space_diff - padding;
sync_ctx->space_diff = padding;
} else {
sync_ctx->expunged_space = 0;
}
if (mbox_sync_rewrite(sync_ctx, sync_ctx->space_diff,
sync_ctx->need_space_seq, sync_ctx->seq) < 0)
return -1;
if (update_from_offsets(sync_ctx) < 0)
return -1;
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
memset(mail_ctx, 0, sizeof(*mail_ctx));
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
return 0;
}
static int
mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
{
struct index_mailbox *ibox = sync_ctx->ibox;
uoff_t old_offset;
int ret, deleted;
if (seq == 0) {
if (istream_raw_mbox_seek(ibox->mbox_stream, 0) < 0) {
mail_storage_set_error(ibox->box.storage,
"Mailbox isn't a valid mbox file");
return -1;
}
seq++;
} else {
old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
ret = mbox_file_seek(ibox, sync_ctx->sync_view, seq, &deleted);
if (ret < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
if (ret == 0) {
if (istream_raw_mbox_seek(ibox->mbox_stream,
old_offset) < 0) {
mail_storage_set_critical(ibox->box.storage,
"Error seeking back to original "
"offset %s in mbox file %s",
dec2str(old_offset), ibox->path);
return -1;
}
return 0;
}
}
/* set to -1, since it's always increased later */
sync_ctx->seq = seq-1;
if (sync_ctx->seq == 0 &&
istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
/* this mbox has pseudo mail which contains the X-IMAP header */
sync_ctx->seq++;
}
sync_ctx->idx_seq = seq;
sync_ctx->dest_first_mail = sync_ctx->seq == 0;
(void)istream_raw_mbox_get_body_offset(sync_ctx->input);
return 1;
}
static int
mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
{
uint32_t seq1, seq2;
if (mail_index_lookup_uid_range(sync_ctx->sync_view, uid, (uint32_t)-1,
&seq1, &seq2) < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
return mbox_sync_seek_to_seq(sync_ctx, seq1);
}
static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
uint32_t min_message_count, int partial)
{
const struct mail_index_record *rec;
uint32_t uid, messages_count;
uoff_t offset;
int ret, expunged;
messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
if (!mail_index_sync_have_more(sync_ctx->index_sync_ctx) ||
(!partial && min_message_count != 0)) {
ret = mbox_sync_seek_to_seq(sync_ctx, partial ?
messages_count : 0);
} else {
/* we sync only what we need to. jump to first record that
needs updating */
const struct mail_index_sync_rec *sync_rec;
size_t size;
if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
sync_ctx->sync_rec.uid1 == 0) {
if (mbox_sync_read_index_syncs(sync_ctx, 1,
&expunged) < 0)
return -1;
if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
sync_ctx->sync_rec.uid1 == 0) {
/* nothing to do */
return 1;
}
}
sync_rec = buffer_get_data(sync_ctx->syncs, &size);
if (size == 0)
sync_rec = &sync_ctx->sync_rec;
ret = mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1);
}
if (ret <= 0)
return ret;
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
uid = mail_ctx->mail.uid;
if (mail_ctx->pseudo)
uid = 0;
/* get all sync records related to this message */
if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
return -1;
rec = NULL;
if (uid != 0 && sync_ctx->ibox->md5hdr_extra_idx == 0) {
ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
if (ret < 0)
return -1;
if (ret == 0)
uid = 0;
}
if (sync_ctx->ibox->mbox_readonly && !mail_ctx->pseudo) {
/* Read-only mboxes use MD5 sums. */
if (sync_ctx->ibox->md5hdr_extra_idx == 0) {
sync_ctx->ibox->md5hdr_extra_idx =
mail_index_register_record_extra(
sync_ctx->ibox->index,
"header-md5", 0, 16);
}
if (mbox_sync_find_index_md5(sync_ctx,
mail_ctx->hdr_md5_sum,
&rec) < 0)
return -1;
if (rec != NULL)
uid = rec->uid;
}
if (uid == 0 && !mail_ctx->pseudo) {
/* missing/broken X-UID. all the rest of the mails
need new UIDs. */
while (sync_ctx->idx_seq <= messages_count) {
mail_index_expunge(sync_ctx->t,
sync_ctx->idx_seq++);
}
mail_ctx->need_rewrite = TRUE;
mail_ctx->mail.uid = sync_ctx->next_uid++;
sync_ctx->prev_msg_uid = mail_ctx->mail.uid;
}
mail_ctx->mail.idx_seq = sync_ctx->idx_seq;
if (!expunged) {
ret = mbox_sync_handle_header(mail_ctx);
sync_ctx->dest_first_mail = FALSE;
} else {
mail_ctx->mail.uid = 0;
ret = mbox_sync_handle_expunge(mail_ctx);
}
if (ret < 0)
return -1;
if (!mail_ctx->pseudo) {
if (!expunged) {
if (mbox_sync_update_index(sync_ctx, mail_ctx,
rec) < 0)
return -1;
}
sync_ctx->idx_seq++;
}
istream_raw_mbox_next(sync_ctx->input,
mail_ctx->mail.body_size);
offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
if (mbox_move(sync_ctx,
mail_ctx->body_offset -
sync_ctx->expunged_space,
mail_ctx->body_offset,
mail_ctx->mail.body_size) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
}
} else if (sync_ctx->seq >= min_message_count) {
mbox_sync_buffer_delete_old(sync_ctx->syncs, uid+1);
if (buffer_get_used_size(sync_ctx->syncs) == 0) {
/* if there's no sync records left,
we can stop */
if (sync_ctx->sync_rec.uid1 == 0)
break;
/* we can skip forward to next record which
needs updating. if it failes because the
offset is dirty, just ignore and continue
from where we are now. */
uid = sync_ctx->sync_rec.uid1;
if (mbox_sync_seek_to_uid(sync_ctx, uid) < 0)
return -1;
}
}
}
if (istream_raw_mbox_is_eof(sync_ctx->input)) {
/* rest of the messages in index don't exist -> expunge them */
while (sync_ctx->idx_seq <= messages_count)
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
}
if (!partial)
sync_ctx->ibox->mbox_sync_dirty = FALSE;
return 1;
}
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
uoff_t offset, padding, trailer_size;
int need_rewrite;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
i_assert(sync_ctx->expunged_space == 0);
return 0;
}
trailer_size = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->file_input->v_offset;
if (sync_ctx->need_space_seq != 0) {
i_assert(sync_ctx->space_diff < 0);
padding = MBOX_HEADER_PADDING *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
sync_ctx->space_diff -= padding;
sync_ctx->space_diff += sync_ctx->expunged_space;
if (sync_ctx->expunged_space <= -sync_ctx->space_diff)
sync_ctx->expunged_space = 0;
else
sync_ctx->expunged_space -= -sync_ctx->space_diff;
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
if (sync_ctx->space_diff < 0 &&
mbox_sync_grow_file(sync_ctx, mail_ctx,
-sync_ctx->space_diff) < 0)
return -1;
need_rewrite = sync_ctx->seq != sync_ctx->need_space_seq;
if (mbox_sync_try_rewrite(mail_ctx, 0, need_rewrite) < 0)
return -1;
if (need_rewrite) {
buffer_set_used_size(sync_ctx->mails,
(sync_ctx->seq -
sync_ctx->need_space_seq) *
sizeof(mail_ctx->mail));
buffer_append(sync_ctx->mails, &mail_ctx->mail,
sizeof(mail_ctx->mail));
if (mbox_sync_rewrite(sync_ctx, padding,
sync_ctx->need_space_seq,
sync_ctx->seq) < 0)
return -1;
}
if (update_from_offsets(sync_ctx) < 0)
return -1;
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
}
if (sync_ctx->expunged_space > 0) {
/* copy trailer, then truncate the file */
offset = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->expunged_space - trailer_size;
if (mbox_move(sync_ctx, offset,
offset + sync_ctx->expunged_space,
trailer_size) < 0)
return -1;
if (ftruncate(sync_ctx->fd, offset + trailer_size) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
return -1;
}
sync_ctx->expunged_space = 0;
istream_raw_mbox_flush(sync_ctx->input);
}
return 0;
}
static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
{
struct stat st;
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
return -1;
}
if ((sync_ctx->base_uid_validity != 0 &&
sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) ||
(sync_ctx->hdr->uid_validity == 0 && sync_ctx->seen_first_mail)) {
if (sync_ctx->base_uid_validity == 0) {
/* we couldn't rewrite X-IMAPbase because it's
a read-only mbox */
i_assert(sync_ctx->ibox->mbox_readonly);
sync_ctx->base_uid_validity = time(NULL);
}
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, uid_validity),
&sync_ctx->base_uid_validity,
sizeof(sync_ctx->base_uid_validity));
}
if (istream_raw_mbox_is_eof(sync_ctx->input) &&
sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
i_assert(sync_ctx->next_uid != 0);
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, next_uid),
&sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
}
if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp &&
!sync_ctx->ibox->mbox_sync_dirty) {
uint32_t sync_stamp = st.st_mtime;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_stamp),
&sync_stamp, sizeof(sync_stamp));
}
if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size &&
!sync_ctx->ibox->mbox_sync_dirty) {
uint64_t sync_size = st.st_size;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_size),
&sync_size, sizeof(sync_size));
}
sync_ctx->ibox->mbox_dirty_stamp = st.st_mtime;
sync_ctx->ibox->mbox_dirty_size = st.st_size;
return 0;
}
static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->next_uid = sync_ctx->hdr->next_uid;
sync_ctx->prev_msg_uid = 0;
sync_ctx->seq = 0;
sync_ctx->idx_seq = 1;
sync_ctx->dest_first_mail = TRUE;
sync_ctx->seen_first_mail = FALSE;
}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx,
enum mbox_sync_flags flags)
{
struct mbox_sync_mail_context mail_ctx;
struct stat st;
uint32_t min_msg_count;
int ret, partial;
partial = FALSE;
if ((flags & MBOX_SYNC_HEADER) != 0)
min_msg_count = 1;
else {
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "stat()");
return -1;
}
if ((uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
(uint64_t)st.st_size == sync_ctx->hdr->sync_size) {
/* file is fully synced */
sync_ctx->ibox->mbox_sync_dirty = FALSE;
min_msg_count = 0;
} else if ((flags & MBOX_SYNC_UNDIRTY) != 0) {
/* we want to do full syncing */
min_msg_count = (uint32_t)-1;
sync_ctx->ibox->mbox_sync_dirty = TRUE;
} else {
/* see if we can delay syncing the whole file.
normally we only notice expunges and appends
in partial syncing. */
partial = TRUE;
min_msg_count = (uint32_t)-1;
sync_ctx->ibox->mbox_sync_dirty = TRUE;
}
}
mbox_sync_restart(sync_ctx);
ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count, partial);
if (ret <= 0) {
if (ret < 0)
return -1;
/* partial syncing didn't work, do it again */
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx,
(uint32_t)-1, FALSE) < 0)
return -1;
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
buffer_set_used_size(sync_ctx->syncs, 0);
memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync_has_changed(struct index_mailbox *ibox, int leave_dirty)
{
const struct mail_index_header *hdr;
struct stat st;
if (mail_index_get_header(ibox->view, &hdr) < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
if (stat(ibox->path, &st) < 0) {
mbox_set_syscall_error(ibox, "stat()");
return -1;
}
if ((uint32_t)st.st_mtime == hdr->sync_stamp &&
(uint64_t)st.st_size == hdr->sync_size) {
/* fully synced */
ibox->mbox_sync_dirty = FALSE;
return 0;
}
if (!ibox->mbox_sync_dirty || !leave_dirty)
return 1;
return st.st_mtime != ibox->mbox_dirty_stamp ||
st.st_size != ibox->mbox_dirty_size;
}
static int mbox_sync_update_imap_base(struct mbox_sync_context *sync_ctx)
{
struct mbox_sync_mail_context mail_ctx;
if (mbox_sync_seek(sync_ctx, 0) < 0)
return -1;
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
sync_ctx->update_base_uid_last = sync_ctx->next_uid-1;
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, 1, 0) < 0)
return -1;
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync(struct index_mailbox *ibox, enum mbox_sync_flags flags)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
uint32_t seq;
uoff_t offset;
unsigned int lock_id = 0;
int ret, changed;
ibox->sync_last_check = ioloop_time;
if (!ibox->mbox_do_dirty_syncs)
flags |= MBOX_SYNC_UNDIRTY;
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
if (mbox_lock(ibox, F_RDLCK, &lock_id) <= 0)
return -1;
}
if ((flags & MBOX_SYNC_HEADER) != 0)
changed = 1;
else {
int leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
if ((changed = mbox_sync_has_changed(ibox, leave_dirty)) < 0) {
if ((flags & MBOX_SYNC_LOCK_READING) != 0)
(void)mbox_unlock(ibox, lock_id);
return -1;
}
}
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
/* we just want to lock it for reading. if mbox hasn't been
modified don't do any syncing. */
if (!changed)
return 0;
/* have to sync to make sure offsets have stayed the same */
(void)mbox_unlock(ibox, lock_id);
lock_id = 0;
}
__again:
if (changed) {
/* we're most likely modifying the mbox while syncing, just
lock it for writing immediately. the mbox must be locked
before index syncing is started to avoid deadlocks, so we
don't have much choice either (well, easy ones anyway). */
int lock_type = ibox->mbox_readonly ? F_RDLCK : F_WRLCK;
if (mbox_lock(ibox, lock_type, &lock_id) <= 0)
return -1;
}
if ((flags & MBOX_SYNC_LAST_COMMIT) != 0) {
seq = ibox->commit_log_file_seq;
offset = ibox->commit_log_file_offset;
} else {
seq = (uint32_t)-1;
offset = (uoff_t)-1;
}
ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
seq, offset, !ibox->keep_recent);
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(ibox);
if (lock_id != 0)
(void)mbox_unlock(ibox, lock_id);
return ret;
}
if (!changed && !mail_index_sync_have_more(index_sync_ctx)) {
/* nothing to do */
if (lock_id != 0)
(void)mbox_unlock(ibox, lock_id);
/* index may need to do internal syncing though, so commit
instead of rollbacking. */
if (mail_index_sync_commit(index_sync_ctx) < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
return 0;
}
if (lock_id == 0) {
/* ok, we have something to do but no locks. we'll have to
restart syncing to avoid deadlocking. */
mail_index_sync_rollback(index_sync_ctx);
changed = 1;
goto __again;
}
if (mbox_file_open_stream(ibox) < 0) {
mail_index_sync_rollback(index_sync_ctx);
(void)mbox_unlock(ibox, lock_id);
return -1;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.ibox = ibox;
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.uidl = str_new(default_pool, 128);
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
i_assert(ret == 0);
sync_ctx.file_input = sync_ctx.ibox->mbox_file_stream;
sync_ctx.input = sync_ctx.ibox->mbox_stream;
sync_ctx.fd = sync_ctx.ibox->mbox_fd;
if (mbox_sync_do(&sync_ctx, flags) < 0)
ret = -1;
if (ret < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
} else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
sync_ctx.t = NULL;
if (ret < 0)
mail_index_sync_rollback(index_sync_ctx);
else if (mail_index_sync_commit(index_sync_ctx) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
if (sync_ctx.seen_first_mail &&
sync_ctx.base_uid_last != sync_ctx.next_uid-1 &&
ret == 0 && !ibox->mbox_readonly) {
/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
so previous transactions have been committed. */
/* FIXME: ugly .. */
ret = mail_index_sync_begin(ibox->index,
&sync_ctx.index_sync_ctx,
&sync_ctx.sync_view,
(uint32_t)-1, (uoff_t)-1, FALSE);
if (ret < 0)
mail_storage_set_index_error(ibox);
else {
(void)mail_index_get_header(sync_ctx.sync_view,
&sync_ctx.hdr);
if ((ret = mbox_sync_update_imap_base(&sync_ctx)) < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t,
&seq,
&offset) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
if (mail_index_sync_commit(sync_ctx.
index_sync_ctx) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
}
}
if (ret == 0 && ibox->mbox_lock_type == F_WRLCK) {
if (fsync(ibox->mbox_fd) < 0) {
mbox_set_syscall_error(ibox, "fsync()");
ret = -1;
}
}
if (lock_id != 0 && ibox->mbox_lock_type != F_RDLCK) {
/* drop to read lock */
unsigned int read_lock_id = 0;
if (mbox_lock(ibox, F_RDLCK, &read_lock_id) <= 0)
ret = -1;
else {
if (mbox_unlock(ibox, lock_id) < 0)
ret = -1;
lock_id = read_lock_id;
}
}
if (lock_id != 0 && (flags & MBOX_SYNC_LOCK_READING) == 0) {
/* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
notice changes made by others .. and this has to be done
even if lock_reading is set.. except if
mbox_sync_dirty = TRUE */
if (mbox_unlock(ibox, lock_id) < 0)
ret = -1;
}
str_free(sync_ctx.uidl);
str_free(sync_ctx.header);
str_free(sync_ctx.from_line);
buffer_free(sync_ctx.mails);
buffer_free(sync_ctx.syncs);
return ret;
}
struct mailbox_sync_context *
mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
enum mbox_sync_flags mbox_sync_flags = 0;
int ret = 0;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time)
ret = mbox_sync(ibox, mbox_sync_flags);
return index_mailbox_sync_init(box, flags, ret < 0);
}