mbox-sync.c revision a97a3ce5d82b16e8979de2d6cafbf6b0a129fe4e
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (c) 2004-2013 Dovecot authors, see the included COPYING file */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen required disk I/O. We may need to:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing X-UID and X-IMAPbase headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing or broken Content-Length header if there's space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Expunge specified messages
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Here's how we do it:
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Start reading the mails from the beginning
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen of them, remember how much each message has and offset to beginning of the
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If header needs to be rewritten and there's enough space, do it
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If we didn't have enough space, remember how much was missing
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Continue reading and counting the padding in each message. If available
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen padding is enough to rewrite all the previous messages needing it, do it
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - When we encounter expunged message, treat all of it as padding and
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen rewrite previous messages if needed (and there's enough space).
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen Afterwards keep moving messages backwards to fill the expunged space.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen Moving is done by rewriting each message's headers, with possibly adding
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen missing Content-Length header and padding. Message bodies are moved
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen without modifications.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If we encounter end of file, grow the file and rewrite needed messages
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Rewriting is done by moving message body forward, rewriting message's
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen header and doing the same for previous message, until all of them are
f7d43647acc6dc80064c8c4cacf5bf86f754c530Timo Sirainen/* The text below was taken exactly as c-client wrote it to my mailbox,
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen so it's probably copyrighted by University of Washington. */
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"This text is part of the internal format of your mail folder, and is not\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"a real message. It is created automatically by the mail system software.\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"If deleted, important folder data will be lost, and it will be re-created\n" \
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen"with the data reset to initial values.\n"
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainenvoid mbox_sync_set_critical(struct mbox_sync_context *sync_ctx,
8153fdec343e40e2a78f5c12353e89b994b28f74Timo Sirainen const char *fmt, ...)
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen "mbox file %s was modified while we were syncing, "
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen "check your locking settings",
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen "Sync failed for mbox file %s: %s",
445f9e31c6c3aa6c0a72be8565da8f6e594d24fbTimo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenvoid mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* Do this even if ext_modified is already set. Expunging code relies
178511b57faa7c3f8203dd8b7e4059d00cbfc23aTimo Sirainen on last_stat being updated. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (st.st_size != sync_ctx->last_stat.st_size ||
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainenvoid mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen /* just mark the stat as dirty. */
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0)
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
e8a35266a5ceacdfafeeffd6bddae77931ff97ebTimo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* get EOF */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (istream_raw_mbox_is_corrupted(sync_ctx->input))
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 &&
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen /* need to add 'O' flag to Status-header */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic void mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
36e2fa21c22452470c1509cc63de20f7415c7b5eTimo Sirainen /* nothing for this or the future ones */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen index_sync_changes_read(sync_ctx->sync_changes, uid, sync_expunge_r,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* we can't expunge anything from read-only mboxes */
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* externally expunged message, remove from index */
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen if (rec == NULL && uid < sync_ctx->idx_next_uid) {
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* this UID was already in index and it was expunged */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen "Expunged message reappeared to mailbox "
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen "(UID %u < %u, seq=%u, idx_msgs=%u)",
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen "UID inserted in the middle of mailbox "
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen "(%u > %u, seq=%u, idx_msgs=%u)",
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen rec->uid, uid, sync_ctx->seq, messages_count);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenstatic void mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen unsigned char hdr_md5_sum[],
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* externally expunged message, remove from index */
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
aa38d1a0945f0bc13a225d043f53fad2eec666b1Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen /* see if from_offset needs updating */
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen *((const uint64_t *)data) == mail->from_offset)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainenmbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen struct mail_index *index = sync_ctx->mbox->box.index;
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen keywords = !array_is_created(&mail_ctx->mail.keywords) ?
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mail_index_keywords_create_from_indexes(index,
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainenmbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen sync_ctx->mbox->md5hdr_ext_idx, &ext_data, NULL);
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) {
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainenstatic void mbox_sync_get_dirty_flags(struct mbox_sync_mail_context *mail_ctx,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen /* default to undirtying the message. it gets added back if
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen flags/keywords don't match what is in the index. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->mail.flags &= ~MAIL_INDEX_MAIL_FLAG_DIRTY;
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen /* replace flags */
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen idx_flags = rec->flags & MAIL_FLAGS_NONRECENT;
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen mbox_flags = mail_ctx->mail.flags & MAIL_FLAGS_NONRECENT;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mail_ctx->mail.flags = (mail_ctx->mail.flags & MAIL_RECENT) |
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen /* replace keywords */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_lookup_keywords(sync_ctx->sync_view, sync_ctx->idx_seq,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (!index_keyword_array_cmp(&idx_keywords, &mail_ctx->mail.keywords)) {
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen mail_ctx->mail.flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (!array_is_created(&mail_ctx->mail.keywords)) {
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen array_append_array(&mail_ctx->mail.keywords, &idx_keywords);
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainenstatic void mbox_sync_update_flags(struct mbox_sync_mail_context *mail_ctx,
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen ARRAY_TYPE(keyword_indexes) orig_keywords = ARRAY_INIT;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen /* flags and keywords are dirty. replace the current
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen ones from the flags in index file. */
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen flags = orig_flags = mail->flags & MAIL_FLAGS_NONRECENT;
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen array_append_array(&orig_keywords, &mail->keywords);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* apply new changes */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen index_sync_changes_apply(sync_ctx->sync_changes,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen !index_keyword_array_cmp(&mail->keywords, &orig_keywords)) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mail->flags = flags | (mail->flags & MAIL_RECENT) |
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (sync_type != 0 && box->v.sync_notify != NULL) {
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainenstatic void mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* changes are written to the mbox file */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* make sure this message gets written later */
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen /* new message */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen /* flags other than recent/dirty have changed */
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen /* only dirty flag state changed */
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen /* see if keywords changed */
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen mail_index_lookup_keywords(sync_ctx->sync_view,
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen if (!index_keyword_array_cmp(&idx_keywords, &mail->keywords))
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen /* see if we need to update md5 sum. */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* Mail has "Status: O" header. No messages before this
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen can be recent. */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen rewriting would just move it anyway. */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen bool nocheck = rec == NULL || sync_ctx->expunged_space > 0;
e8a35266a5ceacdfafeeffd6bddae77931ff97ebTimo Sirainen mbox_sync_update_from_offset(sync_ctx, mail, nocheck);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct istream *input = ctx->sync_ctx->file_input;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen const unsigned char *data;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen const char *str;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen unsigned int i;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen i_assert(sync_ctx->base_uid_last_offset != 0);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* first check that the 10 bytes are there and they're exactly as
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen expected. just an extra safety check to make sure we never write
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen to wrong location in the mbox file. */
16aba431c576c1dbd99cbaae4f9d65eea9ad73c2Timo Sirainen ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf),
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pread_full()");
16aba431c576c1dbd99cbaae4f9d65eea9ad73c2Timo Sirainen "X-IMAPbase uid-last offset unexpectedly outside mbox");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen for (i = 0, uid_last = 0; i < sizeof(buf); i++) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "X-IMAPbase uid-last unexpectedly lost");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* and write it */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen str = t_strdup_printf("%010u", sync_ctx->next_uid - 1);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen sync_ctx->base_uid_last = sync_ctx->next_uid - 1;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str),
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()");
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainenstatic void update_from_offsets(struct mbox_sync_context *sync_ctx)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen unsigned int i, count;
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen for (i = 0; i < count; i++) {
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen if (mails[i].idx_seq == 0 || mails[i].expunged)
3021a062b16ff0138408be6107d6bcd0ced280b9Timo Sirainen mail_index_update_ext(sync_ctx->t, mails[i].idx_seq,
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainenstatic void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->mail.offset = mail_ctx->mail.from_offset;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->body_offset - mail_ctx->mail.from_offset +
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* expunging first message, fix space to contain next
2be66b9eddad3841a1195fe9aeb1eaf0f28f1116Timo Sirainen message's \n header too since it will be removed. */
2be66b9eddad3841a1195fe9aeb1eaf0f28f1116Timo Sirainen if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) {
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen /* uid-last offset is invalid now */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->expunged_space += mail_ctx->mail.space;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen /* move the header backwards to fill expunged space */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen orig_from_offset = mail_ctx->mail.from_offset;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* we're moving this mail to beginning of file.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen skip the initial \n (it's already counted in
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen expunged_space) */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* read the From-line before rewriting overwrites it */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen /* rewrite successful, write From-line to
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen new location */
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen /* didn't have enough space, move the offset
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen back so seeking into it doesn't fail */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->mail.from_offset = orig_from_offset;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* mark it dirty and do it later */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* nothing to do */
829c036d4ddfbd9ea49bd8a7c54e3057177d346eTimo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
36816b5af1472ae76a1909ae3cf29fd614b2ebfcTimo Sirainen /* first mail with no space to write it */
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen /* create dummy message to describe the expunged data */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->space_diff = sync_ctx->expunged_space;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen i_assert(sync_ctx->space_diff < -mail_ctx->mail.space);
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainenmbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen uoff_t end_offset, move_diff, extra_space, needed_space;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 ||
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->mail.offset == mail_ctx->hdr_offset);
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen if (array_is_created(&mail_ctx->mail.keywords)) {
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen /* mail's keywords are allocated from a pool that's cleared
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen for each mail. we'll need to copy it to something more
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen permanent. */
8d5991f5c4a8840bf1ea754093dbec505564ab78Timo Sirainen p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool,
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen array_append_array(&keywords_copy, &mail_ctx->mail.keywords);
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen array_append(&sync_ctx->mails, &mail_ctx->mail, 1);
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen /* we have enough space now */
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen /* this message was expunged. fill more or less of the space.
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen space_diff now consists of a negative "bytes needed" sum,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen plus the expunged space of this message. so it contains how
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen many bytes of _extra_ space we have. */
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen i_assert(mail_ctx->mail.space >= sync_ctx->space_diff);
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen /* don't waste too much on padding */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* this message gave enough space from headers. rewriting stops
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen at the end of this message's headers. */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* mail_ctx may contain wrong data after rewrite, so make sure we
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen don't try to access it */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainenmbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mail_storage_set_error(&mbox->storage->storage,
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen "Mailbox isn't a valid mbox file");
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted);
cece2b9cd692c06025cc0a7a0ff54d996a8c90efTimo Sirainen "Message was expunged unexpectedly");
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen "Error seeking back to original "
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid);
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* set to -1, since it's always increased later */
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* this mbox has pseudo mail which contains the X-IMAP header */
8153fdec343e40e2a78f5c12353e89b994b28f74Timo Sirainen sync_ctx->dest_first_mail = sync_ctx->seq == 0;
0a6f8311541ae59381171620b77f82be58be562eTimo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainenmbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
0a6f8311541ae59381171620b77f82be58be562eTimo Sirainen struct mail_index_view *sync_view = sync_ctx->sync_view;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (!mail_index_lookup_seq_range(sync_view, uid, (uint32_t)-1,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* doesn't exist anymore, seek to end of file */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen ret = i_stream_get_size(sync_ctx->file_input, TRUE, &size);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "i_stream_get_size()");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "Error seeking to end of mbox");
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen mail_index_view_get_messages_count(sync_view) + 1;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenstatic int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* delete sync records up to next message. so if there's still
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen something left in array, it means the next message needs modifying */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen index_sync_changes_delete_to(sync_ctx->sync_changes, next_uid);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (index_sync_changes_have(sync_ctx->sync_changes))
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen if (sync_ctx->hdr->first_recent_uid <= next_uid &&
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* we'll need to rewrite Status: O headers */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen uid = index_sync_changes_get_next_uid(sync_ctx->sync_changes);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (sync_ctx->hdr->first_recent_uid < sync_ctx->hdr->next_uid &&
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen (uid > sync_ctx->hdr->first_recent_uid || uid == 0) &&
cdaf255d6a3daeef0ac85edaa60bfa6d1f945bffTimo Sirainen /* we'll need to rewrite Status: O headers */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* we can skip forward to next record which needs updating. */
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen ret = mbox_sync_seek_to_uid(sync_ctx, next_uid);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* if there's no sync records left, we can stop. except if
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen this is a dirty sync, check if there are new messages. */
cdaf255d6a3daeef0ac85edaa60bfa6d1f945bffTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
8153fdec343e40e2a78f5c12353e89b994b28f74Timo Sirainen ret = mbox_sync_seek_to_seq(sync_ctx, messages_count);
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen /* seek failed because the offset is dirty. just ignore and
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen continue from where we are now. */
b8765f6093ab35fc2345293d78132d35794cbff5Timo Sirainenstatic void mbox_sync_hdr_update(struct mbox_sync_context *sync_ctx,
b8765f6093ab35fc2345293d78132d35794cbff5Timo Sirainen const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update;
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen sync_ctx->base_uid_validity = update->uid_validity;
907723f35f4d3dfc774ca42d00a8a7b8ef90dd5dTimo Sirainen sync_ctx->base_uid_last+1 < update->min_next_uid) {
2ae575a66f2a302f047f6de062a70b75f8bebc7bTimo Sirainen i_assert(sync_ctx->next_uid <= update->min_next_uid);
cdaf255d6a3daeef0ac85edaa60bfa6d1f945bffTimo Sirainen sync_ctx->base_uid_last = update->min_next_uid-1;
2ae575a66f2a302f047f6de062a70b75f8bebc7bTimo Sirainenstatic bool mbox_sync_imapbase(struct mbox_sync_context *sync_ctx,
2ae575a66f2a302f047f6de062a70b75f8bebc7bTimo Sirainen sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s",
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainenstatic int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen /* always start from first message so we can read X-IMAP or
e9d68b41c007f0e545de361f8012f6f231bfec8bTimo Sirainen X-IMAPbase header */
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen /* expunge everything */
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen /* UID ordering problems, resync everything to make
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sure we get everything right */
907723f35f4d3dfc774ca42d00a8a7b8ef90dd5dTimo Sirainen "UIDs broken with partial sync");
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (!mbox_sync_read_index_rec(sync_ctx, uid, &rec))
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* UID found but it's broken */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen } else if (uid == 0 &&
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* If we can't use/store X-UID header, use MD5 sum.
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen Also check for existing MD5 sums when we're actually
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen able to write X-UIDs. */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* get all sync records related to this message. with pseudo
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen message just get the first sync record so we can jump to
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it with partial seeking. */
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen /* if it was set, it was for the next message */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* message wasn't found from index. we have to
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen read everything from now on, no skipping */
9d2040fbb941f411d57fd850b4cdc3b1cccc1168Timo Sirainen /* missing/broken X-UID. all the rest of the mails
9d2040fbb941f411d57fd850b4cdc3b1cccc1168Timo Sirainen need new UIDs. */
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen /* oh no, we're out of UIDs. this shouldn't
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen happen normally, so just try to get it fixed
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen without crashing. */
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen "Out of UIDs, renumbering them in mbox "
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (mbox_sync_handle_missing_space(mail_ctx) < 0)
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* move the body */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen } else if (partial) {
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1,
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input)) {
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* rest of the messages in index don't exist -> expunge them */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen /* once we get around to writing the changes, we'll need to do
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen a full sync to avoid the "UIDs broken in partial sync"
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainenstatic int mbox_write_pseudo(struct mbox_sync_context *sync_ctx)
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen if (sync_ctx->mbox->sync_hdr_update != NULL) {
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen sync_ctx->base_uid_validity = update->uid_validity;
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen sync_ctx->base_uid_last = update->min_next_uid-1;
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen uid_validity = sync_ctx->base_uid_validity != 0 ?
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "From: Mail System Internal Data <MAILER-DAEMON@%s>\n"
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA"
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "\nMessage-ID: <%s@%s>\n"
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "X-IMAP: %u %010u\n"
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "Status: RO\n"
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_from_create("MAILER_DAEMON", ioloop_time),
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen my_hostname, dec2str(ioloop_time), my_hostname,
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen "pwrite_full()");
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* out of disk space, truncate to empty */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx->base_uid_last = sync_ctx->next_uid-1;
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainenstatic int mbox_append_zero(struct mbox_sync_context *sync_ctx,
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen memset(block, 0, I_MIN(sizeof(block), count));
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen while (count > 0) {
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pwrite()");
178511b57faa7c3f8203dd8b7e4059d00cbfc23aTimo Sirainen if (ftruncate(sync_ctx->write_fd, orig_file_size) < 0)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainenstatic int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen uoff_t file_size, offset, padding, trailer_size;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen ret = i_stream_get_size(sync_ctx->file_input, TRUE, &file_size);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_get_size()");
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen /* Not a file - allow anyway */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (file_size < sync_ctx->file_input->v_offset) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen "file size unexpectedly shrank "
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "(%"PRIuUOFF_T" vs %"PRIuUOFF_T")", file_size,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen trailer_size = file_size - sync_ctx->file_input->v_offset;
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff);
c263e92d73889da530b308c9ab28b4b74031550eTimo Sirainen sync_ctx->space_diff += sync_ctx->expunged_space;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* copy trailer, then truncate the file */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (file_size == (uoff_t)sync_ctx->expunged_space) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* everything deleted, the trailer_size still contains
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen the \n trailer though */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen } else if (sync_ctx->expunged_space == (off_t)file_size + 1 ||
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx->expunged_space == (off_t)file_size + 2) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* everything deleted and we didn't have a proper
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen i_assert(file_size >= sync_ctx->expunged_space + trailer_size);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen offset = file_size - sync_ctx->expunged_space - trailer_size;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (file_size == 0 && sync_ctx->mbox->sync_hdr_update != NULL) {
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainenmbox_sync_index_update_ext_header(struct mbox_mailbox *mbox,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen const struct mailbox_update *update = mbox->sync_hdr_update;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) {
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen memcpy(mbox->mbox_hdr.mailbox_guid, update->mailbox_guid,
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen } else if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen guid_128_generate(mbox->mbox_hdr.mailbox_guid);
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen memcmp(data, &mbox->mbox_hdr, data_size) != 0) {
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen /* upgrading from v1.x */
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen mail_index_ext_resize(trans, mbox->mbox_ext_idx,
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen mail_index_update_header_ext(trans, mbox->mbox_ext_idx,
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainenstatic uint32_t mbox_get_uidvalidity_next(struct mailbox_list *list)
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL);
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen path = t_strconcat(path, "/"MBOX_UIDVALIDITY_FNAME, NULL);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainenstatic int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) {
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen ((uint64_t)st->st_size == sync_ctx->mbox->mbox_hdr.sync_size ||
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen (uint64_t)st->st_size == sync_ctx->orig_size)) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* We moved messages inside the mbox file without changing
6597506f9317f3be644e708c9153dacaeeece84cTimo Sirainen the file's size. If mtime doesn't change, another process
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen not using the same index file as us can't know that the file
d252f81a2ff1bdd5439f9d2b3df715b70a4bcd3dTimo Sirainen was changed. So make sure the mtime changes. This should
d252f81a2ff1bdd5439f9d2b3df715b70a4bcd3dTimo Sirainen happen rarely enough that the sleeping doesn't become a
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen performance problem.
63a61b7a739ae0f3f520215137d9c50f94d0f34fTimo Sirainen Note that to do this perfectly safe we should do this wait
63a61b7a739ae0f3f520215137d9c50f94d0f34fTimo Sirainen whenever mails are moved or expunged, regardless of whether
63a61b7a739ae0f3f520215137d9c50f94d0f34fTimo Sirainen the file's size changed. That however could become a
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainen performance problem and the consequences of being wrong are
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainen quite minimal (an extra logged error message). */
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen while (sync_ctx->orig_mtime == st->st_mtime) {
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen if (utime(mailbox_get_path(&sync_ctx->mbox->box), NULL) < 0) {
63a61b7a739ae0f3f520215137d9c50f94d0f34fTimo Sirainen if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) {
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen "i_stream_stat()");
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen sync_ctx->mbox->mbox_hdr.sync_mtime = st->st_mtime;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx->mbox->mbox_hdr.sync_size = st->st_size;
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen mbox_sync_index_update_ext_header(sync_ctx->mbox, sync_ctx->t);
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen /* only reason not to have UID validity at this point is if the file
f2b79667fc7a8f7c2c72cad18bd71d49730e36f6Timo Sirainen is entirely empty. In that case just make up a new one if needed. */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen i_assert(sync_ctx->base_uid_validity != 0 || st->st_size <= 0);
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ?
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen mbox_get_uidvalidity_next(sync_ctx->mbox->box.list);
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen offsetof(struct mail_index_header, uid_validity),
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input) &&
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen &sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (sync_ctx->last_nonrecent_uid < sync_ctx->hdr->first_recent_uid) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* other sessions have already marked more messages as
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* mark recent messages */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen view = mail_index_transaction_open_updated_view(sync_ctx->t);
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen if (mail_index_lookup_seq_range(view, sync_ctx->last_nonrecent_uid + 1,
88553367d677170a4b703b9d52aac9eabf91c656Timo Sirainen index_mailbox_set_recent_seq(&sync_ctx->mbox->box,
4d10cf8c7879ccd377e7fb136913b2a258ba8d93Timo Sirainen sync_ctx->next_uid : sync_ctx->last_nonrecent_uid + 1;
4d10cf8c7879ccd377e7fb136913b2a258ba8d93Timo Sirainen if (sync_ctx->hdr->first_recent_uid < first_recent_uid) {
4d10cf8c7879ccd377e7fb136913b2a258ba8d93Timo Sirainen offsetof(struct mail_index_header, first_recent_uid),
4d10cf8c7879ccd377e7fb136913b2a258ba8d93Timo Sirainen &first_recent_uid, sizeof(first_recent_uid), FALSE);
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainenstatic void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen index_sync_changes_reset(sync_ctx->sync_changes);
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen mail_index_sync_reset(sync_ctx->index_sync_ctx);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen index_mailbox_reset_uidvalidity(&sync_ctx->mbox->box);
7c95b03620a03a43dd72d39608cea5fc77393ad6Timo Sirainen sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainenstatic int mbox_sync_do(struct mbox_sync_context *sync_ctx,
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen struct mbox_index_header *mbox_hdr = &sync_ctx->mbox->mbox_hdr;
e5ee67f18b03015c88b579c8c1f17ebe6ce19b76Timo Sirainen unsigned int i;
de4288b7369945a31c4001add9445fd0195a358dTimo Sirainen if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* forcing a full sync. assume file has changed. */
79c9f3069bde51f799a64ca5923d68c9a5bc2ad2Timo Sirainen } else if ((uint32_t)st->st_mtime == mbox_hdr->sync_mtime &&
79c9f3069bde51f799a64ca5923d68c9a5bc2ad2Timo Sirainen (uint64_t)st->st_size == mbox_hdr->sync_size) {
79c9f3069bde51f799a64ca5923d68c9a5bc2ad2Timo Sirainen /* file is fully synced */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (mbox_hdr->dirty_flag && (flags & MBOX_SYNC_UNDIRTY) != 0)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen } else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
59ef34eafaf74d31ed88af444b22d1a0738a30aaTimo Sirainen (uint64_t)st->st_size == mbox_hdr->sync_size) {
59ef34eafaf74d31ed88af444b22d1a0738a30aaTimo Sirainen /* we want to do full syncing. always do this if
59ef34eafaf74d31ed88af444b22d1a0738a30aaTimo Sirainen file size hasn't changed but timestamp has. it most
59ef34eafaf74d31ed88af444b22d1a0738a30aaTimo Sirainen likely means that someone had modified some header
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen and we probably want to know about it */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* see if we can delay syncing the whole file.
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen normally we only notice expunges and appends
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen in partial syncing. */
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen for (i = 0;;) {
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial);
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen /* a) partial sync didn't work
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen b) we ran out of UIDs
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen c) syncing had errors */
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen (sync_ctx->errors || sync_ctx->renumber_uids)) {
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* fixing a broken mbox state, be sure to write
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen the changes (except if we're readonly). */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (++i == 3)
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen /* only syncs left should be just appends (and their updates)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen which weren't synced yet for some reason (crash). we'll just
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen ignore them, as we've overwritten them above. */
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen index_sync_changes_reset(sync_ctx->sync_changes);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (sync_ctx->base_uid_last != sync_ctx->next_uid-1 &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* Rewrite uid_last in X-IMAPbase header if we've seen it
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen (ie. the file isn't empty) */
e5ee67f18b03015c88b579c8c1f17ebe6ce19b76Timo Sirainen if (mbox_sync_update_index_header(sync_ctx) < 0)
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainenint mbox_sync_header_refresh(struct mbox_mailbox *mbox)
const void *data;
if (data_size == 0) {
unsigned int lock_id;
int ret;
if (ret == 0) {
return ret;
bool empty;
bool *empty_r)
unsigned int *lock_id)
if (!changed)
*lock_id = 0;
if (changed) {
if (ret <= 0) {
if (ret < 0)
return ret;
bool expunged;
if (uid == 0) {
goto nothing_to_do;
if (*lock_id == 0) {
goto again;
if (ret < 0)
!readonly) {
return ret;
unsigned int lock_id = 0;
int ret;
if (lock_id != 0) {
if (ret < 0) {
unsigned int read_lock_id = 0;
return ret;
struct mailbox_sync_context *
int ret = 0;