mbox-sync.c revision 6060b7c8edf8fce73470d0df6a2479b69b01c537
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen required disk I/O. We may need to:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing X-UID and X-IMAPbase headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing or broken Content-Length header if there's space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Expunge specified messages
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Here's how we do it:
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Start reading the mails from the beginning
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen of them, remember how much each message has and offset to beginning of the
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If header needs to be rewritten and there's enough space, do it
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If we didn't have enough space, remember how much was missing
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Continue reading and counting the padding in each message. If available
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen padding is enough to rewrite all the previous messages needing it, do it
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - When we encounter expunged message, treat all of it as padding and
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen rewrite previous messages if needed (and there's enough space).
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen Afterwards keep moving messages backwards to fill the expunged space.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen Moving is done by rewriting each message's headers, with possibly adding
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen missing Content-Length header and padding. Message bodies are moved
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen without modifications.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If we encounter end of file, grow the file and rewrite needed messages
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Rewriting is done by moving message body forward, rewriting message's
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen header and doing the same for previous message, until all of them are
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen/* The text below was taken exactly as c-client wrote it to my mailbox,
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen so it's probably copyrighted by University of Washington. */
f7d43647acc6dc80064c8c4cacf5bf86f754c530Timo Sirainen"This text is part of the internal format of your mail folder, and is not\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"a real message. It is created automatically by the mail system software.\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"If deleted, important folder data will be lost, and it will be re-created\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"with the data reset to initial values.\n"
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
8153fdec343e40e2a78f5c12353e89b994b28f74Timo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainenstatic void mbox_sync_array_delete_to(array_t *syncs_arr, uint32_t last_uid)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen ARRAY_SET_TYPE(syncs_arr, struct mail_index_sync_rec);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen syncs = array_get_modifyable(syncs_arr, &count);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* keep it */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen /* get EOF */
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 && !mail_ctx->pseudo) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* need to add 'O' flag to Status-header */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_buf_have_expunges(array_t *syncs_arr)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen ARRAY_SET_TYPE(syncs_arr, struct mail_index_sync_rec);
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen unsigned int i, count;
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen for (i = 0; i < count; i++) {
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen if (syncs[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen /* nothing for this or the future ones */
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen mbox_sync_array_delete_to(&sync_ctx->syncs, uid);
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND &&
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE ||
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
36e2fa21c22452470c1509cc63de20f7415c7b5eTimo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) {
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen *sync_expunge_r = mbox_sync_buf_have_expunges(&sync_ctx->syncs);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainenvoid mbox_sync_apply_index_syncs(struct mbox_sync_context *sync_ctx,
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen unsigned int i, count;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen for (i = 0; i < count; i++) {
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_sync_flags_apply(&syncs[i], &mail->flags);
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* no existing keywords */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* adding, create the array */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen unsigned int,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* externally expunged message, remove from index */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (ret == 0 && uid < sync_ctx->idx_next_uid) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* this UID was already in index and it was expunged */
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen "mbox sync: Expunged message reappeared in mailbox %s "
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
2ae575a66f2a302f047f6de062a70b75f8bebc7bTimo Sirainen mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen "mbox sync: UID inserted in the middle of mailbox %s "
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "(%u > %u)", sync_ctx->mbox->path, rec->uid, uid);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen unsigned char hdr_md5_sum[],
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (mail_index_lookup_ext(sync_ctx->sync_view,
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen /* externally expunged message, remove from index */
f0f9c8e94abac18f8acd91b9e724c4c32863723aTimo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen /* see if from_offset needs updating */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mail_index_lookup_ext(sync_ctx->sync_view,
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen *((const uint64_t *)data) == mail->from_offset)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen keywords = !array_is_created(&mail_ctx->mail.keywords) ?
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_keywords_create(sync_ctx->t, NULL) :
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_keywords_create_from_indexes(sync_ctx->t,
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainenstatic int mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen /* new message */
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* see if we need to update flags in index file. the flags in
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync records are automatically applied to rec->flags at the
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen end of index syncing, so calculate those new flags first */
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen /* get old keywords */
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen ARRAY_CREATE(&idx_mail.keywords, pool_datastack_create(),
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen unsigned int, 32);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mail_index_lookup_keywords(sync_ctx->sync_view,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_sync_apply_index_syncs(sync_ctx, &idx_mail,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen#define SYNC_FLAGS (MAIL_RECENT | MAIL_INDEX_MAIL_FLAG_DIRTY)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* flags are dirty. ignore whatever was in the mbox,
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen but update recent/dirty flag states if needed. */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* keep index's internal flags */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* flags other than recent/dirty have changed */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* drop recent flag (it can only be dropped) */
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen /* dirty flag state changed */
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0 &&
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen !array_cmp(&idx_mail.keywords, &mail_ctx->mail.keywords))
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen (rec == NULL || (rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0 ||
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen index_mailbox_set_recent(&sync_ctx->mbox->ibox,
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen rewriting would just move it anyway. */
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen struct istream *input = ctx->sync_ctx->file_input;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen const unsigned char *data;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen const char *str;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen unsigned int i;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen i_assert(sync_ctx->base_uid_last_offset != 0);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen /* first check that the 10 bytes are there and they're exactly as
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen expected. just an extra safety check to make sure we never write
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen to wrong location in the mbox file. */
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf),
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pread_full()");
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen "X-IMAPbase uid-last unexpectedly points outside "
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen for (i = 0, uid_last = 0; i < sizeof(buf); i++) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "X-IMAPbase uid-last unexpectedly lost in mbox file %s",
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* and write it */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen str = t_strdup_printf("%010u", sync_ctx->next_uid - 1);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
16aba431c576c1dbd99cbaae4f9d65eea9ad73c2Timo Sirainen sync_ctx->base_uid_last = sync_ctx->next_uid - 1;
16aba431c576c1dbd99cbaae4f9d65eea9ad73c2Timo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx)
16aba431c576c1dbd99cbaae4f9d65eea9ad73c2Timo Sirainen if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str),
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenstatic void update_from_offsets(struct mbox_sync_context *sync_ctx)
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen unsigned int i, count;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen for (i = 0; i < count; i++) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_index_update_ext(sync_ctx->t, mails[i].idx_seq,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen mail_ctx->mail.offset = mail_ctx->mail.from_offset;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mail_ctx->body_offset - mail_ctx->mail.from_offset +
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen /* expunging first message, fix space to contain next
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen message's \n header too since it will be removed. */
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen /* uid-last offset is invalid now */
ab3c1eab9ca13916358a9e8b12df8212fefb7dbfTimo Sirainen mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
3021a062b16ff0138408be6107d6bcd0ced280b9Timo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen /* move the header backwards to fill expunged space */
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen /* we're moving this mail to beginning of file.
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen skip the initial \n (it's already counted in
8907d617ce7c4f390c0f42f6f694db2fecdd5775Timo Sirainen expunged_space) */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* read the From-line before rewriting overwrites it */
1276e0340fe29495b6694dc7508f070cf6fca1cfTimo Sirainen ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
2be66b9eddad3841a1195fe9aeb1eaf0f28f1116Timo Sirainen /* rewrite successful, write From-line to
2be66b9eddad3841a1195fe9aeb1eaf0f28f1116Timo Sirainen new location */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* didn't have enough space, move the offset
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen back so seeking into it doesn't fail */
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen /* mark it dirty and do it later */
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* nothing to do */
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
60d3fa9883237e896a8704275b6116fa46f7ffdaTimo Sirainen /* first mail with no space to write it */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* create dummy message to describe the expunged data */
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen sync_ctx->space_diff = sync_ctx->expunged_space;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen i_assert(sync_ctx->space_diff < -mail_ctx->mail.space);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen uoff_t end_offset, move_diff, extra_space, needed_space;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 ||
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->mail.offset == mail_ctx->hdr_offset);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen array_append(&sync_ctx->mails, &mail_ctx->mail, 1);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* we have enough space now */
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen /* this message was expunged. fill more or less of the space.
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen space_diff now consists of a negative "bytes needed" sum,
8d5991f5c4a8840bf1ea754093dbec505564ab78Timo Sirainen plus the expunged space of this message. so it contains how
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen many bytes of _extra_ space we have. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen i_assert(mail_ctx->mail.space >= sync_ctx->space_diff);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* don't waste too much on padding */
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen /* this message gave enough space from headers. rewriting stops
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen at the end of this message's headers. */
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen if (mbox_sync_rewrite(sync_ctx, end_offset, move_diff, extra_space,
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen /* mail_ctx may contain wrong data after rewrite, so make sure we
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen don't try to access it */
829c036d4ddfbd9ea49bd8a7c54e3057177d346eTimo Sirainenmbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_storage_set_error(STORAGE(mbox->storage),
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen "Mailbox isn't a valid mbox file");
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted);
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen "Error seeking back to original "
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "offset %s in mbox file %s",
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen else if (mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid) < 0) {
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* set to -1, since it's always increased later */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* this mbox has pseudo mail which contains the X-IMAP header */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen sync_ctx->dest_first_mail = sync_ctx->seq == 0;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
8153fdec343e40e2a78f5c12353e89b994b28f74Timo Sirainenmbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen struct mail_index_view *sync_view = sync_ctx->sync_view;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen if (mail_index_lookup_uid_range(sync_view, uid, (uint32_t)-1,
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* doesn't exist anymore, seek to end of file */
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen st = i_stream_stat(sync_ctx->mbox->mbox_file_stream);
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen "i_stream_stat()");
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream,
cece2b9cd692c06025cc0a7a0ff54d996a8c90efTimo Sirainen "Error seeking to end of mbox file %s",
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen mail_index_view_get_messages_count(sync_view) + 1;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainenstatic int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx,
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* delete sync records up to next message. so if there's still
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen something left in array, it means the next message needs modifying */
0a6f8311541ae59381171620b77f82be58be562eTimo Sirainen mbox_sync_array_delete_to(&sync_ctx->syncs, next_uid);
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* we can skip forward to next record which needs updating. */
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen ret = mbox_sync_seek_to_uid(sync_ctx, next_uid);
8153fdec343e40e2a78f5c12353e89b994b28f74Timo Sirainen /* if there's no sync records left, we can stop. except if
0a6f8311541ae59381171620b77f82be58be562eTimo Sirainen this is a dirty sync, check if there are new messages. */
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
0a6f8311541ae59381171620b77f82be58be562eTimo Sirainen ret = mbox_sync_seek_to_seq(sync_ctx, messages_count);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* seek failed because the offset is dirty. just ignore and
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen continue from where we are now. */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainenstatic int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* always start from first message so we can read X-IMAP or
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen X-IMAPbase header */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (mail_ctx->seq == 1 && sync_ctx->base_uid_validity != 0 &&
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "UIDVALIDITY changed (%u -> %u) "
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "in mbox file %s",
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_index_mark_corrupted(sync_ctx->mbox->ibox.index);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* UID ordering problems, resync everything to make
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen sure we get everything right */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* UID found but it's broken */
cdaf255d6a3daeef0ac85edaa60bfa6d1f945bffTimo Sirainen } else if (uid == 0 &&
8153fdec343e40e2a78f5c12353e89b994b28f74Timo Sirainen /* If we can't use/store X-UID header, use MD5 sum.
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen Also check for existing MD5 sums when we're actually
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen able to write X-UIDs. */
b8765f6093ab35fc2345293d78132d35794cbff5Timo Sirainen /* from now on, don't skip anything */
b8765f6093ab35fc2345293d78132d35794cbff5Timo Sirainen /* get all sync records related to this message */
2ae575a66f2a302f047f6de062a70b75f8bebc7bTimo Sirainen /* missing/broken X-UID. all the rest of the mails
cdaf255d6a3daeef0ac85edaa60bfa6d1f945bffTimo Sirainen need new UIDs. */
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen if (mbox_sync_update_index(mail_ctx, rec) < 0)
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen if (mbox_sync_handle_missing_space(mail_ctx) < 0)
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen /* move the body */
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen } else if (partial) {
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1,
8907d617ce7c4f390c0f42f6f694db2fecdd5775Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input)) {
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen /* rest of the messages in index don't exist -> expunge them */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
907723f35f4d3dfc774ca42d00a8a7b8ef90dd5dTimo Sirainenstatic int mbox_write_pseudo(struct mbox_sync_context *sync_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen uid_validity = sync_ctx->base_uid_validity != 0 ?
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen "From: Mail System Internal Data <MAILER-DAEMON@%s>\n"
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen "Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA"
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen "\nMessage-ID: <%s@%s>\n"
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen "X-IMAP: %u %010u\n"
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen "Status: RO\n"
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen mbox_from_create("MAILER_DAEMON", ioloop_time),
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen my_hostname, dec2str(ioloop_time), my_hostname,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "pwrite_full()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* out of disk space, truncate to empty */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen sync_ctx->base_uid_last = sync_ctx->next_uid-1;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenstatic int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
3cfff0ca01961d885bdbd6ef08d761880116af07Timo Sirainen uoff_t file_size, offset, padding, trailer_size;
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
9d2040fbb941f411d57fd850b4cdc3b1cccc1168Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen i_assert(file_size >= sync_ctx->file_input->v_offset);
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen trailer_size = file_size - sync_ctx->file_input->v_offset;
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff);
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen sync_ctx->space_diff += sync_ctx->expunged_space;
2be66b9eddad3841a1195fe9aeb1eaf0f28f1116Timo Sirainen "file_set_size()");
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* copy trailer, then truncate the file */
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen "i_stream_stat()");
09b9cbde9e7a0f9adea1fb054a7c62f35ad901e1Timo Sirainen if (file_size == (uoff_t)sync_ctx->expunged_space) {
09b9cbde9e7a0f9adea1fb054a7c62f35ad901e1Timo Sirainen /* everything deleted, the trailer_size still contains
09b9cbde9e7a0f9adea1fb054a7c62f35ad901e1Timo Sirainen the \n trailer though */
686cb2418d4daa7bc747d3551783c1895e7c41b9Timo Sirainen i_assert(file_size >= sync_ctx->expunged_space + trailer_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen offset = file_size - sync_ctx->expunged_space - trailer_size;
f5b919e9b07dfd9d2401b998ef8759e5f0312719Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainenstatic int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* only reason not to have UID validity at this point is if the file
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen is entirely empty. In that case just make up a new one if needed. */
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen i_assert(sync_ctx->base_uid_validity != 0 || st->st_size == 0);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity ||
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen (unsigned int)ioloop_time;
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen offsetof(struct mail_index_header, uid_validity),
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input) &&
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
eb276c05bf6b0a383c772d61e31cf09a8dbd36c7Timo Sirainen &sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((uint32_t)st->st_mtime != sync_ctx->hdr->sync_stamp &&
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen offsetof(struct mail_index_header, sync_stamp),
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen if ((uint64_t)st->st_size != sync_ctx->hdr->sync_size &&
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen offsetof(struct mail_index_header, sync_size),
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->mbox->mbox_dirty_stamp = st->st_mtime;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->mbox->mbox_dirty_size = st->st_size;
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainenstatic void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen mail_index_sync_reset(sync_ctx->index_sync_ctx);
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainenstatic int mbox_sync_do(struct mbox_sync_context *sync_ctx,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen "i_stream_stat()");
aa38d1a0945f0bc13a225d043f53fad2eec666b1Timo Sirainen /* forcing a full sync. assume file has changed. */
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen } else if ((uint32_t)st->st_mtime == sync_ctx->hdr->sync_stamp &&
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen (uint64_t)st->st_size == sync_ctx->hdr->sync_size) {
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* file is fully synced */
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen } else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
aa38d1a0945f0bc13a225d043f53fad2eec666b1Timo Sirainen (uint64_t)st->st_size == sync_ctx->hdr->sync_size) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen /* we want to do full syncing. always do this if
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen file size hasn't changed but timestamp has. it most
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen likely means that someone had modified some header
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen and we probably want to know about it */
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen /* see if we can delay syncing the whole file.
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen normally we only notice expunges and appends
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen in partial syncing. */
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial);
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen /* partial syncing didn't work, do it again */
c263e92d73889da530b308c9ab28b4b74031550eTimo Sirainen sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view,
325f4573edfa5b751832ac01023f3e81be992bf0Timo Sirainen ret = mbox_sync_loop(sync_ctx, &mail_ctx, FALSE);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* only syncs left should be just appends (and their updates)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen which weren't synced yet for some reason (crash). we'll just
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen ignore them, as we've overwritten them above. */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (mbox_sync_update_index_header(sync_ctx) < 0)
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainenint mbox_sync_has_changed(struct mbox_mailbox *mbox, int leave_dirty)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* read-only stream */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mbox_set_syscall_error(mbox, "i_stream_stat()");
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen if ((uint32_t)st->st_mtime == hdr->sync_stamp &&
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* fully synced */
7c95b03620a03a43dd72d39608cea5fc77393ad6Timo Sirainen return st->st_mtime != mbox->mbox_dirty_stamp ||
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainenint mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen unsigned int lock_id = 0;
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen int leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen if ((changed = mbox_sync_has_changed(mbox, leave_dirty)) < 0) {
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen /* we just want to lock it for reading. if mbox hasn't been
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen modified don't do any syncing. */
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen /* have to sync to make sure offsets have stayed the same */
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen /* reopen input stream to make sure it has nothing buffered */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* we're most likely modifying the mbox while syncing, just
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen lock it for writing immediately. the mbox must be locked
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen before index syncing is started to avoid deadlocks, so we
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen don't have much choice either (well, easy ones anyway). */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen int lock_type = mbox->mbox_readonly ? F_RDLCK : F_WRLCK;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (mbox_lock(mbox, lock_type, &lock_id) <= 0)
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen ret = mail_index_sync_begin(mbox->ibox.index, &index_sync_ctx,
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainen if (!changed && !mail_index_sync_have_more(index_sync_ctx)) {
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainen /* nothing to do */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* index may need to do internal syncing though, so commit
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen instead of rollbacking. */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (mail_index_sync_commit(index_sync_ctx) < 0) {
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen /* ok, we have something to do but no locks. we'll have to
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen restart syncing to avoid deadlocking. */
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen sync_ctx.hdr = mail_index_get_header(sync_view);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx.from_line = str_new(default_pool, 256);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx.header = str_new(default_pool, 4096);
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE, TRUE);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx.mail_keyword_pool = pool_alloconly_create("keywords", 4096);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* make sure we've read the latest keywords in index */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen (void)mail_index_get_keywords(mbox->ibox.index);
757726d9acbd04cf0d0d4be8ce14e11525476a0bTimo Sirainen sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen sync_ctx.write_fd = sync_ctx.mbox->mbox_readonly ? -1 :
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx.delay_writes = sync_ctx.mbox->mbox_readonly ||
4d10cf8c7879ccd377e7fb136913b2a258ba8d93Timo Sirainen else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0) {
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen else if (mail_index_sync_commit(index_sync_ctx) < 0) {
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen if (sync_ctx.base_uid_last != sync_ctx.next_uid-1 &&
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* Rewrite uid_last in X-IMAPbase header if we've seen it
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen (ie. the file isn't empty) */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (ret == 0 && mbox->mbox_lock_type == F_WRLCK &&
7c95b03620a03a43dd72d39608cea5fc77393ad6Timo Sirainen if (lock_id != 0 && mbox->mbox_lock_type != F_RDLCK) {
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* drop to read lock */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen unsigned int read_lock_id = 0;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0)
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (lock_id != 0 && (flags & MBOX_SYNC_LOCK_READING) == 0) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen notice changes made by others .. and this has to be done
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen even if lock_reading is set.. except if
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_sync_dirty = TRUE */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenmbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
59ef34eafaf74d31ed88af444b22d1a0738a30aaTimo Sirainen if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0)