mbox-sync.c revision f6699a08521aacc4c2bb5b6175691dad5f715f8c
5a580c3a38ced62d4bcc95b8ac7c4f2935b5d294Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen required disk I/O. We may need to:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing X-UID and X-IMAPbase headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing or broken Content-Length header if there's space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Expunge specified messages
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Here's how we do it:
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Start reading the mails from the beginning
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen of them, remember how much each message has and offset to beginning of the
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If header needs to be rewritten and there's enough space, do it
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If we didn't have enough space, remember how much was missing
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Continue reading and counting the padding in each message. If available
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen padding is enough to rewrite all the previous messages needing it, do it
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - When we encounter expunged message, treat all of it as padding and
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen rewrite previous messages if needed (and there's enough space).
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen Afterwards keep moving messages backwards to fill the expunged space.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen Moving is done by rewriting each message's headers, with possibly adding
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen missing Content-Length header and padding. Message bodies are moved
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen without modifications.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If we encounter end of file, grow the file and rewrite needed messages
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Rewriting is done by moving message body forward, rewriting message's
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen header and doing the same for previous message, until all of them are
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen/* The text below was taken exactly as c-client wrote it to my mailbox,
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen so it's probably copyrighted by University of Washington. */
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"This text is part of the internal format of your mail folder, and is not\n" \
f7d43647acc6dc80064c8c4cacf5bf86f754c530Timo Sirainen"a real message. It is created automatically by the mail system software.\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"If deleted, important folder data will be lost, and it will be re-created\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"with the data reset to initial values.\n"
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainenvoid mbox_sync_set_critical(struct mbox_sync_context *sync_ctx,
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen const char *fmt, ...)
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen "mbox file %s was modified while we were syncing, "
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen "check your locking settings", sync_ctx->mbox->path);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainenvoid mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx)
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen /* Do this even if ext_modified is already set. Expunging code relies
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen on last_stat being updated. */
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen if (st.st_size != sync_ctx->last_stat.st_size ||
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainenvoid mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty)
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen /* just mark the stat as dirty. */
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0)
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainenstatic void mbox_sync_array_delete_to(ARRAY_TYPE(sync_recs) *syncs_arr,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen syncs = array_get_modifiable(syncs_arr, &count);
178511b57faa7c3f8203dd8b7e4059d00cbfc23aTimo Sirainen /* keep it */
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen /* get EOF */
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
1e76a5b92f9d82d557f81f080f3dfad1c9d8f200Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
ced118ac5caf6fe83d34339c2c65c63b2aa768acTimo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
b08b33d1f5ce3721dc2d83586c9cb0ca141331fdTimo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 &&
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen /* need to add 'O' flag to Status-header */
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainenstatic bool mbox_sync_buf_have_expunges(ARRAY_TYPE(sync_recs) *syncs_arr)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen unsigned int i, count;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen for (i = 0; i < count; i++) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (syncs[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* nothing for this or the future ones */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_sync_array_delete_to(&sync_ctx->syncs, uid);
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND &&
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE ||
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen /* we're not going to write these yet */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen *sync_expunge_r = mbox_sync_buf_have_expunges(&sync_ctx->syncs);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenvoid mbox_sync_apply_index_syncs(struct mbox_sync_context *sync_ctx,
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen unsigned int i, count;
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen for (i = 0; i < count; i++) {
8af07808ba203f8709e2ff9eaf2291e1c4a4d53dTimo Sirainen mail_index_sync_flags_apply(&syncs[i], &mail->flags);
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* no existing keywords */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* adding, create the array */
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen i_assert(ret != 0); /* we should be looking at head index */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* externally expunged message, remove from index */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen if (rec == NULL && uid < sync_ctx->idx_next_uid) {
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* this UID was already in index and it was expunged */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen "mbox sync: Expunged message reappeared in mailbox %s "
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen "(UID %u < %u, seq=%u, idx_msgs=%u)",
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen sync_ctx->mbox->path, uid, sync_ctx->idx_next_uid,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen "mbox sync: UID inserted in the middle of mailbox %s "
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen "(%u > %u, seq=%u, idx_msgs=%u)", sync_ctx->mbox->path,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen rec->uid, uid, sync_ctx->seq, messages_count);
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainenstatic int mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen unsigned char hdr_md5_sum[],
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen if (mail_index_lookup_ext(sync_ctx->sync_view,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0)
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* externally expunged message, remove from index */
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* see if from_offset needs updating */
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen if (mail_index_lookup_ext(sync_ctx->sync_view,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen *((const uint64_t *)data) == mail->from_offset)
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainenmbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen keywords = !array_is_created(&mail_ctx->mail.keywords) ?
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_keywords_create(sync_ctx->t, NULL) :
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen mail_index_keywords_create_from_indexes(sync_ctx->t,
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainenmbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx)
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainenstatic int mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen struct mailbox *box = &sync_ctx->mbox->ibox.box;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen /* new message */
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* see if we need to update flags in index file. the flags in
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen sync records are automatically applied to rec->flags at the
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen end of index syncing, so calculate those new flags first */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* get old keywords */
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen if (mail_index_lookup_keywords(sync_ctx->sync_view,
16aba431c576c1dbd99cbaae4f9d65eea9ad73c2Timo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mbox_sync_apply_index_syncs(sync_ctx, &idx_mail,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (sync_type != 0 && box->v.sync_notify != NULL)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen#define SYNC_FLAGS (MAIL_RECENT | MAIL_INDEX_MAIL_FLAG_DIRTY)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen /* flags are dirty. ignore whatever was in the mbox,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen but update recent/dirty flag states if needed. */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* keep index's internal flags */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* flags other than recent/dirty have changed */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen /* drop recent flag (it can only be dropped) */
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen /* dirty flag state changed */
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0 &&
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen /* see if we need to update md5 sum. */
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen if (mbox_sync_update_md5_if_changed(mail_ctx) < 0)
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen (rec == NULL || (rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0 ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen index_mailbox_set_recent(&sync_ctx->mbox->ibox,
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen rewriting would just move it anyway. */
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen bool nocheck = rec == NULL || sync_ctx->expunged_space > 0;
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen struct istream *input = ctx->sync_ctx->file_input;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen const unsigned char *data;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx)
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen const char *str;
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen unsigned int i;
c35d0c3eb4ba89432896e1f9770ab31cc63c232bTimo Sirainen i_assert(sync_ctx->base_uid_last_offset != 0);
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen /* first check that the 10 bytes are there and they're exactly as
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen expected. just an extra safety check to make sure we never write
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen to wrong location in the mbox file. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf),
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pread_full()");
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "X-IMAPbase uid-last unexpectedly points outside "
c35d0c3eb4ba89432896e1f9770ab31cc63c232bTimo Sirainen for (i = 0, uid_last = 0; i < sizeof(buf); i++) {
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen "X-IMAPbase uid-last unexpectedly lost in mbox file %s",
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen /* and write it */
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen str = t_strdup_printf("%010u", sync_ctx->next_uid - 1);
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->base_uid_last = sync_ctx->next_uid - 1;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str),
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()");
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic void update_from_offsets(struct mbox_sync_context *sync_ctx)
8d5991f5c4a8840bf1ea754093dbec505564ab78Timo Sirainen unsigned int i, count;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen for (i = 0; i < count; i++) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mails[i].idx_seq == 0 || mails[i].expunged)
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen mail_index_update_ext(sync_ctx->t, mails[i].idx_seq,
2cb565cd978aafd5714792b5161889986d49e431Timo Sirainenstatic void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
5486e1e123484f3a8d7d895e8ac41156f2a96305Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
5486e1e123484f3a8d7d895e8ac41156f2a96305Timo Sirainen struct mailbox *box = &sync_ctx->mbox->ibox.box;
5486e1e123484f3a8d7d895e8ac41156f2a96305Timo Sirainen mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq);
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen mail_ctx->mail.offset = mail_ctx->mail.from_offset;
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen mail_ctx->body_offset - mail_ctx->mail.from_offset +
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* expunging first message, fix space to contain next
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen message's \n header too since it will be removed. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) {
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen /* uid-last offset is invalid now */
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen sync_ctx->expunged_space += mail_ctx->mail.space;
930dcf1576f99057ad572420d9c75f3212e46a2eTimo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
930dcf1576f99057ad572420d9c75f3212e46a2eTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen /* move the header backwards to fill expunged space */
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen orig_from_offset = mail_ctx->mail.from_offset;
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen /* we're moving this mail to beginning of file.
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen skip the initial \n (it's already counted in
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen expunged_space) */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* read the From-line before rewriting overwrites it */
a614397cf1a4dde152eb1a38493a6ec3d817da16Timo Sirainen ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* rewrite successful, write From-line to
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen new location */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* didn't have enough space, move the offset
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen back so seeking into it doesn't fail */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen mail_ctx->mail.from_offset = orig_from_offset;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* mark it dirty and do it later */
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* nothing to do */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen /* first mail with no space to write it */
f12aabfeba81f0d741971d2b7e4a5008eb4383caTimo Sirainen /* create dummy message to describe the expunged data */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen sync_ctx->space_diff = sync_ctx->expunged_space;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen i_assert(sync_ctx->space_diff < -mail_ctx->mail.space);
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainenmbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
cece2b9cd692c06025cc0a7a0ff54d996a8c90efTimo Sirainen uoff_t end_offset, move_diff, extra_space, needed_space;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 ||
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen mail_ctx->mail.offset == mail_ctx->hdr_offset);
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen if (array_is_created(&mail_ctx->mail.keywords)) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* mail's keywords are allocated from a pool that's cleared
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen for each mail. we'll need to copy it to something more
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen permanent. */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool,
a10e5606a9e93f49cf13b3a35c8dc3f5d6ab5909Timo Sirainen array_append_array(&keywords_copy, &mail_ctx->mail.keywords);
4b9f99761df5014c659cd87fddaf6854af428cfcTimo Sirainen array_append(&sync_ctx->mails, &mail_ctx->mail, 1);
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* we have enough space now */
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* this message was expunged. fill more or less of the space.
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen space_diff now consists of a negative "bytes needed" sum,
4b9f99761df5014c659cd87fddaf6854af428cfcTimo Sirainen plus the expunged space of this message. so it contains how
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen many bytes of _extra_ space we have. */
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen i_assert(mail_ctx->mail.space >= sync_ctx->space_diff);
0a6f8311541ae59381171620b77f82be58be562eTimo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
a10e5606a9e93f49cf13b3a35c8dc3f5d6ab5909Timo Sirainen needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
0a6f8311541ae59381171620b77f82be58be562eTimo Sirainen /* don't waste too much on padding */
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen /* this message gave enough space from headers. rewriting stops
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen at the end of this message's headers. */
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen /* mail_ctx may contain wrong data after rewrite, so make sure we
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen don't try to access it */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenmbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_storage_set_error(&mbox->storage->storage,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "Mailbox isn't a valid mbox file");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen "Message was expunged unexpectedly "
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen "Error seeking back to original "
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen "offset %s in mbox file %s",
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen else if (mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid) < 0) {
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen /* set to -1, since it's always increased later */
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen /* this mbox has pseudo mail which contains the X-IMAP header */
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen sync_ctx->dest_first_mail = sync_ctx->seq == 0;
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenmbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen struct mail_index_view *sync_view = sync_ctx->sync_view;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if (mail_index_lookup_uid_range(sync_view, uid, (uint32_t)-1,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_storage_set_index_error(&sync_ctx->mbox->ibox);
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen /* doesn't exist anymore, seek to end of file */
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen st = i_stream_stat(sync_ctx->file_input, TRUE);
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen "i_stream_stat()");
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "Error seeking to end of mbox file %s",
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen mail_index_view_get_messages_count(sync_view) + 1;
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainenstatic int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx,
d31c4d7c161f9d7efa59964c7c958e83e05b218cTimo Sirainen /* delete sync records up to next message. so if there's still
d31c4d7c161f9d7efa59964c7c958e83e05b218cTimo Sirainen something left in array, it means the next message needs modifying */
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen mbox_sync_array_delete_to(&sync_ctx->syncs, next_uid);
cdaf255d6a3daeef0ac85edaa60bfa6d1f945bffTimo Sirainen /* we can skip forward to next record which needs updating. */
2ae575a66f2a302f047f6de062a70b75f8bebc7bTimo Sirainen ret = mbox_sync_seek_to_uid(sync_ctx, next_uid);
2ae575a66f2a302f047f6de062a70b75f8bebc7bTimo Sirainen /* if there's no sync records left, we can stop. except if
cdaf255d6a3daeef0ac85edaa60bfa6d1f945bffTimo Sirainen this is a dirty sync, check if there are new messages. */
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen ret = mbox_sync_seek_to_seq(sync_ctx, messages_count);
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen /* seek failed because the offset is dirty. just ignore and
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen continue from where we are now. */
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainenstatic bool mbox_sync_uidvalidity_changed(struct mbox_sync_context *sync_ctx)
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
e9d68b41c007f0e545de361f8012f6f231bfec8bTimo Sirainen i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s",
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainenstatic int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen /* always start from first message so we can read X-IMAP or
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen X-IMAPbase header */
1e76a5b92f9d82d557f81f080f3dfad1c9d8f200Timo Sirainen /* expunge everything */
8907d617ce7c4f390c0f42f6f694db2fecdd5775Timo Sirainen while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_uidvalidity_changed(sync_ctx)) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen /* UID ordering problems, resync everything to make
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen sure we get everything right */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "UIDs broken with partial sync in mbox file %s",
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* UID found but it's broken */
6f25019b337e27600159b596824da08732965576Timo Sirainen } else if (uid == 0 &&
6f25019b337e27600159b596824da08732965576Timo Sirainen /* If we can't use/store X-UID header, use MD5 sum.
6f25019b337e27600159b596824da08732965576Timo Sirainen Also check for existing MD5 sums when we're actually
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen able to write X-UIDs. */
d31c4d7c161f9d7efa59964c7c958e83e05b218cTimo Sirainen /* get all sync records related to this message. with pseudo
d31c4d7c161f9d7efa59964c7c958e83e05b218cTimo Sirainen message just get the first sync record so we can jump to
d31c4d7c161f9d7efa59964c7c958e83e05b218cTimo Sirainen it with partial seeking. */
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen /* if it was set, it was for the next message */
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen /* message wasn't found from index. we have to
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen read everything from now on, no skipping */
eeeee5297c71eb8badfbdff5241ac05851f1c21fTimo Sirainen /* missing/broken X-UID. all the rest of the mails
eeeee5297c71eb8badfbdff5241ac05851f1c21fTimo Sirainen need new UIDs. */
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen /* oh no, we're out of UIDs. this shouldn't
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen happen normally, so just try to get it fixed
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen without crashing. */
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen "Out of UIDs, renumbering them in mbox "
afe9987e8b791a6421332fcb334401b54a2169daTimo Sirainen if (mbox_sync_update_index(mail_ctx, rec) < 0)
a97a3ce5d82b16e8979de2d6cafbf6b0a129fe4eTimo Sirainen offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
a97a3ce5d82b16e8979de2d6cafbf6b0a129fe4eTimo Sirainen if (mbox_sync_handle_missing_space(mail_ctx) < 0)
a97a3ce5d82b16e8979de2d6cafbf6b0a129fe4eTimo Sirainen /* move the body */
a97a3ce5d82b16e8979de2d6cafbf6b0a129fe4eTimo Sirainen } else if (partial) {
a97a3ce5d82b16e8979de2d6cafbf6b0a129fe4eTimo Sirainen ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input)) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* rest of the messages in index don't exist -> expunge them */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
8166e8a706078efc71923719ca285e84902883c0Timo Sirainen /* once we get around to writing the changes, we'll need to do
8166e8a706078efc71923719ca285e84902883c0Timo Sirainen a full sync to avoid the "UIDs broken in partial sync"
09b9cbde9e7a0f9adea1fb054a7c62f35ad901e1Timo Sirainenstatic int mbox_write_pseudo(struct mbox_sync_context *sync_ctx)
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen uid_validity = sync_ctx->base_uid_validity != 0 ?
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen "From: Mail System Internal Data <MAILER-DAEMON@%s>\n"
f5b919e9b07dfd9d2401b998ef8759e5f0312719Timo Sirainen "Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA"
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "\nMessage-ID: <%s@%s>\n"
f5b919e9b07dfd9d2401b998ef8759e5f0312719Timo Sirainen "X-IMAP: %u %010u\n"
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen "Status: RO\n"
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen mbox_from_create("MAILER_DAEMON", ioloop_time),
a97a3ce5d82b16e8979de2d6cafbf6b0a129fe4eTimo Sirainen my_hostname, dec2str(ioloop_time), my_hostname,
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen "pwrite_full()");
8907d617ce7c4f390c0f42f6f694db2fecdd5775Timo Sirainen /* out of disk space, truncate to empty */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync_ctx->base_uid_last = sync_ctx->next_uid-1;
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainenstatic int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen uoff_t file_size, offset, padding, trailer_size;
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
988922bdf2d461021d210697f1f118956ca388e1Timo Sirainen st = i_stream_stat(sync_ctx->file_input, TRUE);
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
eb276c05bf6b0a383c772d61e31cf09a8dbd36c7Timo Sirainen if (file_size < sync_ctx->file_input->v_offset) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "file size unexpectedly shrinked in mbox file %s "
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen trailer_size = file_size - sync_ctx->file_input->v_offset;
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
eeeee5297c71eb8badfbdff5241ac05851f1c21fTimo Sirainen i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff);
eeeee5297c71eb8badfbdff5241ac05851f1c21fTimo Sirainen sync_ctx->space_diff += sync_ctx->expunged_space;
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen "file_set_size()");
bfa7ef85a07f07b7d2e72471a584689cfb6adc31Timo Sirainen if (ftruncate(sync_ctx->write_fd, file_size) < 0) {
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen "ftruncate()");
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size,
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen /* copy trailer, then truncate the file */
1a073dd6184645b026583274e05afba10dcc20bbTimo Sirainen if (file_size == (uoff_t)sync_ctx->expunged_space) {
1a073dd6184645b026583274e05afba10dcc20bbTimo Sirainen /* everything deleted, the trailer_size still contains
1a073dd6184645b026583274e05afba10dcc20bbTimo Sirainen the \n trailer though */
1a073dd6184645b026583274e05afba10dcc20bbTimo Sirainen i_assert(file_size >= sync_ctx->expunged_space + trailer_size);
1a073dd6184645b026583274e05afba10dcc20bbTimo Sirainen offset = file_size - sync_ctx->expunged_space - trailer_size;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainenstatic int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen st = i_stream_stat(sync_ctx->file_input, FALSE);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen ((uint64_t)st->st_size == sync_ctx->hdr->sync_size ||
01f4ee4a0243f3fe9af763e1a540cd5cff0d63f5Timo Sirainen (uint64_t)st->st_size == sync_ctx->orig_size)) {
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen /* We moved messages inside the mbox file without changing
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen the file's size. If mtime doesn't change, another process
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen not using the same index file as us can't know that the file
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen was changed. So make sure the mtime changes. This should
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen happen rarely enough that the sleeping doesn't become a
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen performance problem.
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen Note that to do this perfectly safe we should do this wait
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen whenever mails are moved or expunged, regardless of whether
bfa7ef85a07f07b7d2e72471a584689cfb6adc31Timo Sirainen the file's size changed. That however could become a
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen performance problem and the consequences of being wrong are
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen quite minimal (an extra logged error message). */
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen while (sync_ctx->orig_mtime == st->st_mtime) {
06fc140d5f0b03524e63a15d45d1cdc8b691372cTimo Sirainen st = i_stream_stat(sync_ctx->file_input, FALSE);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen "i_stream_stat()");
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen /* only reason not to have UID validity at this point is if the file
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen is entirely empty. In that case just make up a new one if needed. */
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen i_assert(sync_ctx->base_uid_validity != 0 || st->st_size == 0);
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity ||
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen (unsigned int)ioloop_time;
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen offsetof(struct mail_index_header, uid_validity),
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input) &&
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen &sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen if ((uint32_t)st->st_mtime != sync_ctx->hdr->sync_stamp &&
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen offsetof(struct mail_index_header, sync_stamp),
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen if ((uint64_t)st->st_size != sync_ctx->hdr->sync_size &&
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen offsetof(struct mail_index_header, sync_size),
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen sync_ctx->mbox->mbox_dirty_stamp = st->st_mtime;
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen sync_ctx->mbox->mbox_dirty_size = st->st_size;
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainenstatic void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen mail_index_sync_reset(sync_ctx->index_sync_ctx);
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainenstatic int mbox_sync_do(struct mbox_sync_context *sync_ctx,
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen unsigned int i;
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen st = i_stream_stat(sync_ctx->file_input, FALSE);
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* forcing a full sync. assume file has changed. */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen } else if ((uint32_t)st->st_mtime == sync_ctx->hdr->sync_stamp &&
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen (uint64_t)st->st_size == sync_ctx->hdr->sync_size) {
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen /* file is fully synced */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen } else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen (uint64_t)st->st_size == sync_ctx->hdr->sync_size) {
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen /* we want to do full syncing. always do this if
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen file size hasn't changed but timestamp has. it most
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen likely means that someone had modified some header
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen and we probably want to know about it */
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen /* see if we can delay syncing the whole file.
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen normally we only notice expunges and appends
42456ec33fe65feb411890f99d436071e0185ee3Timo Sirainen in partial syncing. */
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen for (i = 0; i < 3; i++) {
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* partial syncing didn't work, do it again. we get here
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen also if we ran out of UIDs. */
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
9638727fc341296ae5a7af630e176bc00f742f0bTimo Sirainen /* only syncs left should be just appends (and their updates)
4ca7ca056c5039424c66581582052881854794a3Timo Sirainen which weren't synced yet for some reason (crash). we'll just
4ca7ca056c5039424c66581582052881854794a3Timo Sirainen ignore them, as we've overwritten them above. */
4ca7ca056c5039424c66581582052881854794a3Timo Sirainen memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
81ff42abb5db580f0553c6a4744cdb305089e0bcTimo Sirainen if (mbox_sync_update_index_header(sync_ctx) < 0)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenint mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty)
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) {
d22301419109ed4a38351715e6760011421dadecTimo Sirainen /* read-only stream */
1f1ee8db68d9ae1604350801cd8dc33ebe29fe8aTimo Sirainen st = i_stream_stat(mbox->mbox_file_stream, FALSE);
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen mbox_set_syscall_error(mbox, "i_stream_stat()");
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen if ((uint32_t)st->st_mtime == hdr->sync_stamp &&
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen /* fully synced */
bfa7ef85a07f07b7d2e72471a584689cfb6adc31Timo Sirainen return st->st_mtime != mbox->mbox_dirty_stamp ||
bfa7ef85a07f07b7d2e72471a584689cfb6adc31Timo Sirainenstatic void mbox_sync_context_free(struct mbox_sync_context *sync_ctx)
b66d803de86bfb411165b3465b0d9ef64ecfe2a1Timo Sirainen mail_index_sync_rollback(&sync_ctx->index_sync_ctx);
32d9a75612a5df455e4169b56538bb31dfe359e4Timo Sirainenstatic int mbox_sync_int(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen unsigned int lock_id = 0;
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen bool leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen if ((changed = mbox_sync_has_changed(mbox, leave_dirty)) < 0) {
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen /* we just want to lock it for reading. if mbox hasn't been
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen modified don't do any syncing. */
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen /* have to sync to make sure offsets have stayed the same */
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen /* reopen input stream to make sure it has nothing buffered */
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen /* we're most likely modifying the mbox while syncing, just
648d24583c1574441c4fa0331a90bd4d6e7996c5Timo Sirainen lock it for writing immediately. the mbox must be locked
648d24583c1574441c4fa0331a90bd4d6e7996c5Timo Sirainen before index syncing is started to avoid deadlocks, so we
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen don't have much choice either (well, easy ones anyway). */
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen int lock_type = mbox->mbox_readonly ? F_RDLCK : F_WRLCK;
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainen if (mbox_lock(mbox, lock_type, &lock_id) <= 0)
c000c8eca8f24b2a0c76393ec4bbf76a505a4983Timo Sirainen sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
f140f88a5ab3e2194f214c11f9f418559e949c83Timo Sirainen sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainen ret = mail_index_sync_begin(mbox->ibox.index, &index_sync_ctx,
63a61b7a739ae0f3f520215137d9c50f94d0f34fTimo Sirainen if (!changed && !mail_index_sync_have_more(index_sync_ctx)) {
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainen /* nothing to do */
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen /* index may need to do internal syncing though, so commit
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen instead of rollbacking. */
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen if (mail_index_sync_commit(&index_sync_ctx) < 0) {
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen sync_ctx.hdr = mail_index_get_header(sync_view);
b66d803de86bfb411165b3465b0d9ef64ecfe2a1Timo Sirainen sync_ctx.from_line = str_new(default_pool, 256);
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen sync_ctx.header = str_new(default_pool, 4096);
f2b79667fc7a8f7c2c72cad18bd71d49730e36f6Timo Sirainen pool_alloconly_create("mbox saved keywords", 4096);
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen /* make sure we've read the latest keywords in index */
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen (void)mail_index_get_keywords(mbox->ibox.index);
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen sync_ctx.delay_writes = delay_writes || sync_ctx.mbox->mbox_readonly;
30975737820f5855e2c26d81b574ae5f03a05407Timo Sirainen /* if we have only flag changes, we don't need to open the
c000c8eca8f24b2a0c76393ec4bbf76a505a4983Timo Sirainen if (mbox_sync_read_index_syncs(&sync_ctx, 1, &expunged) < 0)
d22301419109ed4a38351715e6760011421dadecTimo Sirainen /* ok, we have something to do but no locks. we'll have to
fa3be6b0d9d9fc0264a8434e17e26ad26c739daeTimo Sirainen restart syncing to avoid deadlocking. */
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream;
88553367d677170a4b703b9d52aac9eabf91c656Timo Sirainen sync_ctx.write_fd = sync_ctx.mbox->mbox_lock_type != F_WRLCK ? -1 :
4d10cf8c7879ccd377e7fb136913b2a258ba8d93Timo Sirainen else if (mail_index_sync_commit(&index_sync_ctx) < 0) {
b20fb5b1df9d604a7541f5118fc5b4b466d211efTimo Sirainen if (sync_ctx.base_uid_last != sync_ctx.next_uid-1 &&
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* Rewrite uid_last in X-IMAPbase header if we've seen it
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen (ie. the file isn't empty) */
76e3062a60f585a638e0933bb71d3c8c9b0d8e2aTimo Sirainen if (ret == 0 && mbox->mbox_fd != -1 && mbox->ibox.keep_recent) {
343a527f805ca5cce78496b959d6def70e5d0cd4Timo Sirainen /* try to set atime back to its original value */
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainen /* drop to read lock */
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainen unsigned int read_lock_id = 0;
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0)
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainenint mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
d6e2b2761c8e0b6923c883fb2ead2665ee954be5Timo Sirainen mbox->ibox.box.v.sync_notify(&mbox->ibox.box, 0, 0);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
aa0647f2debf0d48d504a321186f66c85596aaf4Timo Sirainen struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
f140f88a5ab3e2194f214c11f9f418559e949c83Timo Sirainen mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
205debb6e5882687f43a98fec7bb577173b2fe28Timo Sirainen if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
9d0e1fa945103f2875cdf6d18b2013809f566ea7Timo Sirainen if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0)