mbox-sync.c revision e59faf65ce864fe95dc00f5d52b8323cdbd0608a
e59faf65ce864fe95dc00f5d52b8323cdbd0608aTimo Sirainen/* Copyright (c) 2004-2010 Dovecot authors, see the included COPYING file */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen required disk I/O. We may need to:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing X-UID and X-IMAPbase headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing or broken Content-Length header if there's space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Expunge specified messages
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Here's how we do it:
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Start reading the mails from the beginning
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen of them, remember how much each message has and offset to beginning of the
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If header needs to be rewritten and there's enough space, do it
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If we didn't have enough space, remember how much was missing
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Continue reading and counting the padding in each message. If available
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen padding is enough to rewrite all the previous messages needing it, do it
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - When we encounter expunged message, treat all of it as padding and
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen rewrite previous messages if needed (and there's enough space).
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen Afterwards keep moving messages backwards to fill the expunged space.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen Moving is done by rewriting each message's headers, with possibly adding
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen missing Content-Length header and padding. Message bodies are moved
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen without modifications.
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - If we encounter end of file, grow the file and rewrite needed messages
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen - Rewriting is done by moving message body forward, rewriting message's
f16c114c20bbd7d292d93415d1e56c8dd6abd3e7Timo Sirainen header and doing the same for previous message, until all of them are
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen/* The text below was taken exactly as c-client wrote it to my mailbox,
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen so it's probably copyrighted by University of Washington. */
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"This text is part of the internal format of your mail folder, and is not\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"a real message. It is created automatically by the mail system software.\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"If deleted, important folder data will be lost, and it will be re-created\n" \
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen"with the data reset to initial values.\n"
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainenvoid mbox_sync_set_critical(struct mbox_sync_context *sync_ctx,
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen const char *fmt, ...)
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen "mbox file %s was modified while we were syncing, "
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "check your locking settings",
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "Sync failed for mbox file %s: %s",
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T,
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainenvoid mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx)
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen /* Do this even if ext_modified is already set. Expunging code relies
a2637488c8d514ec1ac3914811deee814f9761b3Timo Sirainen on last_stat being updated. */
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen if (st.st_size != sync_ctx->last_stat.st_size ||
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainenvoid mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty)
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen /* just mark the stat as dirty. */
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0)
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen /* get EOF */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
6f25019b337e27600159b596824da08732965576Timo Sirainen if (istream_raw_mbox_is_corrupted(sync_ctx->input))
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
1e76a5b92f9d82d557f81f080f3dfad1c9d8f200Timo Sirainen if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 &&
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen /* need to add 'O' flag to Status-header */
b08b33d1f5ce3721dc2d83586c9cb0ca141331fdTimo Sirainenstatic void mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen uint8_t expunged_guid_128[MAIL_GUID_128_SIZE];
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen /* nothing for this or the future ones */
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen index_sync_changes_read(sync_ctx->sync_changes, uid, sync_expunge_r,
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen /* we can't expunge anything from read-only mboxes */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* externally expunged message, remove from index */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen if (rec == NULL && uid < sync_ctx->idx_next_uid) {
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen /* this UID was already in index and it was expunged */
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "Expunged message reappeared to mailbox "
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen "(UID %u < %u, seq=%u, idx_msgs=%u)",
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "UID inserted in the middle of mailbox "
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "(%u > %u, seq=%u, idx_msgs=%u)",
2bf7bb14894faf721518e2122a14a2389ef94078Timo Sirainen rec->uid, uid, sync_ctx->seq, messages_count);
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainenstatic void mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen unsigned char hdr_md5_sum[],
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
f0f9c8e94abac18f8acd91b9e724c4c32863723aTimo Sirainen if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0)
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen /* externally expunged message, remove from index */
7f3be7d885c75cdd77f536929a45bc9764595960Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen /* see if from_offset needs updating */
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen *((const uint64_t *)data) == mail->from_offset)
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainenmbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen keywords = !array_is_created(&mail_ctx->mail.keywords) ?
61f5256ef248d35459b53534ae428bf6d016e1c5Timo Sirainen mail_index_keywords_create(sync_ctx->mbox->ibox.index, NULL) :
61f5256ef248d35459b53534ae428bf6d016e1c5Timo Sirainen sync_ctx->mbox->ibox.index, &mail_ctx->mail.keywords);
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainenmbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx)
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
ea5f188fc29dfaa0c4071e6413e16e1d04263722Timo Sirainen sync_ctx->mbox->md5hdr_ext_idx, &ext_data, NULL);
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) {
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainenstatic void mbox_sync_get_dirty_flags(struct mbox_sync_mail_context *mail_ctx,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* default to undirtying the message. it gets added back if
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen flags/keywords don't match what is in the index. */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_ctx->mail.flags &= ~MAIL_INDEX_MAIL_FLAG_DIRTY;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* replace flags */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen idx_flags = rec->flags & MAIL_FLAGS_NONRECENT;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mbox_flags = mail_ctx->mail.flags & MAIL_FLAGS_NONRECENT;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_ctx->mail.flags = (mail_ctx->mail.flags & MAIL_RECENT) |
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* replace keywords */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_index_lookup_keywords(sync_ctx->sync_view, sync_ctx->idx_seq,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen if (!index_keyword_array_cmp(&idx_keywords, &mail_ctx->mail.keywords)) {
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_ctx->mail.flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen if (!array_is_created(&mail_ctx->mail.keywords)) {
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen array_append_array(&mail_ctx->mail.keywords, &idx_keywords);
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainenstatic void mbox_sync_update_flags(struct mbox_sync_mail_context *mail_ctx,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen struct mailbox *box = &sync_ctx->mbox->ibox.box;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen ARRAY_TYPE(keyword_indexes) orig_keywords = ARRAY_INIT;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* flags and keywords are dirty. replace the current
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen ones from the flags in index file. */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen flags = orig_flags = mail->flags & MAIL_FLAGS_NONRECENT;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen array_append_array(&orig_keywords, &mail->keywords);
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* apply new changes */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen index_sync_changes_apply(sync_ctx->sync_changes,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen !index_keyword_array_cmp(&mail->keywords, &orig_keywords)) {
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail->flags = flags | (mail->flags & MAIL_RECENT) |
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen if (sync_type != 0 && box->v.sync_notify != NULL) {
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainenstatic void mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* changes are written to the mbox file */
5d98a279f2726a8c38ef1e8e3c54d6c62d523a65Timo Sirainen /* make sure this message gets written later */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* new message */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen /* flags other than recent/dirty have changed */
3f26c5aced2e71efc783f26bb8a7ac53f7504622Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* only dirty flag state changed */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen /* see if keywords changed */
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen mail_index_lookup_keywords(sync_ctx->sync_view,
1cd97699af9c77d8f5920832ec3374884544fd68Timo Sirainen if (!index_keyword_array_cmp(&idx_keywords, &mail->keywords))
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen /* see if we need to update md5 sum. */
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen /* Mail has "Status: O" header. No messages before this
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen can be recent. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen rewriting would just move it anyway. */
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen bool nocheck = rec == NULL || sync_ctx->expunged_space > 0;
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen mbox_sync_update_from_offset(sync_ctx, mail, nocheck);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen struct istream *input = ctx->sync_ctx->file_input;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen const unsigned char *data;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenstatic int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen const char *str;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen unsigned int i;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen i_assert(sync_ctx->base_uid_last_offset != 0);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* first check that the 10 bytes are there and they're exactly as
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen expected. just an extra safety check to make sure we never write
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen to wrong location in the mbox file. */
16aba431c576c1dbd99cbaae4f9d65eea9ad73c2Timo Sirainen ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf),
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pread_full()");
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "X-IMAPbase uid-last offset unexpectedly outside mbox");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen for (i = 0, uid_last = 0; i < sizeof(buf); i++) {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "X-IMAPbase uid-last unexpectedly lost");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* and write it */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen str = t_strdup_printf("%010u", sync_ctx->next_uid - 1);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen sync_ctx->base_uid_last = sync_ctx->next_uid - 1;
064bfeee2f9156683b191cc0f3f7b242720942f7Timo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx)
b225c3c65f360d7b833f09f9b2fb3035ed5ea600Timo Sirainen if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str),
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()");
8907d617ce7c4f390c0f42f6f694db2fecdd5775Timo Sirainenstatic void update_from_offsets(struct mbox_sync_context *sync_ctx)
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen unsigned int i, count;
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen for (i = 0; i < count; i++) {
1e76a5b92f9d82d557f81f080f3dfad1c9d8f200Timo Sirainen if (mails[i].idx_seq == 0 || mails[i].expunged)
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen mail_index_update_ext(sync_ctx->t, mails[i].idx_seq,
8907d617ce7c4f390c0f42f6f694db2fecdd5775Timo Sirainenstatic void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
c35d0c3eb4ba89432896e1f9770ab31cc63c232bTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen struct mailbox *box = &sync_ctx->mbox->ibox.box;
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq);
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen mail_ctx->mail.offset = mail_ctx->mail.from_offset;
4d25408732be27e91f0430f71e87242760c2517cTimo Sirainen mail_ctx->body_offset - mail_ctx->mail.from_offset +
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* expunging first message, fix space to contain next
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen message's \n header too since it will be removed. */
c35d0c3eb4ba89432896e1f9770ab31cc63c232bTimo Sirainen if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) {
2be66b9eddad3841a1195fe9aeb1eaf0f28f1116Timo Sirainen /* uid-last offset is invalid now */
c35d0c3eb4ba89432896e1f9770ab31cc63c232bTimo Sirainen sync_ctx->expunged_space += mail_ctx->mail.space;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* move the header backwards to fill expunged space */
c35d0c3eb4ba89432896e1f9770ab31cc63c232bTimo Sirainen orig_from_offset = mail_ctx->mail.from_offset;
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen /* we're moving this mail to beginning of file.
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen skip the initial \n (it's already counted in
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen expunged_space) */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* read the From-line before rewriting overwrites it */
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* rewrite successful, write From-line to
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen new location */
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen /* didn't have enough space, move the offset
7fd72a47d7ddfbd38c8697e228b6951f495dfb61Timo Sirainen back so seeking into it doesn't fail */
c35d0c3eb4ba89432896e1f9770ab31cc63c232bTimo Sirainen mail_ctx->mail.from_offset = orig_from_offset;
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen /* mark it dirty and do it later */
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* nothing to do */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* first mail with no space to write it */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* create dummy message to describe the expunged data */
dc8552739fa29f011ab71ec383ec6d580a5a9661Timo Sirainen sync_ctx->space_diff = sync_ctx->expunged_space;
8d5991f5c4a8840bf1ea754093dbec505564ab78Timo Sirainen i_assert(sync_ctx->space_diff < -mail_ctx->mail.space);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen uoff_t end_offset, move_diff, extra_space, needed_space;
2cb565cd978aafd5714792b5161889986d49e431Timo Sirainen i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 ||
2cb565cd978aafd5714792b5161889986d49e431Timo Sirainen mail_ctx->mail.offset == mail_ctx->hdr_offset);
5486e1e123484f3a8d7d895e8ac41156f2a96305Timo Sirainen if (array_is_created(&mail_ctx->mail.keywords)) {
5486e1e123484f3a8d7d895e8ac41156f2a96305Timo Sirainen /* mail's keywords are allocated from a pool that's cleared
5486e1e123484f3a8d7d895e8ac41156f2a96305Timo Sirainen for each mail. we'll need to copy it to something more
5486e1e123484f3a8d7d895e8ac41156f2a96305Timo Sirainen permanent. */
f5b4979e2780c4df112a300967d647e2fdd73511Timo Sirainen p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool,
5486e1e123484f3a8d7d895e8ac41156f2a96305Timo Sirainen array_append_array(&keywords_copy, &mail_ctx->mail.keywords);
6e4cd4ba520bc22ce375de378f4751136ebcf75aTimo Sirainen array_append(&sync_ctx->mails, &mail_ctx->mail, 1);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* we have enough space now */
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen /* this message was expunged. fill more or less of the space.
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen space_diff now consists of a negative "bytes needed" sum,
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen plus the expunged space of this message. so it contains how
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen many bytes of _extra_ space we have. */
8d5991f5c4a8840bf1ea754093dbec505564ab78Timo Sirainen i_assert(mail_ctx->mail.space >= sync_ctx->space_diff);
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
4214b59ac7f3899f8d887d055ef519f5a622d249Timo Sirainen if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen /* don't waste too much on padding */
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen /* this message gave enough space from headers. rewriting stops
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen at the end of this message's headers. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* mail_ctx may contain wrong data after rewrite, so make sure we
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen don't try to access it */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainenmbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) {
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen mail_storage_set_error(&mbox->storage->storage,
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen "Mailbox isn't a valid mbox file");
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "Message was expunged unexpectedly");
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen "Error seeking back to original "
88187ee880b4829443e0d55ea7d145d9d5880217Timo Sirainen mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid);
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen /* set to -1, since it's always increased later */
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
cece2b9cd692c06025cc0a7a0ff54d996a8c90efTimo Sirainen /* this mbox has pseudo mail which contains the X-IMAP header */
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen sync_ctx->dest_first_mail = sync_ctx->seq == 0;
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainenmbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
a10e5606a9e93f49cf13b3a35c8dc3f5d6ab5909Timo Sirainen struct mail_index_view *sync_view = sync_ctx->sync_view;
3da614c39dd29f536c485089e67839b4cf89fed3Timo Sirainen if (!mail_index_lookup_seq_range(sync_view, uid, (uint32_t)-1,
0a6f8311541ae59381171620b77f82be58be562eTimo Sirainen /* doesn't exist anymore, seek to end of file */
4b9f99761df5014c659cd87fddaf6854af428cfcTimo Sirainen ret = i_stream_get_size(sync_ctx->file_input, TRUE, &size);
4b9f99761df5014c659cd87fddaf6854af428cfcTimo Sirainen "i_stream_get_size()");
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "Error seeking to end of mbox");
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen mail_index_view_get_messages_count(sync_view) + 1;
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenstatic int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* delete sync records up to next message. so if there's still
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen something left in array, it means the next message needs modifying */
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen index_sync_changes_delete_to(sync_ctx->sync_changes, next_uid);
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen if (index_sync_changes_have(sync_ctx->sync_changes))
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen if (sync_ctx->hdr->first_recent_uid <= next_uid &&
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen /* we'll need to rewrite Status: O headers */
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen uid = index_sync_changes_get_next_uid(sync_ctx->sync_changes);
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen if (sync_ctx->hdr->first_recent_uid < sync_ctx->hdr->next_uid &&
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen (uid > sync_ctx->hdr->first_recent_uid || uid == 0) &&
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen /* we'll need to rewrite Status: O headers */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* we can skip forward to next record which needs updating. */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen ret = mbox_sync_seek_to_uid(sync_ctx, next_uid);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* if there's no sync records left, we can stop. except if
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen this is a dirty sync, check if there are new messages. */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen ret = mbox_sync_seek_to_seq(sync_ctx, messages_count);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* seek failed because the offset is dirty. just ignore and
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen continue from where we are now. */
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainenstatic void mbox_sync_hdr_update(struct mbox_sync_context *sync_ctx,
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update;
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen sync_ctx->base_uid_validity = update->uid_validity;
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen sync_ctx->base_uid_last <= update->min_next_uid) {
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen sync_ctx->base_uid_last = update->min_next_uid-1;
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainenstatic bool mbox_sync_imapbase(struct mbox_sync_context *sync_ctx,
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
f6699a08521aacc4c2bb5b6175691dad5f715f8cTimo Sirainen i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s",
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainenstatic int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* always start from first message so we can read X-IMAP or
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen X-IMAPbase header */
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen /* expunge everything */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen /* UID ordering problems, resync everything to make
319944c0f35b311c998854e96d6463a084fd90aeTimo Sirainen sure we get everything right */
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "UIDs broken with partial sync");
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen if (!mbox_sync_read_index_rec(sync_ctx, uid, &rec))
2ae575a66f2a302f047f6de062a70b75f8bebc7bTimo Sirainen /* UID found but it's broken */
cdaf255d6a3daeef0ac85edaa60bfa6d1f945bffTimo Sirainen } else if (uid == 0 &&
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen /* If we can't use/store X-UID header, use MD5 sum.
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen Also check for existing MD5 sums when we're actually
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen able to write X-UIDs. */
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen /* get all sync records related to this message. with pseudo
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen message just get the first sync record so we can jump to
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen it with partial seeking. */
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen /* if it was set, it was for the next message */
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen /* message wasn't found from index. we have to
c9fe52d819c608b890620f7fe36ff509b14eb350Timo Sirainen read everything from now on, no skipping */
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen /* missing/broken X-UID. all the rest of the mails
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen need new UIDs. */
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen /* oh no, we're out of UIDs. this shouldn't
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen happen normally, so just try to get it fixed
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen without crashing. */
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen "Out of UIDs, renumbering them in mbox "
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_handle_missing_space(mail_ctx) < 0)
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* move the body */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen } else if (partial) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1,
178511b57faa7c3f8203dd8b7e4059d00cbfc23aTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input)) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* rest of the messages in index don't exist -> expunge them */
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
d31c4d7c161f9d7efa59964c7c958e83e05b218cTimo Sirainen /* once we get around to writing the changes, we'll need to do
d31c4d7c161f9d7efa59964c7c958e83e05b218cTimo Sirainen a full sync to avoid the "UIDs broken in partial sync"
f7d43647acc6dc80064c8c4cacf5bf86f754c530Timo Sirainenstatic int mbox_write_pseudo(struct mbox_sync_context *sync_ctx)
eeeee5297c71eb8badfbdff5241ac05851f1c21fTimo Sirainen if (sync_ctx->mbox->sync_hdr_update != NULL) {
eeeee5297c71eb8badfbdff5241ac05851f1c21fTimo Sirainen sync_ctx->base_uid_validity = update->uid_validity;
eeeee5297c71eb8badfbdff5241ac05851f1c21fTimo Sirainen sync_ctx->base_uid_last = update->min_next_uid-1;
9d2040fbb941f411d57fd850b4cdc3b1cccc1168Timo Sirainen uid_validity = sync_ctx->base_uid_validity != 0 ?
9d2040fbb941f411d57fd850b4cdc3b1cccc1168Timo Sirainen sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity;
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen "From: Mail System Internal Data <MAILER-DAEMON@%s>\n"
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen "Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA"
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen "\nMessage-ID: <%s@%s>\n"
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen "X-IMAP: %u %010u\n"
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen "Status: RO\n"
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen mbox_from_create("MAILER_DAEMON", ioloop_time),
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen my_hostname, dec2str(ioloop_time), my_hostname,
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen "pwrite_full()");
98c1cf256927e254f0c092acd2ddcd7ea50bd009Timo Sirainen /* out of disk space, truncate to empty */
b8bbfab97eed17fcb00b5a86128e1d7a3babc35cTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
2be66b9eddad3841a1195fe9aeb1eaf0f28f1116Timo Sirainen sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */
2be66b9eddad3841a1195fe9aeb1eaf0f28f1116Timo Sirainen sync_ctx->base_uid_last = sync_ctx->next_uid-1;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
f5b919e9b07dfd9d2401b998ef8759e5f0312719Timo Sirainen uoff_t file_size, offset, padding, trailer_size;
178511b57faa7c3f8203dd8b7e4059d00cbfc23aTimo Sirainen if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
4b9f99761df5014c659cd87fddaf6854af428cfcTimo Sirainen ret = i_stream_get_size(sync_ctx->file_input, TRUE, &file_size);
4b9f99761df5014c659cd87fddaf6854af428cfcTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_get_size()");
8166e8a706078efc71923719ca285e84902883c0Timo Sirainen /* Not a file - allow anyway */
09b9cbde9e7a0f9adea1fb054a7c62f35ad901e1Timo Sirainen if (file_size < sync_ctx->file_input->v_offset) {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "file size unexpectedly shrank "
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "(%"PRIuUOFF_T" vs %"PRIuUOFF_T")", file_size,
5edfc0f1c3c55e906d8316d9cdeaa3b0c7000c19Timo Sirainen trailer_size = file_size - sync_ctx->file_input->v_offset;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
f5b919e9b07dfd9d2401b998ef8759e5f0312719Timo Sirainen i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->space_diff += sync_ctx->expunged_space;
6a1e4eb2c6a267bec1e8704ce9137bebb7792702Timo Sirainen "file_set_size()");
b8bbfab97eed17fcb00b5a86128e1d7a3babc35cTimo Sirainen if (ftruncate(sync_ctx->write_fd, file_size) < 0) {
b8bbfab97eed17fcb00b5a86128e1d7a3babc35cTimo Sirainen "ftruncate()");
f4b93a46e140823a64d88763ea6ef9f03c49844eTimo Sirainen if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size,
299183fbb6ec5d0828a0880da372540421ac4665Timo Sirainen /* copy trailer, then truncate the file */
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen if (file_size == (uoff_t)sync_ctx->expunged_space) {
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen /* everything deleted, the trailer_size still contains
b397a802ec245a9169dab6b62efa4f7f877c07f6Timo Sirainen the \n trailer though */
988922bdf2d461021d210697f1f118956ca388e1Timo Sirainen } else if (sync_ctx->expunged_space == (off_t)file_size + 1 ||
988922bdf2d461021d210697f1f118956ca388e1Timo Sirainen sync_ctx->expunged_space == (off_t)file_size + 2) {
988922bdf2d461021d210697f1f118956ca388e1Timo Sirainen /* everything deleted and we didn't have a proper
5edfc0f1c3c55e906d8316d9cdeaa3b0c7000c19Timo Sirainen i_assert(file_size >= sync_ctx->expunged_space + trailer_size);
5edfc0f1c3c55e906d8316d9cdeaa3b0c7000c19Timo Sirainen offset = file_size - sync_ctx->expunged_space - trailer_size;
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
eeeee5297c71eb8badfbdff5241ac05851f1c21fTimo Sirainen if (file_size == 0 && sync_ctx->mbox->sync_hdr_update != NULL) {
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainenmbox_sync_index_update_ext_header(struct mbox_sync_context *sync_ctx)
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update;
0c3cde7382518b486092bfae64bb6a948405854aTimo Sirainen if (update != NULL && !mail_guid_128_is_empty(update->mailbox_guid)) {
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen memcpy(mbox->mbox_hdr.mailbox_guid, update->mailbox_guid,
0c3cde7382518b486092bfae64bb6a948405854aTimo Sirainen } else if (mail_guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
c0bfb67ba32064347bac3241f1aac9b8a809e2f1Timo Sirainen mail_generate_guid_128(mbox->mbox_hdr.mailbox_guid);
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen mail_index_get_header_ext(mbox->ibox.view, mbox->mbox_ext_idx,
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen memcmp(data, &mbox->mbox_hdr, data_size) != 0) {
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen mail_index_update_header_ext(sync_ctx->t, mbox->mbox_ext_idx,
af6d4a24cb6d18e50d172540cf49b1448a6f9872Timo Sirainenstatic int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen st = i_stream_stat(sync_ctx->file_input, FALSE);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen ((uint64_t)st->st_size == sync_ctx->mbox->mbox_hdr.sync_size ||
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen (uint64_t)st->st_size == sync_ctx->orig_size)) {
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen /* We moved messages inside the mbox file without changing
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen the file's size. If mtime doesn't change, another process
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen not using the same index file as us can't know that the file
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen was changed. So make sure the mtime changes. This should
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen happen rarely enough that the sleeping doesn't become a
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen performance problem.
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen Note that to do this perfectly safe we should do this wait
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen whenever mails are moved or expunged, regardless of whether
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen the file's size changed. That however could become a
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen performance problem and the consequences of being wrong are
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen quite minimal (an extra logged error message). */
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen while (sync_ctx->orig_mtime == st->st_mtime) {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen if (utime(sync_ctx->mbox->ibox.box.path, NULL) < 0) {
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen st = i_stream_stat(sync_ctx->file_input, FALSE);
e05ea8311ae16687295048e88ca205dfe29fbcbfTimo Sirainen "i_stream_stat()");
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen sync_ctx->mbox->mbox_hdr.sync_mtime = st->st_mtime;
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen sync_ctx->mbox->mbox_hdr.sync_size = st->st_size;
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen /* only reason not to have UID validity at this point is if the file
dc42ce2d44e84d9d05a9310c11f8764f319eb3abTimo Sirainen is entirely empty. In that case just make up a new one if needed. */
8166e8a706078efc71923719ca285e84902883c0Timo Sirainen i_assert(sync_ctx->base_uid_validity != 0 || st->st_size <= 0);
06fc140d5f0b03524e63a15d45d1cdc8b691372cTimo Sirainen sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ?
06fc140d5f0b03524e63a15d45d1cdc8b691372cTimo Sirainen if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen offsetof(struct mail_index_header, uid_validity),
178511b57faa7c3f8203dd8b7e4059d00cbfc23aTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input) &&
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
aa38d1a0945f0bc13a225d043f53fad2eec666b1Timo Sirainen &sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen if (sync_ctx->last_nonrecent_uid < sync_ctx->hdr->first_recent_uid) {
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen /* other sessions have already marked more messages as
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen /* mark recent messages */
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen view = mail_index_transaction_open_updated_view(sync_ctx->t);
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen if (mail_index_lookup_seq_range(view, sync_ctx->last_nonrecent_uid + 1,
279cc94ab086f6a3cb764b1b98ff6b936efa3eaeTimo Sirainen index_mailbox_set_recent_seq(&sync_ctx->mbox->ibox,
ed14c2db5436d6d89aa65bc8e09af0c316359553Timo Sirainen sync_ctx->next_uid : sync_ctx->last_nonrecent_uid + 1;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen if (sync_ctx->hdr->first_recent_uid < first_recent_uid) {
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen offsetof(struct mail_index_header, first_recent_uid),
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen &first_recent_uid, sizeof(first_recent_uid), FALSE);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen index_sync_changes_reset(sync_ctx->sync_changes);
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen mail_index_sync_reset(sync_ctx->index_sync_ctx);
28b30e9bd1a0a8b9019124abd821cb2989a03910Timo Sirainen index_mailbox_reset_uidvalidity(&sync_ctx->mbox->ibox);
aa38d1a0945f0bc13a225d043f53fad2eec666b1Timo Sirainen sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainenstatic int mbox_sync_do(struct mbox_sync_context *sync_ctx,
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen struct mbox_index_header *mbox_hdr = &sync_ctx->mbox->mbox_hdr;
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen unsigned int i;
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen st = i_stream_stat(sync_ctx->file_input, FALSE);
34d2ee1fa2b299267fcefd378f80690e7f601dfbTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainen /* forcing a full sync. assume file has changed. */
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen } else if ((uint32_t)st->st_mtime == mbox_hdr->sync_mtime &&
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen (uint64_t)st->st_size == mbox_hdr->sync_size) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* file is fully synced */
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen if (mbox_hdr->dirty_flag && (flags & MBOX_SYNC_UNDIRTY) != 0)
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen } else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen (uint64_t)st->st_size == mbox_hdr->sync_size) {
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* we want to do full syncing. always do this if
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen file size hasn't changed but timestamp has. it most
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen likely means that someone had modified some header
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen and we probably want to know about it */
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen /* see if we can delay syncing the whole file.
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen normally we only notice expunges and appends
84e1634acc701d14e358e27f1beff5ad74f5004aTimo Sirainen in partial syncing. */
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen for (i = 0;;) {
a486ed03dce069ff60ab5a65d0ae24a1862f22fcTimo Sirainen ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial);
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen /* a) partial sync didn't work
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen b) we ran out of UIDs
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen c) syncing had errors */
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen (sync_ctx->errors || sync_ctx->renumber_uids)) {
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen /* fixing a broken mbox state, be sure to write
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen the changes. */
b6d817f0effeff645aadc01fd468a7d4084ba1f2Timo Sirainen if (++i == 3)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen /* only syncs left should be just appends (and their updates)
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen which weren't synced yet for some reason (crash). we'll just
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen ignore them, as we've overwritten them above. */
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen index_sync_changes_reset(sync_ctx->sync_changes);
4ca7ca056c5039424c66581582052881854794a3Timo Sirainen if (sync_ctx->base_uid_last != sync_ctx->next_uid-1 &&
4ca7ca056c5039424c66581582052881854794a3Timo Sirainen /* Rewrite uid_last in X-IMAPbase header if we've seen it
4ca7ca056c5039424c66581582052881854794a3Timo Sirainen (ie. the file isn't empty) */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_update_index_header(sync_ctx) < 0)
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainenint mbox_sync_header_refresh(struct mbox_mailbox *mbox)
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen if (mail_index_refresh(mbox->ibox.index) < 0) {
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen mail_index_get_header_ext(mbox->ibox.view, mbox->mbox_ext_idx,
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen /* doesn't exist. FIXME: backwards compatibility copying */
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen memcpy(&mbox->mbox_hdr, data, I_MIN(sizeof(mbox->mbox_hdr), data_size));
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenint mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty)
32d9a75612a5df455e4169b56538bb31dfe359e4Timo Sirainen return mbox_sync_has_changed_full(mbox, leave_dirty, &empty);
32d9a75612a5df455e4169b56538bb31dfe359e4Timo Sirainenint mbox_sync_has_changed_full(struct mbox_mailbox *mbox, bool leave_dirty,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) {
04ab375449dd97eed50ada88dd0df2abab01cfeeTimo Sirainen /* read-only stream */
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen st = i_stream_stat(mbox->mbox_file_stream, FALSE);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen mbox_set_syscall_error(mbox, "i_stream_stat()");
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen if (stat(mbox->ibox.box.path, &statbuf) < 0) {
0c3cde7382518b486092bfae64bb6a948405854aTimo Sirainen if (mail_guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
c0bfb67ba32064347bac3241f1aac9b8a809e2f1Timo Sirainen /* need to assign mailbox GUID */
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen if ((uint32_t)st->st_mtime == mbox->mbox_hdr.sync_mtime &&
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen (uint64_t)st->st_size == mbox->mbox_hdr.sync_size) {
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* fully synced */
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen if (!mbox->mbox_hdr.dirty_flag || leave_dirty)
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen /* flushing dirtyness */
690bafa70767e3f6e98bbfd62ad4a26be2387ea9Timo Sirainen /* file changed */
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainenstatic void mbox_sync_context_free(struct mbox_sync_context *sync_ctx)
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen index_sync_changes_deinit(&sync_ctx->sync_changes);
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen mail_index_sync_rollback(&sync_ctx->index_sync_ctx);
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainenstatic int mbox_sync_int(struct mbox_mailbox *mbox, enum mbox_sync_flags flags,
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen unsigned int *lock_id)
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen bool leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen if ((changed = mbox_sync_has_changed(mbox, leave_dirty)) < 0)
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen /* we just want to lock it for reading. if mbox hasn't been
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen modified don't do any syncing. */
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen /* have to sync to make sure offsets have stayed the same */
5b8ae5496ae0b23e84f620bb0f85d7e87a5e11fdTimo Sirainen /* flush input streams' buffers */
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen /* we're most likely modifying the mbox while syncing, just
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen lock it for writing immediately. the mbox must be locked
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen before index syncing is started to avoid deadlocks, so we
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen don't have much choice either (well, easy ones anyway). */
42456ec33fe65feb411890f99d436071e0185ee3Timo Sirainen int lock_type = mbox->ibox.backend_readonly ? F_RDLCK : F_WRLCK;
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen if ((ret = mbox_lock(mbox, lock_type, lock_id)) <= 0) {
30975737820f5855e2c26d81b574ae5f03a05407Timo Sirainen /* try as read-only */
ced118ac5caf6fe83d34339c2c65c63b2aa768acTimo Sirainen if ((mbox->ibox.box.flags & MAILBOX_FLAG_KEEP_RECENT) == 0)
2d39dc1a453546892109b35c0d9770369011a13dTimo Sirainen sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
2d39dc1a453546892109b35c0d9770369011a13dTimo Sirainen sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
fa3be6b0d9d9fc0264a8434e17e26ad26c739daeTimo Sirainen ret = mail_index_sync_begin(mbox->ibox.index, &index_sync_ctx,
ced118ac5caf6fe83d34339c2c65c63b2aa768acTimo Sirainen if ((mbox->ibox.box.flags & MAILBOX_FLAG_KEEP_RECENT) == 0) {
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen /* see if we need to drop recent flags */
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen sync_ctx.hdr = mail_index_get_header(sync_view);
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen if (sync_ctx.hdr->first_recent_uid < sync_ctx.hdr->next_uid)
88553367d677170a4b703b9d52aac9eabf91c656Timo Sirainen if (!changed && !mail_index_sync_have_more(index_sync_ctx)) {
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen /* nothing to do */
4d10cf8c7879ccd377e7fb136913b2a258ba8d93Timo Sirainen /* index may need to do internal syncing though, so commit
4d10cf8c7879ccd377e7fb136913b2a258ba8d93Timo Sirainen instead of rollbacking. */
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen if (mail_index_sync_commit(&index_sync_ctx) < 0) {
ced118ac5caf6fe83d34339c2c65c63b2aa768acTimo Sirainen (mbox->ibox.box.flags & MAILBOX_FLAG_KEEP_RECENT) != 0;
b20fb5b1df9d604a7541f5118fc5b4b466d211efTimo Sirainen sync_ctx.hdr = mail_index_get_header(sync_view);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx.from_line = str_new(default_pool, 256);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx.header = str_new(default_pool, 4096);
343a527f805ca5cce78496b959d6def70e5d0cd4Timo Sirainen pool_alloconly_create("mbox saved keywords", 4096);
bb10ebcf076c959c752f583746d83805d7686df8Timo Sirainen /* make sure we've read the latest keywords in index */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen (void)mail_index_get_keywords(mbox->ibox.index);
022412398e56a8f31ef111cfd7271498d64af9a9Timo Sirainen index_sync_changes_init(&mbox->ibox, index_sync_ctx,
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainen /* if we have only flag changes, we don't need to open the
b08b33d1f5ce3721dc2d83586c9cb0ca141331fdTimo Sirainen mbox_sync_read_index_syncs(&sync_ctx, 1, &expunged);
30975737820f5855e2c26d81b574ae5f03a05407Timo Sirainen index_sync_changes_get_next_uid(sync_ctx.sync_changes);
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainen /* ok, we have something to do but no locks. we'll have to
ccb70ccfd9a25e490aab46d15d9b8323ad9ea3bfTimo Sirainen restart syncing to avoid deadlocking. */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream;
d6e2b2761c8e0b6923c883fb2ead2665ee954be5Timo Sirainen sync_ctx.write_fd = sync_ctx.mbox->mbox_lock_type != F_WRLCK ? -1 :
aa0647f2debf0d48d504a321186f66c85596aaf4Timo Sirainen else if (mail_index_sync_commit(&index_sync_ctx) < 0) {
ced118ac5caf6fe83d34339c2c65c63b2aa768acTimo Sirainen if (ret == 0 && mbox->mbox_fd != -1 && sync_ctx.keep_recent &&
9d0e1fa945103f2875cdf6d18b2013809f566ea7Timo Sirainen /* try to set atime back to its original value */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (mbox->storage->storage.set->mail_nfs_storage &&
4b41116563110d00330896a568eff1078c382827Timo Sirainenint mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen unsigned int lock_id = 0;
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen /* syncing failed, don't leave it locked */
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen } else if ((flags & MBOX_SYNC_LOCK_READING) == 0) {
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen /* drop to read lock */
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen unsigned int read_lock_id = 0;
73247459cf41eb1e5ae5bc61354db46d3b05ee75Timo Sirainen if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0)
4b41116563110d00330896a568eff1078c382827Timo Sirainen mbox->ibox.box.v.sync_notify(&mbox->ibox.box, 0, 0);
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainenmbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen if (index_mailbox_want_full_sync(&mbox->ibox, flags) && ret == 0) {
29e945d8550f297707f3a5f627a938401046c0ccTimo Sirainen if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
e5ee67f18b03015c88b579c8c1f17ebe6ce19b76Timo Sirainen if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0)
82c70897a2d0e6144ecc56ca8e0eb9fff768f2c5Timo Sirainen if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) {