mbox-sync.c revision 33d63688ed8b26dc333e3c2edbfb2fe6e412604d
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen required disk I/O. We may need to:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing X-UID and X-IMAPbase headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing or broken Content-Length header if there's space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Expunge specified messages
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Here's how we do it:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Start reading the mails mail headers from the beginning
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - X-Keywords and X-UID headers may contain extra spaces at the end of them,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen remember how much extra each message has and offset to beginning of the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If message flags are dirty and there's enough space to write them, do it
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we didn't have enough space, remember how much was missing and keep
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the total amount of them
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - When we encounter expunged message, check if the amount of empty space in
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen previous messages plus size of expunged message is enough to cover the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen missing space. If yes,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - execute the rewrite plan
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - forget all the messages before the expunged message. only remember
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen how much data we still have to move to cover the expunged message
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we encounter end of file, grow the file and execute the rewrite plan
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Rewrite plan goes:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Start from the first message that needs more space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If there's expunged messages before us, we have to write over them.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Move all messages after it backwards to fill it
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Each moved message's X-Keywords header should have n bytes extra
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen space, unless there's not enough space to do it.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If there's no expunged messages, we can move data either forward or
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen backward to get it. Calculate which requires less moving. Forward
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen counting may encounter more messages which require extra space, count
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we decide to move forwards and we had to go through dirty
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen messages, do the moving from last to first dirty message
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we encounter end of file, grow the file enough to get the required
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen amount of space plus enough space to fill X-Keywords headers full of
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen/* returns -1 = error, 0 = mbox changed since previous lock, 1 = didn't */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainenstatic int mbox_sync_lock(struct mbox_sync_context *sync_ctx, int lock_type)
3f190f4cbb9233a3a6830956cb5c7ae56a577b79Timo Sirainen i_assert(lock_type != F_WRLCK || !ibox->mbox_readonly);
f3f8d30207fd7d6c430361039dbfa88e6ffce411Timo Sirainen if (sync_ctx->lock_id == 0 || sync_ctx->input == NULL) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (mbox_lock(ibox, lock_type, &sync_ctx->lock_id) <= 0)
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen sync_ctx->file_input = sync_ctx->ibox->mbox_file_stream;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen sync_ctx->input = sync_ctx->ibox->mbox_stream;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* we didn't have the file open before -> it changed */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (st.st_mtime != old_st.st_mtime || st.st_size != old_st.st_size ||
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* same as before. we'll have to fix mbox stream to contain
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen correct from_offset, hdr_offset and body_offset. so, seek
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen to from_offset and read through the header. */
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, old_from_offset) < 0) {
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen "Message offset %s changed unexpectedly for mbox file "
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen "%s", dec2str(old_from_offset), sync_ctx->ibox->path);
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* put the extra space between last message's header and body */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (file_set_size(sync_ctx->fd, file_size) < 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
659fe5d24825b160cae512538088020d97a60239Timo Sirainenstatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync = buffer_get_modifyable_data(syncs_buf, &size);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen /* get EOF */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen if (mail_ctx->seq > 1 && sync_ctx->dest_first_mail) {
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen /* First message was expunged and this is the next one.
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen Skip \n header */
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen /* save the offset permanently with recent flag state */
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen mail_ctx->mail.from_offset = mail_ctx->from_offset;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen /* need to add 'O' flag to Status-header */
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen // FIXME: save it somewhere
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainenstatic int mbox_sync_buf_have_expunges(buffer_t *syncs_buf)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen for (i = 0; i < size; i++) {
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen if (sync[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
3f190f4cbb9233a3a6830956cb5c7ae56a577b79Timo Sirainen if (sync_ctx->ibox->mbox_readonly || sync_ctx->index_sync_ctx == NULL)
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen /* nothing for this or the future ones */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) {
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen sync_ctx->update_base_uid_last = sync_rec->uid2;
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen *sync_expunge_r = mbox_sync_buf_have_expunges(sync_ctx->syncs);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenstatic void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen for (i = 0; i < size; i++) {
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen if (sync[i].type != MAIL_INDEX_SYNC_TYPE_FLAGS)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mail_index_sync_flags_apply(&sync[i], flags, keywords);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* externally expunged message, remove from index */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen if (ret == 0 && uid < sync_ctx->hdr->next_uid) {
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen /* this UID was already in index and it was expunged */
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
9a06cabdfdf4d5e2f19a07e506c3c7d08a7e7038Timo Sirainen "mbox sync: Expunged message reappeared in mailbox %s "
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen "mbox sync: UID inserted in the middle of mailbox %s "
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen "(%u > %u)", sync_ctx->ibox->path, rec->uid, uid);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainenstatic int mbox_sync_get_from_offset(struct mbox_sync_context *sync_ctx,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* see if from_offset needs updating */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (mail_index_lookup_extra(sync_ctx->sync_view, seq,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (mbox_sync_get_from_offset(sync_ctx, sync_ctx->idx_seq,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* new message */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen /*FIXME:mail_cache_add(sync_ctx->cache_trans,
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen MAIL_CACHE_UID_STRING,
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen str_data(mail_ctx->uidl),
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen str_len(mail_ctx->uidl));*/
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* see if flags changed */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen (mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT));
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen rewriting would just move it anyway. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen struct istream *input = ctx->sync_ctx->file_input;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen const unsigned char *data;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen from_line_size = ctx->hdr_offset - ctx->from_offset;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainenstatic void update_from_offsets(struct mbox_sync_context *sync_ctx)
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen uint32_t idx, idx_seq, extra_idx = sync_ctx->ibox->mbox_extra_idx;
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen mails = buffer_get_modifyable_data(sync_ctx->mails, &size);
bbc92cfb17eb71d2ee9463c9cfd70dfea9a36bb6Timo Sirainen i_assert(sync_ctx->seq - sync_ctx->need_space_seq + 1 == size);
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen for (; idx < size; idx++, idx_seq++, mails++) {
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen mail_index_update_extra_rec(sync_ctx->t, idx_seq, extra_idx,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainenstatic int mbox_sync_check_excl_lock(struct mbox_sync_context *sync_ctx)
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (sync_ctx->ibox->mbox_lock_type == F_RDLCK) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if ((ret = mbox_sync_lock(sync_ctx, F_WRLCK)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if ((ret = mbox_sync_check_excl_lock(mail_ctx->sync_ctx)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->mail.offset = mail_ctx->from_offset;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->body_offset - mail_ctx->from_offset +
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* expunging first message, fix space to contain next
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen message's \n header too since it will be removed. */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* move the header backwards to fill expunged space */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* read the From-line before rewriting overwrites it */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* rewrite successful, write From-line to
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen new location */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_write_from_line(mail_ctx, move_diff) < 0)
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* nothing to do */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* first mail with no space to write it */
e5c08648676d1989f6e70b95e5990c26b3e8b96bTimo Sirainen sync_ctx->need_space_idx_seq = sync_ctx->idx_seq;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* create dummy message to describe the expunged data */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen buffer_append(sync_ctx->mails, &mail, sizeof(mail));
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenmbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* we have enough space now */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* don't waste too much on extra spacing */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->expunged_space = sync_ctx->space_diff - extra_space;
a205d315b0978985ba77d871f44e4a98273612e6Timo Sirainen if (mbox_sync_rewrite(sync_ctx, sync_ctx->space_diff,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* mail_ctx may contain wrong data after rewrite, so make sure we
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen don't try to access it */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainenmbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen if (mail_index_lookup_uid_range(sync_ctx->sync_view, uid, (uint32_t)-1,
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen if (mbox_sync_get_from_offset(sync_ctx, seq1, &offset) < 0)
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen /* set to -1, since it's always increased later */
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen sync_ctx->dest_first_mail = sync_ctx->seq == 0;
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, offset) < 0) {
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen "Cached message offset %s is invalid for mbox file %s",
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen mail_index_mark_corrupted(sync_ctx->ibox->index);
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainenstatic int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* we sync only what we need to. jump to first record that
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen needs updating */
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
f1b7a02a05fbca580934c7312aae63ea9542aa79Timo Sirainen /* nothing to do */
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen sync_rec = buffer_get_data(sync_ctx->syncs, &size);
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen ret = mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1);
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, 0) < 0) {
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen /* doesn't begin with a From-line */
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen mail_storage_set_error(sync_ctx->ibox->box.storage,
755fe6da51ab7f54aa1d86913cb344bffef60e79Timo Sirainen "Mailbox isn't a valid mbox file");
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* get all sync records related to this message */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen /* missing/broken X-UID. all the rest of the mails
71a74e26cf070a205d31cf6c6fae003f90027b63Timo Sirainen need new UIDs. */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* -1 = error, -2 = need to restart */
33d63688ed8b26dc333e3c2edbfb2fe6e412604dTimo Sirainen if (mbox_sync_update_index(sync_ctx, mail_ctx,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_handle_missing_space(mail_ctx) < 0)
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* move the body */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen } else if (sync_ctx->seq >= min_message_count) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid+1);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (buffer_get_used_size(sync_ctx->syncs) == 0) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* if there's no sync records left,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen we can stop */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* we can skip forward to next record which
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen needs updating. */
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input)) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* rest of the messages in index don't exist -> expunge them */
a393d9d6dabdc46cf724f8cb004a652b4036d53dTimo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen trailer_size = i_stream_get_size(sync_ctx->file_input) -
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx->space_diff += sync_ctx->expunged_space;
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen if (sync_ctx->expunged_space <= -sync_ctx->space_diff)
02b79f9c2636da1829eee5b92753602bba8b67edTimo Sirainen sync_ctx->expunged_space -= -sync_ctx->space_diff;
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (sync_ctx->seq != sync_ctx->need_space_seq) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen buffer_append(sync_ctx->mails, &mail_ctx->mail,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen /* copy trailer, then truncate the file */
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen offset = i_stream_get_size(sync_ctx->file_input) -
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (ftruncate(sync_ctx->fd, offset + trailer_size) < 0) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
292a66475ffe1037c2535063614f8beb71d266bfTimo Sirainen sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) ||
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen (sync_ctx->hdr->uid_validity == 0 && sync_ctx->seen_first_mail)) {
292a66475ffe1037c2535063614f8beb71d266bfTimo Sirainen /* we couldn't rewrite X-IMAPbase because it's
292a66475ffe1037c2535063614f8beb71d266bfTimo Sirainen a read-only mbox */
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen offsetof(struct mail_index_header, uid_validity),
9e59a1f3f095b3099478562cf3f3970a24736970Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input) &&
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen &sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen offsetof(struct mail_index_header, sync_stamp),
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen offsetof(struct mail_index_header, sync_size),
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainenstatic void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainenstatic int mbox_sync_do(struct mbox_sync_context *sync_ctx, int sync_header)
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "stat()");
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen (uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen (uint64_t)st.st_size == sync_ctx->hdr->sync_size ?
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if ((ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count)) == -1)
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* initially we had mbox read-locked, but later we needed a
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen write-lock. doing it required dropping the read lock.
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen we're here because mbox was modified before we got the
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen write-lock. so, restart the whole syncing. */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (mbox_sync_loop(sync_ctx, &mail_ctx, (uint32_t)-1) < 0)
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen /* only syncs left should be just appends (and their updates)
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen which weren't synced yet for some reason (crash). we'll just
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen ignore them, as we've overwritten them above. */
db7c9201c88e3d9bee10485194ee5b0c67249916Timo Sirainen memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen if (mbox_sync_update_index_header(sync_ctx) < 0)
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainenint mbox_sync_has_changed(struct index_mailbox *ibox)
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen if (mail_index_get_header(ibox->view, &hdr) < 0) {
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen return (uint32_t)st.st_mtime != hdr->sync_stamp ||
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainenstatic int mbox_sync_update_imap_base(struct mbox_sync_context *sync_ctx)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen sync_ctx->update_base_uid_last = sync_ctx->next_uid-1;
d17a90f5ae38edc8c0e6bf42c2162c9c18edb891Timo Sirainen if (mbox_sync_check_excl_lock(sync_ctx) == -1)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen if (mbox_sync_loop(sync_ctx, &mail_ctx, 1) < 0)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen if (mbox_sync_update_index_header(sync_ctx) < 0)
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainenint mbox_sync(struct index_mailbox *ibox, int last_commit,
63a61b7a739ae0f3f520215137d9c50f94d0f34fTimo Sirainen unsigned int lock_id = 0;
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen if ((ret = mbox_sync_has_changed(ibox)) < 0) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx.from_line = str_new(default_pool, 256);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx.header = str_new(default_pool, 4096);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
ce87b6ad8740cc66c5ab232081a2b4a535ff91aeTimo Sirainen lock_type = mail_index_sync_have_more(index_sync_ctx) &&
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen else if (mbox_sync_do(&sync_ctx, sync_header) < 0)
36e2fa21c22452470c1509cc63de20f7415c7b5eTimo Sirainen else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0) {
36e2fa21c22452470c1509cc63de20f7415c7b5eTimo Sirainen if (mail_index_sync_end(index_sync_ctx) < 0) {
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen sync_ctx.base_uid_last != sync_ctx.next_uid-1 &&
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen /* rewrite X-IMAPbase header. do it after mail_index_sync_end()
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen so previous transactions have been committed. */
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen /* FIXME: ugly .. */
36e2fa21c22452470c1509cc63de20f7415c7b5eTimo Sirainen (void)mail_index_get_header(sync_ctx.sync_view,
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen if ((ret = mbox_sync_update_imap_base(&sync_ctx)) < 0)
b3febb0933fdce10394d25093e23ce0a5aadddd3Timo Sirainen else if (mail_index_transaction_commit(sync_ctx.t,
36e2fa21c22452470c1509cc63de20f7415c7b5eTimo Sirainen if (mail_index_sync_end(sync_ctx.index_sync_ctx) < 0) {
8573a63139eab50fee1e497276f126f0d04d5637Timo Sirainen if (ret == 0 && ibox->mbox_lock_type == F_WRLCK) {
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen if (sync_ctx.lock_id != 0 && ibox->mbox_lock_type != F_RDLCK) {
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen /* drop to read lock */
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen unsigned int lock_id = 0;
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen /* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
2c7ab05ef98c46eb70c8ba6ea85e49749aafb2a3Timo Sirainen notice changes made by others */
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainenint mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen struct index_mailbox *ibox = (struct index_mailbox *)box;