bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* messages expunged within this sync */
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen /* all messages in this sync, sorted by mailbox_id
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen (but unsorted inside it for now, since it doesn't matter) */
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen uint32_t all_mails_idx, all_mails_prev_mailbox_id;
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainenstatic void virtual_sync_backend_box_deleted(struct virtual_sync_context *ctx,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void virtual_sync_set_uidvalidity(struct virtual_sync_context *ctx)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen offsetof(struct mail_index_header, uid_validity),
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void virtual_sync_external_flags(struct virtual_sync_context *ctx,
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen if (!mail_set_uid(bbox->sync_mail, real_uid)) {
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen /* we may have reopened the mailbox, which could have
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen caused the mail to be expunged already. */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* copy flags */
9badddc243d1a9925586c785a838b3bd81bacc7eAki Tuomi /* we don't need to keep recent flags here */
9badddc243d1a9925586c785a838b3bd81bacc7eAki Tuomi mail_index_update_flags(ctx->trans, vseq, MODIFY_REPLACE, flags & ~(MAIL_RECENT));
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* copy keywords */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen kw_names = mail_get_keywords(bbox->sync_mail);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen keywords = mail_index_keywords_create(ctx->index, kw_names);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen mail_index_update_keywords(ctx->trans, vseq, MODIFY_REPLACE, keywords);
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainenstatic int virtual_sync_mail_uid_cmp(const void *p1, const void *p2)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen const struct virtual_sync_mail *m1 = p1, *m2 = p2;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (m1->vrec.mailbox_id < m2->vrec.mailbox_id)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (m1->vrec.mailbox_id > m2->vrec.mailbox_id)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenvirtual_backend_box_sync_mail_set(struct virtual_backend_box *bbox)
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi trans = mailbox_transaction_begin(bbox->box, 0, __func__);
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainenstatic int bbox_mailbox_id_cmp(struct virtual_backend_box *const *b1,
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainenvirtual_sync_get_backend_box(struct virtual_mailbox *mbox, const char *name,
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen *bbox_r = virtual_backend_box_lookup_name(mbox, name);
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen if (*bbox_r != NULL || !mbox->sync_initialized)
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen /* another process just added a new mailbox.
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen we can't handle this currently. */
8f90ef65db62946aabe1969755edcdefb4eb430aTimo Sirainen mail_storage_set_error(mbox->box.storage, MAIL_ERROR_TEMP, t_strdup_printf(
8f90ef65db62946aabe1969755edcdefb4eb430aTimo Sirainen "Backend mailbox '%s' added by another session. "
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainenint virtual_mailbox_ext_header_read(struct virtual_mailbox *mbox,
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen const char *box_path = mailbox_get_path(&mbox->box);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen const struct virtual_mail_index_header *ext_hdr;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen const struct virtual_mail_index_mailbox_record *mailboxes;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen unsigned int i, count, ext_name_offset, ext_mailbox_count;
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen mail_index_get_header_ext(view, mbox->virtual_ext_id,
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen mbox->prev_uid_validity == hdr->uid_validity &&
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen mbox->prev_change_counter == ext_hdr->change_counter) {
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* fully refreshed */
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen mbox->search_args_crc32 != ext_hdr->search_args_crc32) {
825b0e819a7c48a366ddca23ec78b87e8c30e9b4Aki Tuomi mail_index_get_header_ext(view, mbox->virtual_guid_ext_id,
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen mbox->prev_change_counter = ext_hdr->change_counter;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen ext_hdr->mailbox_count > INT_MAX/sizeof(*mailboxes)) {
939a0d82523538b2de38a02bc9f790a67b7ebf47Timo Sirainen i_error("virtual index %s: Broken mailbox_count header",
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* update mailbox backends */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen for (i = 0; i < ext_mailbox_count; i++) {
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (mailboxes[i].id > ext_hdr->highest_mailbox_id ||
939a0d82523538b2de38a02bc9f790a67b7ebf47Timo Sirainen i_error("virtual index %s: Broken mailbox id",
939a0d82523538b2de38a02bc9f790a67b7ebf47Timo Sirainen i_error("virtual index %s: Broken mailbox name_len",
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (ext_name_offset + mailboxes[i].name_len > ext_size) {
939a0d82523538b2de38a02bc9f790a67b7ebf47Timo Sirainen i_error("virtual index %s: Broken mailbox list",
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen nameptr = CONST_PTR_OFFSET(ext_data, ext_name_offset);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen name = t_strndup(nameptr, mailboxes[i].name_len);
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen if (virtual_sync_get_backend_box(mbox, name, &bbox) < 0)
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen /* mailbox no longer exists. */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen bbox->sync_uid_validity = mailboxes[i].uid_validity;
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen mbox->highest_mailbox_id = ext_hdr == NULL ? 0 :
59a236b2ed657d694ab35242614dca32be4944daAki Tuomi /* do not mark it initialized if it's broken */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* assign new mailbox IDs if any are missing */
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen bboxes = array_get_modifiable(&mbox->backend_boxes, &count);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen for (i = 0; i < count; i++) {
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen bboxes[i]->mailbox_id = ++mbox->highest_mailbox_id;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* sort the backend mailboxes by mailbox_id. */
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen array_sort(&mbox->backend_boxes, bbox_mailbox_id_cmp);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainenstatic void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx)
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen struct virtual_mail_index_mailbox_record mailbox;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen bboxes = array_get_modifiable(&ctx->mbox->backend_boxes, &count);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen name_pos = mailbox_pos + sizeof(mailbox) * count;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen ext_hdr.change_counter = ++ctx->mbox->prev_change_counter;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen ext_hdr.highest_mailbox_id = ctx->mbox->highest_mailbox_id;
7e209b78ca757294dbbc15604c88673b3a6b0c39Timo Sirainen ext_hdr.search_args_crc32 = ctx->mbox->search_args_crc32;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen buffer_append(buf, &ext_hdr, sizeof(ext_hdr));
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen for (i = 0; i < count; i++) {
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen bboxes[i]->mailbox_id > bboxes[i-1]->mailbox_id);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mailbox.uid_validity = bboxes[i]->sync_uid_validity;
6d6c1517ef20e340a3aace406724fc8916f2d13fTimo Sirainen mailbox.highest_modseq = bboxes[i]->ondisk_highest_modseq;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen buffer_write(buf, mailbox_pos, &mailbox, sizeof(mailbox));
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen buffer_write(buf, name_pos, bboxes[i]->name, mailbox.name_len);
4299aef7cde9dcd3dd40fdb9fd1114f939754759Aki Tuomi /* repair the value */
4299aef7cde9dcd3dd40fdb9fd1114f939754759Aki Tuomi if (ctx->mbox->highest_mailbox_id != ext_hdr.highest_mailbox_id) {
4299aef7cde9dcd3dd40fdb9fd1114f939754759Aki Tuomi ext_hdr.highest_mailbox_id = ctx->mbox->highest_mailbox_id;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mail_index_get_header_ext(ctx->sync_view, ctx->mbox->virtual_ext_id,
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mail_index_ext_resize(ctx->trans, ctx->mbox->virtual_ext_id,
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainenstatic void virtual_sync_ext_header_update(struct virtual_sync_context *ctx)
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* we changed something - update the change counter in header */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen ext_hdr.change_counter = ++ctx->mbox->prev_change_counter;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen offsetof(struct virtual_mail_index_header, change_counter),
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen &ext_hdr.change_counter, sizeof(ext_hdr.change_counter));
a406615fbcef37b1d12f0be95a70abf23238c5efTimo Sirainenstatic int virtual_sync_index_rec(struct virtual_sync_context *ctx,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (!mail_index_lookup_seq_range(ctx->sync_view,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* already expunged, nothing to do. */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_index_lookup_ext(ctx->sync_view, vseq, virtual_ext_id,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen bbox = virtual_backend_box_lookup(ctx->mbox, vrec->mailbox_id);
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen if (virtual_backend_box_open(ctx->mbox, bbox) < 0) {
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen virtual_backend_box_accessed(ctx->mbox, bbox);
42270b2d8d9bb0d04b16e0ead727154b32399a3fTimo Sirainen if (!mail_set_uid(bbox->sync_mail, vrec->real_uid)) {
42270b2d8d9bb0d04b16e0ead727154b32399a3fTimo Sirainen /* message is already expunged from backend mailbox. */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen flags = sync_rec->add_flags & MAIL_FLAGS_NONRECENT;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen flags = sync_rec->remove_flags & MAIL_FLAGS_NONRECENT;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen kw_names[0] = ctx->kw_all[sync_rec->keyword_idx];
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen keywords = mailbox_keywords_create_valid(bbox->box,
a406615fbcef37b1d12f0be95a70abf23238c5efTimo Sirainenstatic int virtual_sync_index_changes(struct virtual_sync_context *ctx)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen keywords = mail_index_get_keywords(ctx->index);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen ctx->kw_all = array_count(keywords) == 0 ? NULL :
a406615fbcef37b1d12f0be95a70abf23238c5efTimo Sirainen while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
a406615fbcef37b1d12f0be95a70abf23238c5efTimo Sirainen if (virtual_sync_index_rec(ctx, &sync_rec) < 0)
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainenstatic void virtual_sync_index_finish(struct virtual_sync_context *ctx)
cff36c02dd80217d0c6615c614d5b5cc642df066Aki Tuomi view = mail_index_transaction_open_updated_view(ctx->trans);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* mark the newly seen messages as recent */
cff36c02dd80217d0c6615c614d5b5cc642df066Aki Tuomi if (mail_index_lookup_seq_range(view, hdr->first_recent_uid,
cff36c02dd80217d0c6615c614d5b5cc642df066Aki Tuomi mailbox_recent_flags_set_seqs(&ctx->mbox->box, view,
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* entire mailbox list needs to be rewritten */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* update only changed parts in the header */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenstatic int virtual_sync_backend_box_init(struct virtual_backend_box *bbox)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen enum mailbox_search_result_flags result_flags;
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi trans = mailbox_transaction_begin(bbox->box, 0, __func__);
3e1d94509fd76c56f2eb54f98ea02ade9ac2d44aTimo Sirainen mail_search_args_init(bbox->search_args, bbox->box, FALSE, NULL);
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen search_ctx = mailbox_search_init(trans, bbox->search_args, NULL,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* save the result and keep it updated */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen result_flags = MAILBOX_SEARCH_RESULT_FLAG_UPDATE |
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen mailbox_search_result_save(search_ctx, result_flags);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* add the found UIDs to uidmap. virtual_uid gets assigned later. */
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen while (mailbox_search_next(search_ctx, &mail)) {
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainenvirtual_backend_uidmap_bsearch_cmp(const uint32_t *uidp,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenvirtual_sync_mailbox_box_remove(struct virtual_sync_context *ctx,
e3736b5d480878031c386ac55d201fcf08e68766Timo Sirainen unsigned int i, src, dest, uid_count, rec_count;
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* everything in removed_uids should exist in bbox->uids */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &rec_count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* find the first uidmap record to be removed */
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen if (!array_bsearch_insert_pos(&bbox->uids, &uids[0].seq1,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* remove the unwanted messages */
e3736b5d480878031c386ac55d201fcf08e68766Timo Sirainen for (i = 0; i < uid_count; i++) {
781a9337941148bf1fb69bc55b205ab27fd08af0Aki Tuomi /* has not been assigned yet */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenvirtual_sync_mailbox_box_add(struct virtual_sync_context *ctx,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen unsigned int i, src, dest, uid_count, add_count, rec_count;
a21823d90cee6a18aeab0378637472c7e3fbbab2Timo Sirainen added_uids = array_get(added_uids_arr, &uid_count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* none of added_uids should exist in bbox->uids. find the position
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen of the first inserted index. */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &rec_count);
a21823d90cee6a18aeab0378637472c7e3fbbab2Timo Sirainen added_uids[0].seq1 > uidmap[rec_count-1].real_uid) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* fast path: usually messages are appended */
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen } else if (array_bsearch_insert_pos(&bbox->uids, &added_uids[0].seq1,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* make space for all added UIDs. */
a21823d90cee6a18aeab0378637472c7e3fbbab2Timo Sirainen array_idx_clear(&bbox->uids, dest + add_count-1);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &rec_count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* add/move the UIDs to their correct positions */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen for (i = 0; i < uid_count; i++) {
41955c400476941fa274f18b106a5922866fd780Timo Sirainen while (src < rec_count && uidmap[src].real_uid < add_uid)
a21823d90cee6a18aeab0378637472c7e3fbbab2Timo Sirainen for (; add_uid <= added_uids[i].seq2; add_uid++, dest++) {
5fe2339f8ad2698f0ea7c87a968c8ed2b853bad9Aki Tuomivirtual_sync_mailbox_box_update_flags(struct virtual_sync_context *ctx,
5fe2339f8ad2698f0ea7c87a968c8ed2b853bad9Aki Tuomi while(seq_range_array_iter_nth(&iter, i++, &uid)) {
5fe2339f8ad2698f0ea7c87a968c8ed2b853bad9Aki Tuomi /* the entry has been already removed either by
5fe2339f8ad2698f0ea7c87a968c8ed2b853bad9Aki Tuomi us or some other session. doesn't matter,
5fe2339f8ad2698f0ea7c87a968c8ed2b853bad9Aki Tuomi we don't need to update the flags.
5fe2339f8ad2698f0ea7c87a968c8ed2b853bad9Aki Tuomi it might also have not yet been assigned a uid
5fe2339f8ad2698f0ea7c87a968c8ed2b853bad9Aki Tuomi so we don't want to update the flags then either.
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainenstatic int virtual_backend_uidmap_cmp(const struct virtual_backend_uidmap *u1,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainenstatic void virtual_sync_bbox_uids_sort(struct virtual_backend_box *bbox)
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen /* the uidmap must be sorted by real_uids */
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen array_sort(&bbox->uids, virtual_backend_uidmap_cmp);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainenstatic void virtual_sync_backend_boxes_sort_uids(struct virtual_mailbox *mbox)
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen bboxes = array_get(&mbox->backend_boxes, &count);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen for (i = 0; i < count; i++) {
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainenvirtual_sync_backend_add_vmsgs_results(struct virtual_sync_context *ctx,
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen mail_index_lookup_uid(ctx->sync_view, vseq, &vuid);
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen else if (mail_index_lookup_seq(bbox->box->view, real_uid, &seq))
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen seq_range_array_add(&result->removed_uids, real_uid);
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainenvirtual_sync_backend_handle_old_vmsgs(struct virtual_sync_context *ctx,
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen const struct virtual_sync_mail *sync_mail, *sync_mails;
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen /* find the messages that currently exist in virtual index and add them
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen to the backend mailbox's list of uids. */
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen i_assert(ctx->all_mails_prev_mailbox_id < bbox->mailbox_id);
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen sync_mails = array_get(&ctx->all_mails, &messages);
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen for (i = ctx->all_mails_idx; i < messages; i++) {
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen if (sync_mail->vrec.mailbox_id != bbox->mailbox_id) {
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen if (sync_mail->vrec.mailbox_id < bbox->mailbox_id) {
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen /* stale mailbox_id, ignore */
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen /* Should be in mailbox_id order,
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen so skip to next box */
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen virtual_sync_backend_add_vmsgs_results(ctx, bbox,
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen sync_mail->vrec.real_uid, result, sync_mail->vseq);
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen ctx->all_mails_prev_mailbox_id = bbox->mailbox_id;
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen /* there should be only a single backend mailbox, but in the
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen existing index there may be stale mailbox_ids that we'll
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen just skip over. */
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen messages = mail_index_view_get_messages_count(ctx->sync_view);
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainenstatic int virtual_sync_backend_box_continue(struct virtual_sync_context *ctx,
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen const enum mailbox_search_result_flags result_flags =
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen struct mail_index_view *view = bbox->box->view;
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen ARRAY_TYPE(seq_range) expunged_uids = ARRAY_INIT, removed_uids;
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen ARRAY_TYPE(seq_range) added_uids, flag_update_uids;
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen /* initialize the search result from all the existing messages in
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen virtual index. */
3e1d94509fd76c56f2eb54f98ea02ade9ac2d44aTimo Sirainen mail_search_args_init(bbox->search_args, bbox->box, FALSE, NULL);
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen result = mailbox_search_result_alloc(bbox->box, bbox->search_args,
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen i_assert(array_count(&result->removed_uids) == 0);
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen virtual_sync_backend_handle_old_vmsgs(ctx, bbox, result);
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen /* these are all expunged messages. treat them separately from
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen "no longer matching messages" (=removed_uids) */
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen t_array_init(&expunged_uids, array_count(&result->removed_uids));
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen array_append_array(&expunged_uids, &result->removed_uids);
6cd263d9a9812174a63e852ad6a9a8cdf63cfd8eTimo Sirainen /* get list of changed old messages (messages already once seen by
6cd263d9a9812174a63e852ad6a9a8cdf63cfd8eTimo Sirainen virtual index), based on modseq changes. (we'll assume all modseq
6cd263d9a9812174a63e852ad6a9a8cdf63cfd8eTimo Sirainen changes are due to flag changes, which may not be true in future) */
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen !mail_index_lookup_seq_range(view, 1, bbox->sync_next_uid-1,
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen old_highest_modseq = mail_index_modseq_get_highest(view);
6cd263d9a9812174a63e852ad6a9a8cdf63cfd8eTimo Sirainen t_array_init(&flag_update_uids, I_MIN(128, old_msg_count));
3b1dd975d0c543d76679593f86e136d43bf40635Timo Sirainen if (bbox->sync_highest_modseq < old_highest_modseq) {
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen /* update the search result based on the flag changes and
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen new messages */
6cd263d9a9812174a63e852ad6a9a8cdf63cfd8eTimo Sirainen if (index_search_result_update_flags(result, &flag_update_uids) < 0 ||
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen index_search_result_update_appends(result, old_msg_count) < 0) {
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen mailbox_search_result_sync(result, &removed_uids, &added_uids);
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen seq_range_array_remove_seq_range(&removed_uids, &expunged_uids);
7bdd29cf24fa3feaef27dcd28679b1cf00357541Timo Sirainen virtual_sync_mailbox_box_remove(ctx, bbox, &expunged_uids);
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen virtual_sync_mailbox_box_remove(ctx, bbox, &removed_uids);
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen /* delayed remove */
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen seq_range_array_merge(&bbox->sync_pending_removes,
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen virtual_sync_mailbox_box_add(ctx, bbox, &added_uids);
5fe2339f8ad2698f0ea7c87a968c8ed2b853bad9Aki Tuomi virtual_sync_mailbox_box_update_flags(ctx, bbox, &flag_update_uids);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainenstatic void virtual_sync_drop_existing(struct virtual_backend_box *bbox,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (!seq_range_array_iter_nth(&iter, n++, &add_uid))
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen (void)array_bsearch_insert_pos(&bbox->uids, &add_uid,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &count);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen t_array_init(&drop_uids, array_count(added_uids));
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen for (; i < count; ) {
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (!seq_range_array_iter_nth(&iter, n++, &add_uid))
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen seq_range_array_remove_seq_range(added_uids, &drop_uids);
7fc0f80480063a9d4cb9e8c07b50db2a5627799eTimo Sirainenstatic void virtual_sync_drop_nonexistent(struct virtual_backend_box *bbox,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen seq_range_array_iter_init(&iter, removed_uids);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (!seq_range_array_iter_nth(&iter, n++, &remove_uid))
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen (void)array_bsearch_insert_pos(&bbox->uids, &remove_uid,
c20d64512d687f0abf7c1a0aa4fa0174da0a18e3Timo Sirainen t_array_init(&drop_uids, array_count(removed_uids)); iter_done = FALSE;
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &count);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen for (; i < count; ) {
c20d64512d687f0abf7c1a0aa4fa0174da0a18e3Timo Sirainen if (!seq_range_array_iter_nth(&iter, n++, &remove_uid)) {
280b771b7ec7e49caa399540dfe8faf42999a1adTimo Sirainen } while (seq_range_array_iter_nth(&iter, n++, &remove_uid));
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen seq_range_array_remove_seq_range(removed_uids, &drop_uids);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenstatic void virtual_sync_mailbox_box_update(struct virtual_sync_context *ctx,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen ARRAY_TYPE(seq_range) removed_uids, added_uids, temp_uids;
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen mailbox_search_result_sync(bbox->search_result,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (array_is_created(&bbox->sync_outside_expunges)) {
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen seq_range_array_remove_seq_range(&bbox->sync_outside_expunges,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen virtual_sync_drop_existing(bbox, &added_uids);
7fc0f80480063a9d4cb9e8c07b50db2a5627799eTimo Sirainen virtual_sync_drop_nonexistent(bbox, &removed_uids);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* if any of the pending removes came back, we don't want to expunge
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen them anymore. also since they already exist, remove them from
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen added_uids. */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen count1 = array_count(&bbox->sync_pending_removes);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen array_append_array(&temp_uids, &bbox->sync_pending_removes);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen &bbox->sync_pending_removes, &added_uids) > 0) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* delay removing messages that don't match the search
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen criteria, but don't delay removing expunged messages */
41955c400476941fa274f18b106a5922866fd780Timo Sirainen seq_range_array_remove_seq_range(&bbox->sync_pending_removes,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen seq_range_array_remove_seq_range(&removed_uids,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen seq_range_array_merge(&bbox->sync_pending_removes,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen } else if (array_count(&bbox->sync_pending_removes) > 0) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* remove all current and old */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen seq_range_array_merge(&bbox->sync_pending_removes,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen virtual_sync_mailbox_box_remove(ctx, bbox, &removed_uids);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen virtual_sync_mailbox_box_add(ctx, bbox, &added_uids);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenstatic bool virtual_sync_find_seqs(struct virtual_backend_box *bbox,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen unsigned int *idx1_r,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen unsigned int *idx2_r)
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen mail_index_lookup_uid(bbox->box->view, sync_rec->seq1, &uid1);
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen mail_index_lookup_uid(bbox->box->view, sync_rec->seq2, &uid2);
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen (void)array_bsearch_insert_pos(&bbox->uids, &uid1,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if (idx == count || uidmap[idx].real_uid > uid2)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen while (idx < count && uidmap[idx].real_uid <= uid2) idx++;
41955c400476941fa274f18b106a5922866fd780Timo Sirainenstatic void virtual_sync_expunge_add(struct virtual_sync_context *ctx,
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen mail_index_lookup_uid(bbox->box->view, sync_rec->seq1, &uid1);
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen mail_index_lookup_uid(bbox->box->view, sync_rec->seq2, &uid2);
41955c400476941fa274f18b106a5922866fd780Timo Sirainen /* remember only the expunges for messages that
41955c400476941fa274f18b106a5922866fd780Timo Sirainen already exist for this mailbox */
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen (void)array_bsearch_insert_pos(&bbox->uids, &uid1,
41955c400476941fa274f18b106a5922866fd780Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &count);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&ctx->sync_expunges, uidmap[i].real_uid);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenstatic int virtual_sync_backend_box_sync(struct virtual_sync_context *ctx,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen sync_ctx = mailbox_sync_init(bbox->box, sync_flags);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen while (mailbox_sync_next(sync_ctx, &sync_rec)) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* no need to keep track of expunges */
41955c400476941fa274f18b106a5922866fd780Timo Sirainen virtual_sync_expunge_add(ctx, bbox, &sync_rec);
781a9337941148bf1fb69bc55b205ab27fd08af0Aki Tuomi /* has not been even assigned yet */
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen /* expunged by another session,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen but we haven't yet updated
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen bbox->uids. */
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainen if (mailbox_sync_deinit(&sync_ctx, &sync_status) < 0) {
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainen if (mailbox_get_last_mail_error(bbox->box) != MAIL_ERROR_NOTFOUND)
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainen /* mailbox was deleted */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainenstatic void virtual_sync_backend_ext_header(struct virtual_sync_context *ctx,
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen offsetof(struct virtual_mail_index_mailbox_record,
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen struct virtual_mail_index_mailbox_record mailbox;
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen mailbox_get_open_status(bbox->box, STATUS_UIDVALIDITY |
6d6c1517ef20e340a3aace406724fc8916f2d13fTimo Sirainen array_count(&bbox->sync_pending_removes) > 0 ? 0 :
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (bbox->sync_uid_validity == status.uidvalidity &&
6d6c1517ef20e340a3aace406724fc8916f2d13fTimo Sirainen bbox->sync_highest_modseq == status.highest_modseq &&
6d6c1517ef20e340a3aace406724fc8916f2d13fTimo Sirainen bbox->ondisk_highest_modseq == wanted_ondisk_highest_modseq)
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* mailbox changed - update extension header */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen bbox->sync_highest_modseq = status.highest_modseq;
6d6c1517ef20e340a3aace406724fc8916f2d13fTimo Sirainen bbox->ondisk_highest_modseq = wanted_ondisk_highest_modseq;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* we'll rewrite the entire header later */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mailbox.uid_validity = bbox->sync_uid_validity;
6d6c1517ef20e340a3aace406724fc8916f2d13fTimo Sirainen mailbox.highest_modseq = bbox->ondisk_highest_modseq;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mailbox_offset = sizeof(struct virtual_mail_index_header) +
9599d7ecbc84e9d33295e1e9a9be937faddce68aTimo Sirainen (bbox->sync_mailbox_idx1-1) * sizeof(mailbox);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,
e5536fd672aa08bce4bd57ca6a844d8d97991b6fTimo Sirainenstatic void virtual_sync_backend_box_deleted(struct virtual_sync_context *ctx,
e5536fd672aa08bce4bd57ca6a844d8d97991b6fTimo Sirainen /* delay its full removal until the next time we open the virtual
e5536fd672aa08bce4bd57ca6a844d8d97991b6fTimo Sirainen mailbox. for now just treat it as if it was empty. */
e5536fd672aa08bce4bd57ca6a844d8d97991b6fTimo Sirainen seq_range_array_add(&removed_uids, uidmap->real_uid);
e5536fd672aa08bce4bd57ca6a844d8d97991b6fTimo Sirainen virtual_sync_mailbox_box_remove(ctx, bbox, &removed_uids);
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainenvirtual_try_open_and_sync_backend_box(struct virtual_sync_context *ctx,
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainen ret = virtual_backend_box_open(ctx->mbox, bbox);
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainen if (mailbox_get_last_mail_error(bbox->box) != MAIL_ERROR_NOTFOUND)
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainen /* mailbox was deleted */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenstatic int virtual_sync_backend_box(struct virtual_sync_context *ctx,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* if we already did some changes to index, commit them before
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen syncing starts. */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen sync_flags = ctx->flags & (MAILBOX_SYNC_FLAG_FULL_READ |
ce930f99c6a78f2c74b00df1ad2337095978a9dbTimo Sirainen /* a) first sync in this process.
ce930f99c6a78f2c74b00df1ad2337095978a9dbTimo Sirainen b) we had auto-closed this backend mailbox.
ce930f99c6a78f2c74b00df1ad2337095978a9dbTimo Sirainen first try to quickly check if the mailbox has changed.
ce930f99c6a78f2c74b00df1ad2337095978a9dbTimo Sirainen if we can do that check from mailbox list index, we don't
ce930f99c6a78f2c74b00df1ad2337095978a9dbTimo Sirainen even need to open the mailbox. */
ce930f99c6a78f2c74b00df1ad2337095978a9dbTimo Sirainen i_assert(array_count(&bbox->sync_pending_removes) == 0);
a385399497bdb50d4dfce729ffc852b75ed46a36Timo Sirainen /* a) index already opened, refresh it
a385399497bdb50d4dfce729ffc852b75ed46a36Timo Sirainen b) delayed error handling for mailbox_open()
a385399497bdb50d4dfce729ffc852b75ed46a36Timo Sirainen that failed in virtual_notify_changes() */
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainen if ((ret = virtual_try_open_and_sync_backend_box(ctx, bbox, sync_flags)) <= 0)
dd37e2ff291fbebac1b94e8aad50f3bdf7531049Timo Sirainen if (mailbox_get_status(bbox->box, STATUS_UIDVALIDITY |
e5536fd672aa08bce4bd57ca6a844d8d97991b6fTimo Sirainen if (mailbox_get_last_mail_error(bbox->box) != MAIL_ERROR_NOTFOUND)
e5536fd672aa08bce4bd57ca6a844d8d97991b6fTimo Sirainen /* mailbox was deleted */
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen if (status.uidvalidity == bbox->sync_uid_validity &&
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen status.highest_modseq == bbox->sync_highest_modseq) {
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen /* mailbox hasn't changed since we last opened it,
7ab62198bfb9ac210b11ea7a147234e0f72f6448Timo Sirainen skip it for now.
7ab62198bfb9ac210b11ea7a147234e0f72f6448Timo Sirainen we'll still need to create the bbox->uids mapping
7ab62198bfb9ac210b11ea7a147234e0f72f6448Timo Sirainen using the current index. */
7ab62198bfb9ac210b11ea7a147234e0f72f6448Timo Sirainen virtual_sync_backend_handle_old_vmsgs(ctx, bbox, NULL);
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen /* first time we're opening the index */
5b7ff90a6c078b07406bce434c719017a0f51825Timo Sirainen if ((ret = virtual_try_open_and_sync_backend_box(ctx, bbox, sync_flags)) <= 0)
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (status.uidvalidity != bbox->sync_uid_validity) {
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* UID validity changed since last sync (or this is
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen the first sync), do a full search */
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen /* build the initial search using the saved modseq. */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen ret = virtual_sync_backend_box_continue(ctx, bbox);
7ab62198bfb9ac210b11ea7a147234e0f72f6448Timo Sirainen i_assert(bbox->search_result != NULL || ret < 0);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* sync using the existing search result */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen ret = virtual_sync_backend_box_sync(ctx, bbox, sync_flags);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainenstatic void virtual_sync_backend_map_uids(struct virtual_sync_context *ctx)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen messages = mail_index_view_get_messages_count(ctx->sync_view);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* sort the messages in current view by their backend mailbox and
ddd5fb9499ae08f7f0d4b41306bb4fca053e493bTimo Sirainen vmails = i_new(struct virtual_sync_mail, messages);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen mail_index_lookup_ext(ctx->sync_view, vseq, virtual_ext_id,
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen qsort(vmails, messages, sizeof(*vmails), virtual_sync_mail_uid_cmp);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* create real mailbox uid -> virtual uid mapping and expunge
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen messages no longer matching the search rule */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen for (i = 0; i < messages; i++) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* add the rest of the newly seen messages */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen for (; j < uidmap_count; j++) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* the entire mailbox is lost */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen mail_index_lookup_uid(ctx->sync_view, vseq, &vuid);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* if virtual record doesn't exist in uidmap, it's expunged */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen for (; j < uidmap_count; j++) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* newly seen message */
3b49aee9ced3b0370a3be396aca53acd5f21418cTimo Sirainen if (j == uidmap_count || uidmap[j].real_uid != vrec->real_uid)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* exists - update uidmap and flags */
96d85fb3606d13a38dac1b2def55eff414b2b31eTimo Sirainen /* mailbox is completely unchanged since last
96d85fb3606d13a38dac1b2def55eff414b2b31eTimo Sirainen sync - no need to sync flags */
2b9fd042e701cfe7d79c4294a5ab401d6ec9ce18Timo Sirainen /* finish adding messages to the last mailbox */
2b9fd042e701cfe7d79c4294a5ab401d6ec9ce18Timo Sirainen for (; j < uidmap_count; j++) {
ddd5fb9499ae08f7f0d4b41306bb4fca053e493bTimo Sirainenstatic void virtual_sync_new_backend_boxes(struct virtual_sync_context *ctx)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* if there are any mailboxes we didn't yet sync, add new messages in
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen bboxes = array_get(&ctx->mbox->backend_boxes, &count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen for (i = 0; i < count; i++) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen add_rec.rec.mailbox_id = bboxes[i]->mailbox_id;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen uidmap = array_get_modifiable(&bboxes[i]->uids, &uidmap_count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen for (j = 0; j < uidmap_count; j++) {
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainenstatic int virtual_add_record_cmp(const struct virtual_add_record *add1,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if (add1->received_date < add2->received_date)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if (add1->received_date > add2->received_date)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* if they're in same mailbox, we can order them correctly by the UID.
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if they're in different mailboxes, ordering by UID doesn't really
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen help but it doesn't really harm either. */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* two messages in different mailboxes have the same received date
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainenstatic int virtual_sync_backend_sort_new(struct virtual_sync_context *ctx)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* get all messages' received dates */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen adds = array_get_modifiable(&ctx->all_adds, &count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen virtual_backend_box_open(ctx->mbox, bbox) < 0)
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen if (!mail_set_uid(bbox->sync_mail, vrec->real_uid)) {
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen /* we may have reopened the mailbox, which could have
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen caused the mail to be expunged already. */
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen } else if (mail_get_received_date(bbox->sync_mail,
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen /* expunged already, just add it somewhere */
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen array_sort(&ctx->all_adds, virtual_add_record_cmp);
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainenstatic int virtual_sync_backend_add_new(struct virtual_sync_context *ctx)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen adds = array_get_modifiable(&ctx->all_adds, &count);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen ctx->mbox->sync_virtual_next_uid = hdr->next_uid;
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if (adds[0].rec.mailbox_id == adds[count-1].rec.mailbox_id) {
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* all messages are from a single mailbox. add them in
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen the same order. */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* sort new messages by received date to get the add order */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
212e9e43a7d49242446331fd43ba519eda936d60Timo Sirainen virtual_backend_box_open(ctx->mbox, bbox) < 0)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen mail_index_update_ext(ctx->trans, vseq, virtual_ext_id,
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen virtual_sync_external_flags(ctx, bbox, vseq, vrec->real_uid);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* assign UIDs to new messages */
e5601cd11edce275b4c423523bd00ee3e4eeae42Timo Sirainen mail_index_append_finish_uids(ctx->trans, first_uid, &saved_uids);
e5601cd11edce275b4c423523bd00ee3e4eeae42Timo Sirainen i_assert(seq_range_count(&saved_uids) == count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* update virtual UIDs in uidmap */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen if (!array_bsearch_insert_pos(&bbox->uids, &vrec->real_uid,
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen uidmap = array_idx_modifiable(&bbox->uids, idx);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen ctx->mbox->sync_virtual_next_uid = first_uid + i;
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainenvirtual_sync_apply_existing_appends(struct virtual_sync_context *ctx)
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (ctx->mbox->sync_virtual_next_uid >= hdr->next_uid)
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen /* another process added messages to virtual index. get backend boxes'
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen uid lists up-to-date by adding the new messages there. */
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (!mail_index_lookup_seq_range(ctx->sync_view,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen mail_index_lookup_ext(ctx->sync_view, seq, virtual_ext_id,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen mail_index_lookup_uid(ctx->sync_view, seq, &uidmap.virtual_uid);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen virtual_sync_backend_boxes_sort_uids(ctx->mbox);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainenvirtual_sync_apply_existing_expunges(struct virtual_mailbox *mbox,
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen struct index_mailbox_sync_context *isync_ctx =
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen (struct index_mailbox_sync_context *)sync_ctx;
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen unsigned int n = 0;
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen seq_range_array_iter_init(&iter, isync_ctx->expunges);
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen while (seq_range_array_iter_nth(&iter, n++, &seq)) {
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen if (!array_is_created(&bbox->sync_outside_expunges))
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen i_array_init(&bbox->sync_outside_expunges, 32);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&bbox->sync_outside_expunges,
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainenstatic int virtual_sync_mail_mailbox_cmp(const struct virtual_sync_mail *m1,
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen if (m1->vrec.mailbox_id < m2->vrec.mailbox_id)
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen if (m1->vrec.mailbox_id > m2->vrec.mailbox_id)
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainenstatic void virtual_sync_bboxes_get_mails(struct virtual_sync_context *ctx)
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen messages = mail_index_view_get_messages_count(ctx->sync_view);
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen sync_mail = array_append_space(&ctx->all_mails);
7fc0ca6c1f664de6506afa816200d115f9f80391Timo Sirainen array_sort(&ctx->all_mails, virtual_sync_mail_mailbox_cmp);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic int virtual_sync_backend_boxes(struct virtual_sync_context *ctx)
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen if (virtual_sync_apply_existing_appends(ctx) < 0)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen bboxes = array_get(&ctx->mbox->backend_boxes, &count);
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen /* we have different optimizations depending on whether the virtual
5948e3e120c5dcaf0aeb44405566381045fa82d6Timo Sirainen mailbox consists of multiple backend boxes or just one */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen for (i = 0; i < count; i++) {
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen if (virtual_sync_backend_box(ctx, bboxes[i]) < 0) {
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen /* backend failed, copy the error */
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen /* initial sync: assign virtual UIDs to existing messages and
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen sync all flags */
a5a4433751bb3dce9c870ea95e1b69efe697d417Timo Sirainen for (i = 0; i < count; i++) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void virtual_sync_backend_boxes_finish(struct virtual_sync_context *ctx)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen bboxes = array_get(&ctx->mbox->backend_boxes, &count);
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen for (i = 0; i < count; i++)
dee43975a70bcdb9dc83d34d6a2b177d37bb7194Timo Sirainen virtual_backend_box_sync_mail_unset(bboxes[i]);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic int virtual_sync_finish(struct virtual_sync_context *ctx, bool success)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
939a0d82523538b2de38a02bc9f790a67b7ebf47Timo Sirainen /* make sure we don't complain about the same errors
939a0d82523538b2de38a02bc9f790a67b7ebf47Timo Sirainen over and over again. */
939a0d82523538b2de38a02bc9f790a67b7ebf47Timo Sirainen i_error("virtual index %s: Failed to unlink() "
939a0d82523538b2de38a02bc9f790a67b7ebf47Timo Sirainen "broken indexes: %m",
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_index_sync_rollback(&ctx->index_sync_ctx);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic int virtual_sync(struct virtual_mailbox *mbox,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* Removed messages are expunged when
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen a) EXPUNGE is used
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen b) Mailbox is being opened (FIX_INCONSISTENT is set) */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen index_sync_flags = MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY |
4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen index_sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen ret = mail_index_sync_begin(ctx->index, &ctx->index_sync_ctx,
d85a1a9d9af4a36ded4d30cb277905c807de2ec5Timo Sirainen ret = virtual_mailbox_ext_header_read(mbox, ctx->sync_view, &broken);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* apply changes from virtual index to backend mailboxes */
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* update list of UIDs in backend mailboxes */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenvirtual_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
d22301419109ed4a38351715e6760011421dadecTimo Sirainen if (index_mailbox_want_full_sync(&mbox->box, flags) && ret == 0)
905457e0982fc15930d90e174f271dc69f9afcf9Timo Sirainen sync_ctx = index_mailbox_sync_init(box, flags, ret < 0);