dsync-mailbox-import.c revision 1c95b8403d2d4dcf35e23fc0a1b51922f120a82a
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* linked list of mails for this GUID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* if non-NULL, this mail exists in both local and remote. this link
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen points to the other side. */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo SirainenHASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *);
a75d470c9223a75801418fcdda258885c36317e0Timo SirainenHASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *);
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t last_common_modseq, last_common_pvt_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mailbox_transaction_context *trans, *ext_trans;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* UID => struct dsync_mail_change */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct dsync_mail_change *) maybe_saves;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID => struct importer_new_mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* UID => struct importer_new_mail */
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct dsync_mail_request) mail_requests;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t prev_uid, next_local_seq, local_uid_next;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&sarg->value.seqset, search_args->pool, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_search_init(importer->trans, search_args, NULL,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_search_next(importer->search_ctx, &importer->cur_mail))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->next_local_seq = importer->cur_mail->seq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this flag causes cur_guid to be looked up later */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const enum mailbox_transaction_flags ext_trans_flags =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer = p_new(pool, struct dsync_mailbox_importer, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->last_common_modseq = last_common_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->last_common_pvt_modseq = last_common_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last_common_uid != 0 || last_common_modseq != 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->remote_first_recent_uid = remote_first_recent_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->remote_highest_modseq = remote_highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create_direct(&importer->import_uids, pool, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_array_init(&importer->maybe_expunge_uids, 16);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->trans = mailbox_transaction_begin(importer->box,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->ext_trans = mailbox_transaction_begin(box, ext_trans_flags);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->mail = mail_alloc(importer->trans, 0, NULL);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0;
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen mailbox_get_open_status(importer->box, STATUS_UIDNEXT |
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->local_initial_highestmodseq = status.highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq;
08e9fec5ba9e1a26e658c4224207d666b6ced27dTimo Sirainen importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainenstatic void dsync_mail_error(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen errstr = mailbox_get_last_error(importer->box, &error);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("Can't lookup %s for UID=%u: %s", field, mail->uid, errstr);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenimporter_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* end of search */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (importer->cur_mail->seq < importer->next_local_seq ||
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this message exists locally, but remote didn't send
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen expunge-change for it. if the message's
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid <= last-common-uid, it should be deleted */
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&importer->maybe_expunge_uids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!mailbox_search_next(importer->search_ctx,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_uid_has_change = importer->cur_mail != NULL &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_error(importer, importer->cur_mail, "GUID");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return importer_next_mail(importer, wanted_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* make sure next_local_seq gets updated in case we came here
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen because of min_uid */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->next_local_seq = importer->cur_mail->seq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenimporter_mail_cmp(const struct importer_mail *m1,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void importer_mail_request(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->want_mail_requests && !newmail->uid_in_local) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen request = array_append_space(&importer->mail_requests);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void newmail_link(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *first_mail, **last, *mail, *link = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen first_mail = hash_table_lookup(importer->import_guids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* first mail for this GUID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* FIXME: ? */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen first_mail = hash_table_lookup(importer->import_uids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* first mail for this UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* 1) add the newmail to the end of the linked list
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen 2) find our link */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (mail = first_mail; mail != NULL; mail = mail->next) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* add a record for local mail */
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->guid = p_strdup(importer->pool, importer->cur_guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (diff > 0) {
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* identical */
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* NOTE: assumes save_change is allocated from importer pool */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->newmails, &newmail, 1);
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainendsync_mailbox_try_save(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!importer_next_mail(importer, 0) && save_change == NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mailbox_try_save_cur(importer, save_change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void dsync_mailbox_save(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (!dsync_mailbox_try_save(importer, save_change)) ;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_import_set_mail(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!mail_set_uid(importer->mail, change->uid))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID is unknown */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* backend doesn't support GUIDs. if hdr_hash is set, we could
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen verify it, but since this message really is supposed to
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen match, it's probably too much trouble. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* verify that GUID matches, just in case */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_error(importer, importer->mail, "GUID");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("Mailbox %s: Unexpected GUID mismatch for "
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "UID=%u: %s != %s", mailbox_get_vname(importer->box),
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainenstatic bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer,
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen if (change->guid == NULL || *change->guid == '\0')
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen if (strcmp(importer->cur_guid, change->guid) != 0) {
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen i_error("Mailbox %s: Unexpected GUID mismatch for "
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen "UID=%u: %s != %s", mailbox_get_vname(importer->box),
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen change->uid, importer->cur_guid, change->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenmerge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove,
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *change_add_r, uint32_t *change_remove_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t combined_add, combined_remove, conflict_flags;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint32_t local_wanted, remote_wanted, conflict_pvt_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* resolve conflicts */
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_pvt_flags = conflict_flags & pvt_mask;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_flags = (local_remove & remote_add) & ~pvt_mask;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_pvt_flags = conflict_flags & pvt_mask;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert((combined_add & combined_remove) == 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* see if there are conflicting final flags */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_wanted = (local_final|combined_add) & ~combined_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_wanted = (remote_final|combined_add) & ~combined_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen conflict_flags = local_wanted ^ remote_wanted;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else if (prefer_remote && !prefer_pvt_remote) {
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen } else if (!prefer_remote && prefer_pvt_remote) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *change_remove_r = local_final & ~local_wanted;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenkeyword_find(ARRAY_TYPE(const_string) *keywords, const char *name,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int *idx_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *names;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void keywords_append(ARRAY_TYPE(const_string) *dest,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *namep;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < 32; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenmerge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const ARRAY_TYPE(const_string) *remote_changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* local_changes and remote_changes are assumed to have no
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen duplicates names */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *local_add, *local_remove, *local_final;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *remote_add, *remote_remove, *remote_final;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *changes, *name, *const *local_keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we'll assign a common index for each keyword name and place
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen the changes to separate bit arrays. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_size = str_array_length(local_keywords) + count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this message has no keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* @UNSAFE: create large enough arrays to fit all keyword indexes. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get remote changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen switch (changes[i][0]) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_add[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* fall through */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_final[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_remove[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get local changes. use existing indexes for names when they exist. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!keyword_find(&all_keywords, name, &name_idx)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen switch (changes[i][0]) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_add[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_remove[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!keyword_find(&all_keywords, name, &name_idx)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_final[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(array_count(&all_keywords) <= array_size*32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_size = (array_count(&all_keywords)+31) / 32;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < array_size; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_flags(local_final[i], local_add[i], local_remove[i],
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_final[i], remote_add[i], remote_remove[i],
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change_add[i] != 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keywords_append(&remove_keywords, &all_keywords,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* apply changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_keywords(mail, MODIFY_REMOVE, kw);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainendsync_mailbox_import_replace_flags(struct mail *mail,
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen unsigned int i, count;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen if (array_is_created(&change->keyword_changes))
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen changes = array_get(&change->keyword_changes, &count);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen for (i = 0; i < count; i++) {
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen switch (changes[i][0]) {
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen kw = mailbox_keywords_create_valid(mail->box, array_idx(&keywords, 0));
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen mail_update_keywords(mail, MODIFY_REPLACE, kw);
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen mail_update_pvt_modseq(mail, change->pvt_modseq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert((change->add_flags & change->remove_flags) == 0);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen /* dsync backup: just make the local look like remote. */
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen dsync_mailbox_import_replace_flags(mail, change);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen local_change = hash_table_lookup(importer->local_changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_keyword_changes = local_change->keyword_changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen else if (mail_get_modseq(mail) > change->modseq)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* identical modseq, we'll just have to pick one.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen Note that both brains need to pick the same one, otherwise
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen they become unsynced. */
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else if (mail_get_pvt_modseq(mail) > change->pvt_modseq)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge flags */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_flags(mail_get_flags(mail), local_add, local_remove,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->final_flags, change->add_flags, change->remove_flags,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_flags(mail, MODIFY_ADD, change_add);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_flags(mail, MODIFY_REMOVE, change_remove);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_keywords(mail, &local_keyword_changes, &change->keyword_changes,
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen mail_update_pvt_modseq(mail, change->pvt_modseq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_save(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid == importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we've already verified that the GUID matches.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen apply flag changes if there are any. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_flag_change(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save = p_new(importer->pool, struct dsync_mail_change, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_change_dup(importer->pool, change, save);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this is a new mail. its UID may or may not conflict with
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen an existing local mail, we'll figure it out later. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* the local mail is expunged. we'll decide later if we want
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen to save this mail locally or expunge it form remote. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid < importer->cur_mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->maybe_saves, &save, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* expunge the message, unless its GUID unexpectedly doesn't
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid <= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already expunged locally, we can ignore this.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid=last_common_uid if we managed to verify from
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen transaction log that the GUIDs match */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid >= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (change->uid == importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already verified that the GUID matches */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(importer->cur_mail->uid == change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we don't know yet if we should expunge this
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen message or not. queue it until we do. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&importer->maybe_expunge_uids, change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* If there are local mails after last_common_uid which we skipped
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while trying to match the next message, we need to now go back */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_mail->uid <= importer->last_common_uid+1)
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)mailbox_search_deinit(&importer->search_ctx);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int n, i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* expunge the messages whose expunge-decision we delayed previously */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (seq_range_array_iter_nth(&iter, n++, &uid)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we expunge messages only up to last_common_uid,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ignore the rest */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* handle pending saves */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen saves = array_get(&importer->maybe_saves, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (saves[i]->uid > importer->last_common_uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (*change->guid != '\0' && *importer->cur_guid != '\0') {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we have GUIDs, verify them */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return strcmp(change->guid, importer->cur_guid) == 0 ? 1 : 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* verify hdr_hash if it exists */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "sync with header hashes instead",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (dsync_mail_get_hdr_hash(importer->cur_mail, &hdr_hash) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_error(importer, importer->cur_mail, "hdr-stream");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return strcmp(change->hdr_hash, hdr_hash) == 0 ? 1 : 0;
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainendsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer,
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* remote doesn't support GUIDs, can't verify expunge */
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* local message is expunged. see if we can find its GUID from
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen transaction log and check if the GUIDs match. The GUID in
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen log is a 128bit GUID, so we may need to convert the remote's
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen GUID string to 128bit GUID first. */
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen local_change = hash_table_lookup(importer->local_changes,
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen if (local_change == NULL || local_change->guid == NULL)
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen if (guid_128_from_string(local_change->guid, guid_128) < 0)
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen mail_generate_guid_128_hash(change->guid, change_guid_128);
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) {
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* mismatch - found the first non-common UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* try to find the matching local mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!importer_next_mail(importer, change->uid)) {
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* no more local mails. we can still try to match
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen expunged mails though. */
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen /* mail doesn't exist remotely either, don't bother
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen looking it up locally. */
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen !dsync_mailbox_find_common_expunged_uid(importer, change)) {
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* couldn't match it for an expunged mail. use the last
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen message with a matching GUID as the last common
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we can't know if this UID matches */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we have a matching local UID. check GUID to see if it's
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen really the same mail or not */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = dsync_mailbox_import_match_msg(importer, change)) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* unknown */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* mismatch - found the first non-common UID */
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen dsync_mailbox_find_common_expunged_uid(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenvoid dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_find_common_uid(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* a) uid <= last_common_uid for flag changes and expunges.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen this happens only when last_common_uid was originally given
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen as parameter to importer.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen when we're finding the last_common_uid ourself,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid>last_common_uid always in here, because
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last_common_uid_found=TRUE only after we find the first
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen b) uid > last_common_uid for i) new messages, ii) expunges
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen that were sent "just in case" */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid <= importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* a) uid < last_common_uid can never happen */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid >= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* b) uid = last_common_uid if we've verified that the
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen messages' GUIDs match so far.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen c) uid > last_common_uid: i) TYPE_EXPUNGE change has
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen GUID=NULL, so we couldn't verify yet if it matches our
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local message, ii) local message is expunged and we couldn't
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen find its GUID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid > importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_expunge(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_flag_change(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_msg_update_uid(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = mailbox_save_alloc(importer->ext_trans);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_copy_flags(save_ctx, importer->mail);
3e6903afa5bd125e8bb14f996a558e032674dfbfTimo Sirainen if (mailbox_move(&save_ctx, importer->mail) == 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->wanted_uids, &new_uid, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *newmail, *const *newmailp;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen common_uid_next = I_MAX(importer->local_uid_next,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_foreach_modifiable(&importer->newmails, newmailp) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already assigned */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_set_uid(importer->mail, newmail->uid))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* figure out what UID to use for the mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* keep the UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (newmail->uid_in_local && newmail->uid != new_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* local UID changed, reassign it by copying */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_msg_update_uid(importer, newmail->uid, new_uid);
c45baa023828a14f440eb184c24d186ffe8666edTimo Sirainen } else if (linked_uid && newmail->link->uid_in_local) {
c45baa023828a14f440eb184c24d186ffe8666edTimo Sirainen /* the linked message already exists. we'll just need
c45baa023828a14f440eb184c24d186ffe8666edTimo Sirainen to forget about this message. */
c45baa023828a14f440eb184c24d186ffe8666edTimo Sirainen if (newmail->link != NULL && !newmail->skip) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* skip the linked mail */
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen importer->last_common_uid = common_uid_next-1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenvoid dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* handle pending expunges and flag updates */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* skip common local mails */
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)importer_next_mail(importer, importer->last_common_uid+1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* if there are any local mails left, add them to newmails list */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_assign_new_uids(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen requests = array_get(&importer->mail_requests, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return &requests[importer->mail_request_idx++];
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic const char *const *
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_get_final_keywords(const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!array_is_created(&change->keyword_changes))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(&change->keyword_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *keyword_names;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keyword_names = dsync_mailbox_get_final_keywords(change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_flags(save_ctx, change->final_flags, keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_save_date(save_ctx, change->save_timestamp);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_min_modseq(save_ctx, change->modseq);
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen /* FIXME: if there already are private flags, they get lost because
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen saving can't handle updating private index. they get added on the
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen next sync though. if this is fixed here, set min_pvt_modseq also. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_msg_try_copy(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (inst = all_newmails; inst != NULL; inst = inst->next) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (inst->uid_in_local && !inst->copy_failed &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_copy(save_ctx_p, importer->mail) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = mailbox_save_alloc(importer->ext_trans);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
89f0ab8c7daae308fbcb1c8537ee415d236816c0Timo Sirainen if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* try to save the mail by copying an existing mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails)) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->wanted_uids, &newmail->uid, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* fallback to saving from remote stream */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* it was just expunged in remote, skip it */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_save_begin(&save_ctx, mail->input) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("Can't save message to mailbox %s: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while ((ret = i_stream_read(mail->input)) > 0 || ret == -2) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("Can't save message to mailbox %s: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->wanted_uids, &newmail->uid, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenvoid dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
89f0ab8c7daae308fbcb1c8537ee415d236816c0Timo Sirainen i_assert(mail->input == NULL || mail->input->seekable);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_lookup(importer->import_guids, mail->guid) :
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("%s: Remote sent unwanted message body for "
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "GUID=%s UID=%u",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_remove(importer->import_guids, mail->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* save all instances of the message */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (; newmail != NULL; newmail = newmail->next) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* no need to do anything for this mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we already handled this by copying the mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_save_body(importer, mail, newmail,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenreassign_uids_in_seq_range(struct mailbox *box, uint32_t seq1, uint32_t seq2)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const enum mailbox_transaction_flags trans_flags =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen trans = mailbox_transaction_begin(box, trans_flags);
3e6903afa5bd125e8bb14f996a558e032674dfbfTimo Sirainen i_error("Couldn't move mail within mailbox %s: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("UID reassign commit failed to mailbox %s: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenreassign_unwanted_uids(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct mail_transaction_commit_changes *changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t seq1, seq2, lowest_saved_uid = (uint32_t)-1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, n, wanted_count;
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen /* wanted_uids contains the UIDs we tried to save mails with.
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen if nothing changed during dsync, we should have the expected UIDs
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen (changes->saved_uids) and all is well.
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen if any new messages got inserted during dsync, we'll need to fix up
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen the UIDs and let the next dsync fix up the other side. for example:
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen remote uids = 5,7,9 = wanted_uids
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen remote uidnext = 12
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen locally added new uid=5 ->
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen saved_uids = 10,7,9
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen we'll now need to reassign UIDs 5 and 10. or more generally, we
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen need to reassign UIDs [original local uidnext .. lowest saved_uid-1]
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen and [lowest unwanted uid .. remote uidnext-1] */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* find the highest wanted UID that doesn't match what we got */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen wanted_uids = array_get(&importer->wanted_uids, &wanted_count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_iter_init(&iter, &changes->saved_uids); i = n = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (seq_range_array_iter_nth(&iter, n++, &saved_uid)) {
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen i_assert(lowest_unwanted_uid == (uint32_t)-1 ||
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen lowest_unwanted_uid == highest_wanted_uid+1 ||
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen if (importer->local_uid_next != lowest_saved_uid &&
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen /* [original local uidnext .. lowest saved_uid-1] */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_seq_range(importer->box, importer->local_uid_next,
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen if (lowest_unwanted_uid < importer->remote_uid_next) {
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen /* [highest wanted_uid+1 .. remote uidnext-1] */
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen mailbox_get_seq_range(importer->box, lowest_unwanted_uid,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_transaction_commit_changes changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* commit saves */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_transaction_commit_get_changes(&importer->ext_trans,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("Save commit failed to mailbox %s: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_transaction_rollback(&importer->trans);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* commit flag changes and expunges */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_transaction_commit(&importer->trans) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* update mailbox metadata. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen update.min_next_uid = importer->remote_uid_next;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen update.min_highest_modseq = importer->remote_highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_update(importer->box, &update) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("Mailbox update failed to mailbox %s: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* sync mailbox to finish flag changes and expunges. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("Mailbox sync failed to mailbox %s: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (reassign_unwanted_uids(importer, &changes,
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainendsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer)
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen const char *key;
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen iter = hash_table_iterate_init(importer->import_guids);
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)",
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainendsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer)
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen iter = hash_table_iterate_init(importer->import_uids);
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) {
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen i_error("Mailbox %s: Remote didn't send mail UID=%u",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenint dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mailbox_importer *importer = *_importer;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_assign_new_uids(importer);
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen dsync_mailbox_import_check_missing_guid_imports(importer);
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen dsync_mailbox_import_check_missing_uid_imports(importer);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen if (mailbox_search_deinit(&importer->search_ctx) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (dsync_mailbox_import_commit(importer, changes_during_sync_r) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_is_created(&importer->mail_requests))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *last_common_uid_r = importer->last_common_uid;
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen *last_common_modseq_r = importer->remote_highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen *last_common_pvt_modseq_r = importer->remote_highest_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* local changes occurred during dsync. we exported changes up
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen to local_initial_highestmodseq, so all of the changes have
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen happened after it. we want the next run to see those changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen so return it as the last common modseq */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *last_common_modseq_r = importer->local_initial_highestmodseq;