bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* linked list of mails for this GUID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* if non-NULL, this mail exists in both local and remote. this link
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen points to the other side. */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* the final UID for the message */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* the original local UID, or 0 if exists only remotely */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* the original remote UID, or 0 if exists only remotely */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen /* UID for the mail in the virtual \All mailbox */
9eef11df882f9c14d164f42cb438f32fe724041cTimo Sirainen/* for quickly testing that two-way sync doesn't actually do any unexpected
9eef11df882f9c14d164f42cb438f32fe724041cTimo Sirainen modifications. */
9eef11df882f9c14d164f42cb438f32fe724041cTimo Sirainen#define IMPORTER_DEBUG_CHANGE(importer) /*i_assert(!importer->master_brain)*/
678d0463849ba777106eb7875f27db07a5d8e3dfTimo SirainenHASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *);
a75d470c9223a75801418fcdda258885c36317e0Timo SirainenHASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *);
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t last_common_modseq, last_common_pvt_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen enum mailbox_transaction_flags transaction_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mailbox_transaction_context *trans, *ext_trans;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mailbox_transaction_context *virtual_trans;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* UID => struct dsync_mail_change */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen HASH_TABLE_TYPE(dsync_attr_change) local_attr_changes;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct dsync_mail_change *) maybe_saves;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID => struct importer_new_mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* UID => struct importer_new_mail */
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct dsync_mail_request) mail_requests;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t prev_uid, next_local_seq, local_uid_next;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen unsigned int first_unsaved_idx, saves_since_commit;
589bc28b5489c76626f022adc74a5324ae02996aTimo Sirainenstatic const char *dsync_mail_change_type_names[] = {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainenstatic bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainenstatic int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer,
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainenimp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...)
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainendsync_import_unexpected_state(struct dsync_mailbox_importer *importer,
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen i_error("Mailbox %s: %s", mailbox_get_vname(importer->box),
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen i_warning("Mailbox %s doesn't match previous state: %s "
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen "(dsync must be run again without the state)",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&sarg->value.seqset, search_args->pool, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_search_init(importer->trans, search_args, NULL,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_search_next(importer->search_ctx, &importer->cur_mail))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->next_local_seq = importer->cur_mail->seq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this flag causes cur_guid to be looked up later */
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainendsync_mailbox_import_transaction_begin(struct dsync_mailbox_importer *importer)
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen const enum mailbox_transaction_flags ext_trans_flags =
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->trans = mailbox_transaction_begin(importer->box,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi "dsync import");
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->ext_trans = mailbox_transaction_begin(importer->box,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi "dsync ext import");
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->mail = mail_alloc(importer->trans, 0, NULL);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL);
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi const char *const *hashed_headers)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer = p_new(pool, struct dsync_mailbox_importer, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->last_common_modseq = last_common_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->last_common_pvt_modseq = last_common_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last_common_uid != 0 || last_common_modseq != 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->remote_first_recent_uid = remote_first_recent_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->remote_highest_modseq = remote_highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen importer->sync_since_timestamp = sync_since_timestamp;
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi importer->sync_until_timestamp = sync_until_timestamp;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen importer->stateful_import = importer->last_common_uid_found;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen importer->sync_flag = imap_parse_system_flag(sync_flag);
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen importer->sync_keyword = p_strdup(pool, sync_flag);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen importer->commit_msgs_interval = commit_msgs_interval;
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen importer->transaction_flags = MAILBOX_TRANSACTION_FLAG_SYNC;
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY) != 0)
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen importer->transaction_flags |= MAILBOX_TRANSACTION_FLAG_NO_NOTIFY;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create_direct(&importer->import_uids, pool, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_array_init(&importer->maybe_expunge_uids, 16);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen dsync_mailbox_import_transaction_begin(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0;
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0;
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0;
f3c24c2c92802cb773315eba1132254932d8709bTimo Sirainen importer->hdr_hash_version = hdr_hash_version;
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen mailbox_get_open_status(importer->box, STATUS_UIDNEXT |
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->local_initial_highestmodseq = status.highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq;
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen else if (importer->local_uid_next <= last_common_uid) {
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen "local UIDNEXT %u <= last common UID %u",
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen } else if (importer->local_initial_highestmodseq < last_common_modseq) {
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64,
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen } else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) {
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64,
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi importer->local_initial_highestpvtmodseq,
08e9fec5ba9e1a26e658c4224207d666b6ced27dTimo Sirainen importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan);
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan);
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainendsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer,
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen enum mail_attribute_type type, const char *key,
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen struct dsync_mailbox_attribute lookup_attr, *attr;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen const struct dsync_mailbox_attribute *attr_change;
66c87722e0fd2a85cd59797326bad3d1c409dc3aAki Tuomi if (mailbox_attribute_get_stream(importer->box, type, key, &value) < 0) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen i_error("Mailbox %s: Failed to get attribute %s: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr_change = hash_table_lookup(importer->local_attr_changes,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen value.value == NULL && value.value_stream == NULL) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* we have no knowledge of this attribute */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr = t_new(struct dsync_mailbox_attribute, 1);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_istreams_cmp(struct istream *input1, struct istream *input2, int *cmp_r)
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody (void)i_stream_read_more(input1, &data1, &size1);
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody (void)i_stream_read_more(input2, &data2, &size2);
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen i_error("read(%s) failed: %s", i_stream_get_name(input1),
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen i_error("read(%s) failed: %s", i_stream_get_name(input2),
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1,
ea918d503ae61496e10103935028d3cc6419f1cbTimo Sirainen i_assert(attr1->value_stream != NULL || attr1->value != NULL);
ea918d503ae61496e10103935028d3cc6419f1cbTimo Sirainen i_assert(attr2->value_stream != NULL || attr2->value != NULL);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr1->value != NULL && attr2->value != NULL) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* at least one of them is a stream. make both of them streams. */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen input1 = attr1->value_stream != NULL ? attr1->value_stream :
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_create_from_data(attr1->value, strlen(attr1->value));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen input2 = attr2->value_stream != NULL ? attr2->value_stream :
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_create_from_data(attr2->value, strlen(attr2->value));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen ret = dsync_istreams_cmp(input1, input2, cmp_r);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_attributes_cmp(const struct dsync_mailbox_attribute *attr,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen const struct dsync_mailbox_attribute *local_attr,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* remote has a value and local doesn't -> use it */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* remote doesn't have a value, bt local does -> skip */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return dsync_attributes_cmp_values(attr, local_attr, cmp_r);
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainendsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer,
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen const struct dsync_mailbox_attribute *local_attr,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* attribute doesn't exist on either side -> ignore */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* we haven't seen this locally -> use whatever remote has */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (local_attr->modseq <= importer->last_common_modseq &&
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->modseq > importer->last_common_modseq &&
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* we're doing incremental syncing, and we can see that the
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attribute was changed remotely, but not locally -> use it */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (local_attr->modseq > importer->last_common_modseq &&
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->modseq <= importer->last_common_modseq &&
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* we're doing incremental syncing, and we can see that the
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attribute was changed locally, but not remotely -> ignore */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (attr->last_change > local_attr->last_change) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* remote has a newer timestamp -> use it */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (attr->last_change < local_attr->last_change) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* remote has an older timestamp -> ignore */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* the timestamps are the same. now we're down to guessing
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen the right answer, unless the values are actually equal,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen so check that first. next try to use modseqs, but if even
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen they are the same, fallback to just picking one based on the
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* identical scripts */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* remote has a higher modseq -> use it */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (attr->modseq < local_attr->modseq) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* remote has an older modseq -> ignore */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen } else if (cmp < 0) {
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Value changed, but unknown which is newer - picking local";
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Value changed, but unknown which is newer - picking remote";
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen if (mailbox_attribute_set(importer->trans, attr->type,
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen i_error("Mailbox %s: Failed to set attribute %s: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box, NULL));
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen /* the attributes aren't vital, don't fail everything just
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen because of them. */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainenint dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen if (dsync_mailbox_import_lookup_attr(importer, attr->type,
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen ret = dsync_mailbox_import_attribute_real(importer, attr,
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen if (local_attr != NULL && local_attr->value_stream != NULL)
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen imp_debug(importer, "Import attribute %s: %s", attr->key,
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainenstatic void dsync_mail_error(struct dsync_mailbox_importer *importer,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi errstr = mailbox_get_last_internal_error(importer->box, &error);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mailbox_get_vname(mail->box), field, mail->uid, errstr);
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainendsync_mail_change_guid_equals(struct dsync_mailbox_importer *importer,
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen if (guid_128_from_string(change->guid, change_guid_128) < 0)
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen mail_generate_guid_128_hash(change->guid, change_guid_128);
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) {
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen *cmp_guid_r = t_strdup_printf("%s(guid128, orig=%s)",
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen binary_to_hex(change_guid_128, sizeof(change_guid_128)),
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainenimporter_try_next_mail(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* end of search */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (importer->cur_mail->seq < importer->next_local_seq ||
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this message exists locally, but remote didn't send
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen expunge-change for it. if the message's
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid <= last-common-uid, it should be deleted */
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&importer->maybe_expunge_uids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!mailbox_search_next(importer->search_ctx,
7be1a5530fcb414588fbe90eaed65eff83e84737Timo Sirainen importer->cur_uid_has_change = importer->cur_mail->uid == wanted_uid;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen dsync_mail_error(importer, importer->cur_mail, "GUID");
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen if (dsync_mail_get_hdr_hash(importer->cur_mail,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen dsync_mail_error(importer, importer->cur_mail,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen "header hash");
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen pmail = (struct mail_private *)importer->cur_mail;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen importer->cur_hdr_hash = p_strdup(pmail->pool, hdr_hash);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* make sure next_local_seq gets updated in case we came here
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen because of min_uid */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->next_local_seq = importer->cur_mail->seq;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainenimporter_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid)
3ef4aca8cae3e335e1fe53591049cb80b299c459Timo Sirainen ret = importer_try_next_mail(importer, wanted_uid);
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen importer->next_local_seq = importer->cur_mail->seq + 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenimporter_mail_cmp(const struct importer_mail *m1,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void newmail_link(struct dsync_mailbox_importer *importer,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen struct importer_new_mail *newmail, uint32_t remote_uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *first_mail, **last, *mail, *link = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen first_mail = hash_table_lookup(importer->import_guids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* first mail for this GUID */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* mail exists only locally. we don't want to request
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen it, and we'll assume it has no duplicate
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen instances. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen first_mail = hash_table_lookup(importer->import_uids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* first mail for this UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* 1) add the newmail to the end of the linked list
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen 2) find our link
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen FIXME: this loop is slow if the same GUID has a ton of instances.
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen Could it be improved in some way? */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (mail = first_mail; mail != NULL; mail = mail->next) {
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainendsync_mailbox_revert_existing_uid(struct dsync_mailbox_importer *importer,
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen /* UID either already exists or UIDNEXT is too high. we can't set the
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen wanted UID, so we'll need to delete the whole mailbox and resync */
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen i_warning("Deleting mailbox '%s': UID=%u already exists locally for a different mail: %s",
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen mailbox_get_vname(importer->box), uid, reason);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi if (importer->empty_hdr_workaround && !importer->mails_have_guids &&
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi importer->cur_mail != NULL && save_change != NULL &&
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi /* one of the headers is empty. assume it's broken and that
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi the header matches what we have currently. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* add a record for local mail */
8c8f6ed0a46db6589c08d8db1f4ed779ef5bc19bTimo Sirainen importer->cur_mail->uid >= importer->remote_uid_next) {
79490ec1a58241d6011fa36713ca651d795855c3Timo Sirainen dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid,
8c8f6ed0a46db6589c08d8db1f4ed779ef5bc19bTimo Sirainen t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next));
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->guid = p_strdup(importer->pool, importer->cur_guid);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->final_uid >= importer->remote_uid_next;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (diff > 0) {
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->final_uid >= importer->local_uid_next;
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen if (!newmail->uid_is_usable && importer->revert_local_changes) {
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen dsync_mailbox_revert_existing_uid(importer, newmail->final_uid,
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen t_strdup_printf("UID >= local UIDNEXT=%u", importer->local_uid_next));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* identical */
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* NOTE: assumes save_change is allocated from importer pool */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->newmails, &newmail, 1);
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainendsync_mailbox_try_save(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!importer_next_mail(importer, 0) && save_change == NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mailbox_try_save_cur(importer, save_change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void dsync_mailbox_save(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (!dsync_mailbox_try_save(importer, save_change)) ;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_import_set_mail(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!mail_set_uid(importer->mail, change->uid))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID is unknown */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* backend doesn't support GUIDs. if hdr_hash is set, we could
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen verify it, but since this message really is supposed to
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen match, it's probably too much trouble. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* verify that GUID matches, just in case */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_error(importer, importer->mail, "GUID");
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen if (!dsync_mail_change_guid_equals(importer, change, guid, &cmp_guid)) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen "Unexpected GUID mismatch for UID=%u: %s != %s",
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainenstatic bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer,
34b8b14ce06c0939932b60f22f61aea124198438Timo Sirainen if (change->guid == NULL || change->guid[0] == '\0' ||
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen if (!dsync_mail_change_guid_equals(importer, change,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen "Unexpected GUID mismatch (2) for UID=%u: %s != %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenmerge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove,
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote,
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen uint32_t *change_add_r, uint32_t *change_remove_r,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen bool *remote_changed, bool *remote_pvt_changed)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t combined_add, combined_remove, conflict_flags;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint32_t local_wanted, remote_wanted, conflict_pvt_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* resolve conflicts */
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_pvt_flags = conflict_flags & pvt_mask;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_pvt_flags = conflict_flags & pvt_mask;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert((combined_add & combined_remove) == 0);
cf848255bf65a5e2382c59c093da72f877f7535aTimo Sirainen /* don't change flags that are currently identical in both sides */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* see if there are conflicting final flags */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_wanted = (local_final|combined_add) & ~combined_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_wanted = (remote_final|combined_add) & ~combined_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen conflict_flags = local_wanted ^ remote_wanted;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else if (prefer_remote && !prefer_pvt_remote) {
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen } else if (!prefer_remote && prefer_pvt_remote) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *change_remove_r = local_final & ~local_wanted;
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen if ((local_wanted & ~pvt_mask) != (remote_final & ~pvt_mask))
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen if ((local_wanted & pvt_mask) != (remote_final & pvt_mask))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenkeyword_find(ARRAY_TYPE(const_string) *keywords, const char *name,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int *idx_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void keywords_append(ARRAY_TYPE(const_string) *dest,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < 32; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenmerge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const ARRAY_TYPE(const_string) *remote_changes,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen bool *remote_changed, bool *remote_pvt_changed)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* local_changes and remote_changes are assumed to have no
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen duplicates names */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *local_add, *local_remove, *local_final;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *remote_add, *remote_remove, *remote_final;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *changes, *name, *const *local_keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we'll assign a common index for each keyword name and place
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen the changes to separate bit arrays. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_size = str_array_length(local_keywords) + count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this message has no keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* @UNSAFE: create large enough arrays to fit all keyword indexes. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get remote changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen switch (changes[i][0]) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_add[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_remove[name_idx/32] |= 1U << (name_idx%32);
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen remote_final[name_idx/32] |= 1U << (name_idx%32);
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen remote_add[name_idx/32] |= 1U << (name_idx%32);
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen remote_final[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get local changes. use existing indexes for names when they exist. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!keyword_find(&all_keywords, name, &name_idx)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen switch (changes[i][0]) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_add[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_remove[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!keyword_find(&all_keywords, name, &name_idx)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_final[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(array_count(&all_keywords) <= array_size*32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_size = (array_count(&all_keywords)+31) / 32;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < array_size; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_flags(local_final[i], local_add[i], local_remove[i],
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_final[i], remote_add[i], remote_remove[i],
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change_add[i] != 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keywords_append(&remove_keywords, &all_keywords,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* apply changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_keywords(mail, MODIFY_REMOVE, kw);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainendsync_mailbox_import_replace_flags(struct mail *mail,
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen if (array_is_created(&change->keyword_changes))
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen changes = array_get(&change->keyword_changes, &count);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen for (i = 0; i < count; i++) {
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen switch (changes[i][0]) {
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen kw = mailbox_keywords_create_valid(mail->box, array_idx(&keywords, 0));
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen mail_update_keywords(mail, MODIFY_REPLACE, kw);
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen mail_update_pvt_modseq(mail, change->pvt_modseq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT;
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen bool remote_changed = FALSE, remote_pvt_changed = FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert((change->add_flags & change->remove_flags) == 0);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen /* dsync backup: just make the local look like remote. */
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen dsync_mailbox_import_replace_flags(mail, change);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen local_change = hash_table_lookup(importer->local_changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_keyword_changes = local_change->keyword_changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen else if (mail_get_modseq(mail) > change->modseq)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* identical modseq, we'll just have to pick one.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen Note that both brains need to pick the same one, otherwise
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen they become unsynced. */
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else if (mail_get_pvt_modseq(mail) > change->pvt_modseq)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge flags */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_flags(mail_get_flags(mail), local_add, local_remove,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->final_flags, change->add_flags, change->remove_flags,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_flags(mail, MODIFY_ADD, change_add);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_flags(mail, MODIFY_REMOVE, change_remove);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_keywords(mail, &local_keyword_changes, &change->keyword_changes,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen prefer_remote, &remote_changed, &remote_pvt_changed);
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen /* update modseqs. try to anticipate when we have to increase modseq
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen to get it closer to what remote has (although we can't guess it
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen exactly correctly) */
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen if (remote_changed && new_modseq <= importer->remote_highest_modseq)
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen new_modseq = importer->remote_highest_modseq+1;
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen if (remote_pvt_changed && new_modseq <= importer->remote_highest_pvt_modseq)
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen new_modseq = importer->remote_highest_pvt_modseq+1;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainendsync_mail_change_have_keyword(const struct dsync_mail_change *change,
845bac8afdbd5f521b8a6d4493f0581f3b1a2692Timo Sirainen if (!array_is_created(&change->keyword_changes))
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen array_foreach(&change->keyword_changes, strp) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen switch ((*strp)[0]) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainendsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer,
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (change->received_timestamp < importer->sync_since_timestamp) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen /* mail has too old timestamp - skip it */
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail with too old timestamp";
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi if (change->received_timestamp > importer->sync_until_timestamp) {
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi /* mail has too new timestamp - skip it */
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi *result_r = "Ignoring missing local mail with too new timestamp";
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen if (change->virtual_size < importer->sync_max_size) {
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen /* mail is too large - skip it */
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen *result_r = "Ignoring missing local mail with too large size";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen bool have_flag = (change->final_flags & importer->sync_flag) != 0;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (have_flag && importer->sync_flag_dontwant) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail that doesn't have wanted flags";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!have_flag && !importer->sync_flag_dontwant) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail that has unwanted flags";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen bool have_kw = dsync_mail_change_have_keyword(change, importer->sync_keyword);
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (have_kw && importer->sync_flag_dontwant) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail that doesn't have wanted keywords";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!have_kw && !importer->sync_flag_dontwant) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail that has unwanted keywords";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_save(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid == importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we've already verified that the GUID matches.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen apply flag changes if there are any. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_flag_change(importer, change);
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!dsync_mailbox_import_want_change(importer, change, &result))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save = p_new(importer->pool, struct dsync_mail_change, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_change_dup(importer->pool, change, save);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this is a new mail. its UID may or may not conflict with
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen an existing local mail, we'll figure it out later. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* the local mail is expunged. we'll decide later if we want
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen to save this mail locally or expunge it form remote. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->maybe_saves, &save, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* expunge the message, unless its GUID unexpectedly doesn't
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid <= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already expunged locally, we can ignore this.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid=last_common_uid if we managed to verify from
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen transaction log that the GUIDs match */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid >= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (change->uid == importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already verified that the GUID matches */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(importer->cur_mail->uid == change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we don't know yet if we should expunge this
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen message or not. queue it until we do. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&importer->maybe_expunge_uids, change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* If there are local mails after last_common_uid which we skipped
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while trying to match the next message, we need to now go back */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_mail->uid <= importer->last_common_uid+1)
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)mailbox_search_deinit(&importer->search_ctx);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer)
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen imap_write_seq_range(expunges, &importer->maybe_expunge_uids);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen imp_debug(importer, "Last common UID=%u. Delayed expunges=%s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* expunge the messages whose expunge-decision we delayed previously */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (seq_range_array_iter_nth(&iter, n++, &uid)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we expunge messages only up to last_common_uid,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ignore the rest */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* handle pending saves */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen saves = array_get(&importer->maybe_saves, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen if (saves[i]->uid > importer->last_common_uid) {
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen imp_debug(importer, "Delayed save UID=%u: Save",
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen imp_debug(importer, "Delayed save UID=%u: Ignore",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (*change->guid != '\0' && *importer->cur_guid != '\0') {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we have GUIDs, verify them */
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen if (dsync_mail_change_guid_equals(importer, change,
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = t_strdup_printf("GUIDs don't match (%s vs %s)",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* verify hdr_hash if it exists */
173fc9ed37637e4609b1ecc9415a9b92067eeeb2Timo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
173fc9ed37637e4609b1ecc9415a9b92067eeeb2Timo Sirainen /* the message was already expunged, so we don't know
173fc9ed37637e4609b1ecc9415a9b92067eeeb2Timo Sirainen its header. return "unknown". */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "sync with header hashes instead",
45af47783693b3ba2768c5ad34eeff68132382d0Timo Sirainen if (dsync_mail_get_hdr_hash(importer->cur_mail,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_error(importer, importer->cur_mail, "hdr-stream");
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen (dsync_mail_hdr_hash_is_empty(change->hdr_hash) ||
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen *result_r = "Empty headers found with workaround enabled - assuming a match";
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen } else if (strcmp(change->hdr_hash, hdr_hash) == 0) {
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = t_strdup_printf("Headers hashes don't match (%s vs %s)",
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainendsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer,
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* remote doesn't support GUIDs, can't verify expunge */
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "GUIDs not supported, can't verify expunge";
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* local message is expunged. see if we can find its GUID from
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen transaction log and check if the GUIDs match. The GUID in
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen log is a 128bit GUID, so we may need to convert the remote's
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen GUID string to 128bit GUID first. */
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen local_change = hash_table_lookup(importer->local_changes,
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen if (local_change == NULL || local_change->guid == NULL) {
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Expunged local mail's GUID not found";
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen if (dsync_mail_change_guid_equals(importer, local_change,
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Expunged local mail's GUID matches remote";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen } else if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Expunged local mail's GUID doesn't match remote GUID";
d097ad375f7fc532ab5cb91020c206c0def55179Timo Sirainen /* GUID mismatch for two expunged mails. dsync can't update
d097ad375f7fc532ab5cb91020c206c0def55179Timo Sirainen GUIDs for already expunged messages, so we can't immediately
d097ad375f7fc532ab5cb91020c206c0def55179Timo Sirainen determine that the rest of the messages are a mismatch. so
d097ad375f7fc532ab5cb91020c206c0def55179Timo Sirainen for now we'll just skip over this pair. */
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Expunged mails' GUIDs don't match - delaying decision";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen /* NOTE: the return value here doesn't matter, because the only
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen caller that checks for it never reaches this code path */
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainendsync_mailbox_revert_missing(struct dsync_mailbox_importer *importer,
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen /* mail exists on remote, but not locally. we'll need to
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen insert this mail back, which means deleting the whole
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen mailbox and resyncing. */
f502a549abbbb69c606243a8800cbe7d05f772f1Timo Sirainen i_warning("Deleting mailbox '%s': UID=%u GUID=%s is missing locally",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen (change->virtual_size != (uoff_t)-1 || importer->sync_max_size == 0)));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* try to find the matching local mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!importer_next_mail(importer, change->uid)) {
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* no more local mails. we can still try to match
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen expunged mails though. */
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen /* mail doesn't exist remotely either, don't bother
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen looking it up locally. */
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen *result_r = "Expunged mail not found locally";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!dsync_mailbox_import_want_change(importer, change, result_r))
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen else if (importer->local_uid_next <= change->uid) {
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Mail's UID is above local UIDNEXT";
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen dsync_mailbox_revert_missing(importer, change);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen *result_r = "Reverting local change by deleting mailbox";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen } else if (!dsync_mailbox_find_common_expunged_uid(importer, change, result_r)) {
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen /* it's unknown if this mail existed locally and was
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen expunged. since we don't want to lose any mails,
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen assume that we need to preserve the mail. use the
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen last message with a matching GUID as the last common
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = t_strdup_printf("%s - No more local mails found", *result_r);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we can't know if this UID matches */
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen *result_r = "Expunged mail has no GUID, can't verify it";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we have a matching local UID. check GUID to see if it's
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen really the same mail or not */
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen if ((ret = dsync_mailbox_import_match_msg(importer, change, result_r)) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* unknown */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* mismatch - found the first non-common UID */
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen /* mismatch and we want to revert local changes -
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen need to delete the mailbox. */
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen dsync_mailbox_revert_existing_uid(importer, change->uid, *result_r);
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen /* mail exists remotely, but doesn't exist locally. */
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!dsync_mailbox_import_want_change(importer, change, result_r))
f6a50ab94799b632c6fc95ec73bfed399ead4e19Timo Sirainen change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen dsync_mailbox_revert_missing(importer, change);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen *result_r = "Reverting local change by deleting mailbox";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r);
bd78cb1d7e05956b08a2031b8618b8fb97c9991cTimo Sirainen *result_r = t_strdup_printf("%s (next local mail UID=%u)",
37312679c89109e8762c3ec33da3dfac4c4177b8Timo Sirainen *result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainenint dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen dsync_mailbox_find_common_uid(importer, change, &result);
589bc28b5489c76626f022adc74a5324ae02996aTimo Sirainen imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s",
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen change->guid != NULL ? change->guid : "<unknown>", change->uid,
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen change->hdr_hash != NULL ? change->hdr_hash : "", result);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* a) uid <= last_common_uid for flag changes and expunges.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen this happens only when last_common_uid was originally given
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen as parameter to importer.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen when we're finding the last_common_uid ourself,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid>last_common_uid always in here, because
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last_common_uid_found=TRUE only after we find the first
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen b) uid > last_common_uid for i) new messages, ii) expunges
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen that were sent "just in case" */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid <= importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* a) uid < last_common_uid can never happen */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid >= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* b) uid = last_common_uid if we've verified that the
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen messages' GUIDs match so far.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen c) uid > last_common_uid: i) TYPE_EXPUNGE change has
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen GUID=NULL, so we couldn't verify yet if it matches our
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local message, ii) local message is expunged and we couldn't
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen find its GUID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid > importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_expunge(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_flag_change(importer, change);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainenimporter_new_mail_final_uid_cmp(struct importer_new_mail *const *newmail1,
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if ((*newmail1)->final_uid < (*newmail2)->final_uid)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if ((*newmail1)->final_uid > (*newmail2)->final_uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *newmail, *const *newmailp;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen common_uid_next = I_MAX(importer->local_uid_next,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_foreach_modifiable(&importer->newmails, newmailp) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already assigned */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* figure out what UID to use for the mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* keep the UID */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* we can use the linked message's UID and expunge
da215fe5cde683b7d1fe2339e7747c1b14bff04aTimo Sirainen imp_debug(importer, "UID %u isn't usable, assigning new UID %u",
fadb1a39a4d7da6592b4cc11ae401d2763ba66b4Timo Sirainen if (newmail->link != NULL && newmail->link != newmail) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* skip processing the linked mail */
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen importer->last_common_uid = common_uid_next-1;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen /* Sort the newmails by their final_uid. This is used for tracking
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen whether an intermediate commit is allowed. */
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen array_sort(&importer->newmails, importer_new_mail_final_uid_cmp);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainendsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mail *mail, uint32_t uid, const char *guid,
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen /* NOTE: Errors are logged, but they don't cause the entire import
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mail_fill(mail, TRUE, dmail_r, &error_field) < 0) {
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi errstr = mailbox_get_last_internal_error(mail->box, &error);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (*guid != '\0' && strcmp(guid, dmail_r->guid) != 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen "Unexpected GUID mismatch (3) for UID=%u: %s != %s",
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainendsync_mailbox_import_saved_uid(struct dsync_mailbox_importer *importer,
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen array_append(&importer->wanted_uids, &uid, 1);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainendsync_mailbox_import_update_first_saved(struct dsync_mailbox_importer *importer)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen newmails = array_get(&importer->newmails, &count);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if (!newmails[importer->first_unsaved_idx]->saved)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainendsync_mailbox_import_saved_newmail(struct dsync_mailbox_importer *importer,
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen dsync_mailbox_import_saved_uid(importer, newmail->final_uid);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen dsync_mailbox_import_update_first_saved(importer);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen /* we can commit only if all the upcoming mails will have UIDs that
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen are larger than we're committing.
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen Note that if any existing UIDs have been changed, the new UID is
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen usually higher than anything that is being saved so we can't do
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen an intermediate commit. It's too much extra work to try to handle
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen that situation. So here this never happens, because then
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen array_count(wanted_uids) is always higher than first_unsaved_idx. */
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if (importer->saves_since_commit >= importer->commit_msgs_interval &&
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen importer->first_unsaved_idx == array_count(&importer->wanted_uids)) {
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if (dsync_mailbox_import_commit(importer, FALSE) < 0)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainendsync_msg_change_uid(struct dsync_mailbox_importer *importer,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen save_ctx = mailbox_save_alloc(importer->ext_trans);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen mailbox_save_copy_flags(save_ctx, importer->mail);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (mailbox_move(&save_ctx, importer->mail) < 0)
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen dsync_mailbox_import_saved_uid(importer, new_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainendsync_mailbox_import_change_uid(struct dsync_mailbox_importer *importer,
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen /* optimize by first trying to use the latest UID */
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (dsync_msg_change_uid(importer, range[count-1].seq2, wanted_uid)) {
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen seq_range_array_remove(unwanted_uids, range[count-1].seq2);
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen seq_range_array_remove(unwanted_uids, range[count-1].seq2);
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen /* now try to use any of them by iterating through them. (would be
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen easier&faster to just iterate backwards, but probably too much
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen trouble to add such API) */
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen n = 0; seq_range_array_iter_init(&iter, unwanted_uids);
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen while (seq_range_array_iter_nth(&iter, n++, &uid)) {
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (dsync_msg_change_uid(importer, uid, wanted_uid)) {
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainendsync_mailbox_import_try_local(struct dsync_mailbox_importer *importer,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen ARRAY_TYPE(seq_range) assigned_uids, unwanted_uids;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen struct seq_range_iter local_iter, wanted_iter;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_iter_init(&local_iter, local_uids);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_iter_init(&wanted_iter, wanted_uids);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* wanted_uids contains UIDs that need to exist at the end. those that
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen don't already exist in local_uids have a higher UID than any
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen existing local UID */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen t_array_init(&assigned_uids, array_count(wanted_uids));
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (seq_range_array_iter_nth(&wanted_iter, wanted_n,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* we have exactly the UID we want. keep it. */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&assigned_uids, wanted_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* we no longer want this local UID. */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&unwanted_uids, local_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* reuse as many existing messages as possible by changing their UIDs */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen while (seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!dsync_mailbox_import_change_uid(importer, &unwanted_uids,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&assigned_uids, wanted_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* expunge all unwanted messages */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen local_n = 0; seq_range_array_iter_init(&local_iter, &unwanted_uids);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* mark mails whose UIDs we got to be skipped over later */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen for (mail = all_newmails; mail != NULL; mail = mail->next) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_exists(&assigned_uids, mail->final_uid))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* we've assigned all wanted UIDs */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* try to find one existing message that we can use to copy to the
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen other instances */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen local_n = 0; seq_range_array_iter_init(&local_iter, local_uids);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mailbox_import_local_uid(importer, importer->mail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mailbox_save_newmails(importer, &dmail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainendsync_mailbox_import_try_virtual_all(struct dsync_mailbox_importer *importer,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mailbox_import_local_uid(importer, importer->virtual_mail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mailbox_save_newmails(importer, &dmail,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainendsync_mailbox_import_handle_mail(struct dsync_mailbox_importer *importer,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen ARRAY_TYPE(seq_range) local_uids, wanted_uids;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* get the list of the current local UIDs and the wanted UIDs.
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen find the first remote instance that we can request in case there are
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen no local instances */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen for (mail = all_newmails; mail != NULL; mail = mail->next) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&local_uids, mail->local_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&wanted_uids, mail->final_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!dsync_mailbox_import_try_local(importer, all_newmails,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen !dsync_mailbox_import_try_virtual_all(importer, all_newmails)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* no local instance. request from remote */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen request = array_append_space(&importer->mail_requests);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* successfully handled all the mails locally */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainendsync_mailbox_import_find_virtual_uids(struct dsync_mailbox_importer *importer)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (mailbox_sync(importer->virtual_all_box, 0) < 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen i_error("Couldn't sync \\All mailbox '%s': %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen mailbox_transaction_begin(importer->virtual_all_box,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen search_ctx = mailbox_search_init(importer->virtual_trans, search_args,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen while (mailbox_search_next(search_ctx, &mail)) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen /* ignore errors */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen newmail = hash_table_lookup(importer->import_guids, guid);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (newmail != NULL && newmail->virtual_all_uid == 0)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen i_error("Couldn't search \\All mailbox '%s': %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen importer->virtual_mail = mail_alloc(importer->virtual_trans, 0, NULL);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainendsync_mailbox_import_handle_local_mails(struct dsync_mailbox_importer *importer)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen hash_table_count(importer->import_guids) > 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen /* find UIDs in \All mailbox for all wanted GUIDs. */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen dsync_mailbox_import_find_virtual_uids(importer);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen iter = hash_table_iterate_init(importer->import_guids);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (dsync_mailbox_import_handle_mail(importer, mail))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen hash_table_remove(importer->import_guids, key);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen iter = hash_table_iterate_init(importer->import_uids);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen while (hash_table_iterate(iter, importer->import_uids, &key2, &mail)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (dsync_mailbox_import_handle_mail(importer, mail))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen hash_table_remove(importer->import_uids, key2);
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainenint dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* handle pending expunges and flag updates */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* skip common local mails */
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)importer_next_mail(importer, importer->last_common_uid+1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* if there are any local mails left, add them to newmails list */
faca9429a68ada074390da8745cf5ee0c958b42bTimo Sirainen while (importer->cur_mail != NULL && !importer->failed)
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (mailbox_search_deinit(&importer->search_ctx) < 0) {
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen importer->import_count = hash_table_count(importer->import_guids) +
933e663cc1805e11d42d4c5ad69038c0866c56a8Timo Sirainen dsync_mailbox_import_assign_new_uids(importer);
933e663cc1805e11d42d4c5ad69038c0866c56a8Timo Sirainen /* save mails from local sources where possible,
933e663cc1805e11d42d4c5ad69038c0866c56a8Timo Sirainen request the rest from remote */
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen dsync_mailbox_import_handle_local_mails(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen requests = array_get(&importer->mail_requests, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return &requests[importer->mail_request_idx++];
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic const char *const *
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_get_final_keywords(const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!array_is_created(&change->keyword_changes))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(&change->keyword_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen changes[i][0] == KEYWORD_CHANGE_ADD_AND_FINAL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keyword_names = dsync_mailbox_get_final_keywords(change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_flags(save_ctx, change->final_flags, keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_min_modseq(save_ctx, change->modseq);
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen /* FIXME: if there already are private flags, they get lost because
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen saving can't handle updating private index. they get added on the
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen next sync though. if this is fixed here, set min_pvt_modseq also. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_msg_try_copy(struct dsync_mailbox_importer *importer,
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen struct importer_new_mail **all_newmails_forcopy)
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen for (inst = *all_newmails_forcopy; inst != NULL; inst = inst->next) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (inst->uid_in_local && !inst->copy_failed &&
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mail_set_uid(importer->mail, inst->local_uid)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_copy(save_ctx_p, importer->mail) < 0) {
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen errstr = mailbox_get_last_internal_error(importer->box, &error);
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen "%s - falling back to other means",
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainendsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx,
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = mailbox_save_alloc(importer->ext_trans);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mailbox_save_set_uid(save_ctx, newmail->final_uid);
8bc87e22fecd20797112b86778a961b08dc1f5c8Timo Sirainen mailbox_save_set_save_date(save_ctx, mail->saved_date);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen dsync_mailbox_save_set_nonminimal(save_ctx, mail);
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainendsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct importer_new_mail **all_newmails_forcopy,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* try to save the mail by copying an existing mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails_forcopy)) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen /* copy using the source mail */
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen i_assert(mail->input_mail->uid == mail->input_mail_uid);
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen if (mailbox_copy(&save_ctx, mail->input_mail) == 0)
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen errstr = mailbox_get_last_internal_error(importer->box, &error);
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen i_warning("Failed to copy source UID=%u mail: "
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen "%s - falling back to regular saving",
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen dsync_mailbox_import_saved_newmail(importer, newmail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* fallback to saving from remote stream */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen /* the mail isn't remote yet. we were just trying to copy a
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen local mail to avoid downloading the remote mail. */
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2,
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen i_error("Mailbox %s: Failed to read mail %s uid=%u: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen dsync_mailbox_save_set_nonminimal(save_ctx, &mail2);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* it was just expunged in remote, skip it */
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (mailbox_save_begin(&save_ctx, input) < 0) {
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen while ((ret = i_stream_read(input)) > 0 || ret == -2) {
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen i_error("Mailbox %s: read(msg input) failed: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen dsync_mailbox_import_saved_newmail(importer, newmail);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainenstatic bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen struct importer_new_mail *newmail, *all_newmails_forcopy;
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen /* if all_newmails list is large, avoid scanning through the
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen uninteresting ones for each newmail */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* save all instances of the message */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen for (newmail = all_newmails; newmail != NULL && ret; newmail = newmail->next) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (!dsync_mailbox_save_body(importer, mail, newmail,
7ae6552482fe6e6a613f883335fdafdf9afbc7b7Timo Sirainenint dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
89f0ab8c7daae308fbcb1c8537ee415d236816c0Timo Sirainen i_assert(mail->input == NULL || mail->input->seekable);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Import mail body for GUID=%s UID=%u",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_lookup(importer->import_guids, mail->guid) :
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid));
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Remote sent unwanted message body for "
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "GUID=%s UID=%u",
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Skip unwanted mail body for "
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_remove(importer->import_guids, mail->guid);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (!dsync_mailbox_save_newmails(importer, mail, all_newmails, TRUE))
ec047a9c54a02338e85fb1767120b0923f6d4148Timo Sirainenreassign_uids_in_seq_range(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const enum mailbox_transaction_flags trans_flags =
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Reassign UIDs: %s", str_c(str));
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen arg = mail_search_build_add(search_args, SEARCH_UIDSET);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen p_array_init(&arg->value.seqset, search_args->pool,
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen array_append_array(&arg->value.seqset, unwanted_uids);
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi trans = mailbox_transaction_begin(box, trans_flags, __func__);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen while (mailbox_search_next(search_ctx, &mail)) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Couldn't move mail within mailbox: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(box, &importer->mail_error));
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen } else if (ret > 0) {
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(box, &importer->mail_error));
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: UID reassign commit failed: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(box, &importer->mail_error));
311cbd9b1c81df050f5019600a2a809bc620be00Timo Sirainen imp_debug(importer, "Mailbox %s: Change during sync: "
311cbd9b1c81df050f5019600a2a809bc620be00Timo Sirainen "Renumbered %u of %u unwanted UIDs",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenreassign_unwanted_uids(struct dsync_mailbox_importer *importer,
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen wanted_uids = array_get(&importer->wanted_uids, &wanted_count);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen saved_uids = array_get(&importer->saved_uids, &saved_count);
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen /* wanted_uids contains the UIDs we tried to save mails with.
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen if nothing changed during dsync, we should have the expected UIDs
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen (saved_uids) and all is well.
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen if any new messages got inserted during dsync, we'll need to fix up
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen the UIDs and let the next dsync fix up the other side. for example:
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen remote uids = 5,7,9 = wanted_uids
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen remote uidnext = 12
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen locally added new uid=5 ->
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen saved_uids = 10,7,9
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen we'll now need to reassign UIDs 5 and 10. to be fully future-proof
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen we'll reassign all UIDs between [original local uidnext .. highest
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen UID we think we know] that aren't in saved_uids. */
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen /* create uidset for the list of UIDs we don't want to exist */
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen highest_seen_uid = I_MAX(importer->remote_uid_next-1,
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen i_assert(importer->local_uid_next <= highest_seen_uid);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen for (i = 0; i < wanted_count; i++) {
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen seq_range_array_remove(&unwanted_uids, saved_uids[i]);
ec047a9c54a02338e85fb1767120b0923f6d4148Timo Sirainen ret = reassign_uids_in_seq_range(importer, &unwanted_uids);
4f7951e71128c120d8a502d6406cc603fcc8eb0bTimo Sirainen "%u UIDs changed due to UID conflicts",
264c6bd26df5380dc52687f6ec0351354012a948Timo Sirainen /* conflicting changes during sync, revert our last-common-uid
264c6bd26df5380dc52687f6ec0351354012a948Timo Sirainen back to a safe value. */
264c6bd26df5380dc52687f6ec0351354012a948Timo Sirainen importer->last_common_uid = importer->local_uid_next - 1;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainendsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, bool final)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_transaction_commit_changes changes;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen unsigned int n;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* commit saves */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_transaction_commit_get_changes(&importer->ext_trans,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box, &importer->mail_error));
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* removed wanted_uids that weren't actually saved */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_transaction_rollback(&importer->trans);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* remember the UIDs that were successfully saved */
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imap_write_seq_range(str, &changes.saved_uids);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Saved UIDs: %s", str_c(str));
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen seq_range_array_iter_init(&iter, &changes.saved_uids); n = 0;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen while (seq_range_array_iter_nth(&iter, n++, &uid))
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* commit flag changes and expunges */
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (mailbox_transaction_commit(&importer->trans) < 0) {
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen dsync_mailbox_import_transaction_begin(importer);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainenstatic int dsync_mailbox_import_finish(struct dsync_mailbox_importer *importer,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen ret = dsync_mailbox_import_commit(importer, TRUE);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* update mailbox metadata if we successfully saved
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen everything. */
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen update.min_next_uid = importer->remote_uid_next;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen update.min_highest_modseq = importer->remote_highest_modseq;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq;
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Finish update: min_next_uid=%u "
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi "min_first_recent_uid=%u min_highest_modseq=%"PRIu64" "
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen update.min_next_uid, update.min_first_recent_uid,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (mailbox_update(importer->box, &update) < 0) {
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* sync mailbox to finish flag changes and expunges. */
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* give new UIDs to messages that got saved with unwanted UIDs.
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen do it only if the whole transaction succeeded. */
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (reassign_unwanted_uids(importer, changes_during_sync_r) < 0)
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainendsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer)
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen iter = hash_table_iterate_init(importer->import_guids);
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)",
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainendsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer)
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen iter = hash_table_iterate_init(importer->import_uids);
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) {
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen i_error("Mailbox %s: Remote didn't send mail UID=%u",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenint dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mailbox_importer *importer = *_importer;
805d7834412465268486c50711962407ad13fbf6Timo Sirainen *require_full_resync_r = importer->require_full_resync;
805d7834412465268486c50711962407ad13fbf6Timo Sirainen if ((!success || importer->require_full_resync) && !importer->failed) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (!importer->new_uids_assigned && !importer->failed)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_assign_new_uids(importer);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_mailbox_import_check_missing_guid_imports(importer);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_mailbox_import_check_missing_uid_imports(importer);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (mailbox_search_deinit(&importer->search_ctx) < 0) {
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (dsync_mailbox_import_finish(importer, changes_during_sync_r) < 0)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen (void)mailbox_transaction_commit(&importer->virtual_trans);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_is_created(&importer->mail_requests))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *last_common_uid_r = importer->last_common_uid;
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen *last_common_modseq_r = importer->remote_highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen *last_common_pvt_modseq_r = importer->remote_highest_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* local changes occurred during dsync. we exported changes up
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen to local_initial_highestmodseq, so all of the changes have
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen happened after it. we want the next run to see those changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen so return it as the last common modseq */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *last_common_modseq_r = importer->local_initial_highestmodseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen *last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq;
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status);
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen i_assert(importer->failed == (importer->mail_error != 0));
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainenconst char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer)