bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "lib.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "array.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "hash.h"
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen#include "str.h"
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen#include "hex-binary.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "istream.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "seq-range-array.h"
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen#include "imap-util.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "mail-storage-private.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "mail-search-build.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "dsync-transaction-log-scan.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "dsync-mail.h"
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen#include "dsync-mailbox.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "dsync-mailbox-import.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstruct importer_mail {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *guid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen};
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstruct importer_new_mail {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* linked list of mails for this GUID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *next;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* if non-NULL, this mail exists in both local and remote. this link
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen points to the other side. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *link;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *guid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* the final UID for the message */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen uint32_t final_uid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* the original local UID, or 0 if exists only remotely */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen uint32_t local_uid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* the original remote UID, or 0 if exists only remotely */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen uint32_t remote_uid;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen /* UID for the mail in the virtual \All mailbox */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen uint32_t virtual_all_uid;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool uid_in_local:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool uid_is_usable:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool skip:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool expunged:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool copy_failed:1;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen bool saved:1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen};
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
9eef11df882f9c14d164f42cb438f32fe724041cTimo Sirainen/* for quickly testing that two-way sync doesn't actually do any unexpected
9eef11df882f9c14d164f42cb438f32fe724041cTimo Sirainen modifications. */
9eef11df882f9c14d164f42cb438f32fe724041cTimo Sirainen#define IMPORTER_DEBUG_CHANGE(importer) /*i_assert(!importer->master_brain)*/
9eef11df882f9c14d164f42cb438f32fe724041cTimo Sirainen
678d0463849ba777106eb7875f27db07a5d8e3dfTimo SirainenHASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *);
a75d470c9223a75801418fcdda258885c36317e0Timo SirainenHASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstruct dsync_mailbox_importer {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool_t pool;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mailbox *box;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t last_common_uid;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t last_common_modseq, last_common_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t remote_uid_next;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t remote_first_recent_uid;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen time_t sync_since_timestamp;
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi time_t sync_until_timestamp;
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen uoff_t sync_max_size;
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen enum mailbox_transaction_flags transaction_flags;
45af47783693b3ba2768c5ad34eeff68132382d0Timo Sirainen unsigned int hdr_hash_version;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen unsigned int commit_msgs_interval;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi const char *const *hashed_headers;
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen enum mail_flags sync_flag;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen const char *sync_keyword;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen bool sync_flag_dontwant;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mailbox_transaction_context *trans, *ext_trans;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_context *search_ctx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail, *ext_mail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mailbox *virtual_all_box;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mailbox_transaction_context *virtual_trans;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mail *virtual_mail;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *cur_mail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *cur_guid;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen const char *cur_hdr_hash;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* UID => struct dsync_mail_change */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen HASH_TABLE_TYPE(dsync_attr_change) local_attr_changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(seq_range) maybe_expunge_uids;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct dsync_mail_change *) maybe_saves;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID => struct importer_new_mail */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE_TYPE(guid_new_mail) import_guids;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* UID => struct importer_new_mail */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE_TYPE(uid_new_mail) import_uids;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct importer_new_mail *) newmails;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(uint32_t) wanted_uids;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen ARRAY_TYPE(uint32_t) saved_uids;
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen uint32_t highest_wanted_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct dsync_mail_request) mail_requests;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int mail_request_idx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t prev_uid, next_local_seq, local_uid_next;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq;
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen unsigned int import_pos, import_count;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen unsigned int first_unsaved_idx, saves_since_commit;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen enum mail_error mail_error;
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool failed:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool require_full_resync:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool debug:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool stateful_import:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool last_common_uid_found:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool cur_uid_has_change:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool cur_mail_skip:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool local_expunged_guids_set:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool new_uids_assigned:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool want_mail_requests:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool master_brain:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool revert_local_changes:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool mails_have_guids:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool mails_use_guid128:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool delete_mailbox:1;
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi bool empty_hdr_workaround:1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen};
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
589bc28b5489c76626f022adc74a5324ae02996aTimo Sirainenstatic const char *dsync_mail_change_type_names[] = {
589bc28b5489c76626f022adc74a5324ae02996aTimo Sirainen "save", "expunge", "flag-change"
589bc28b5489c76626f022adc74a5324ae02996aTimo Sirainen};
589bc28b5489c76626f022adc74a5324ae02996aTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainenstatic bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen const struct dsync_mail *mail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct importer_new_mail *all_newmails,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen bool remote_mail);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainenstatic int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen bool final);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
cd8a262bdd0b55a3bf53d9aa54fc3008a61d7f8dTimo Sirainenstatic void ATTR_FORMAT(2, 3)
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainenimp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...)
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen{
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen va_list args;
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen if (importer->debug) T_BEGIN {
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen va_start(args, fmt);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen i_debug("brain %c: Import %s: %s",
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen importer->master_brain ? 'M' : 'S',
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen mailbox_get_vname(importer->box),
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen t_strdup_vprintf(fmt, args));
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen va_end(args);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen } T_END;
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen}
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainenstatic void
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainendsync_import_unexpected_state(struct dsync_mailbox_importer *importer,
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen const char *error)
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen{
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen if (!importer->stateful_import) {
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen i_error("Mailbox %s: %s", mailbox_get_vname(importer->box),
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen error);
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen } else {
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen i_warning("Mailbox %s doesn't match previous state: %s "
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen "(dsync must be run again without the state)",
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen mailbox_get_vname(importer->box), error);
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen }
805d7834412465268486c50711962407ad13fbf6Timo Sirainen importer->require_full_resync = TRUE;
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen}
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_args *search_args;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_arg *sarg;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_args = mail_search_build_init();
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&sarg->value.seqset, search_args->pool, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->last_common_uid+1, (uint32_t)-1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->search_ctx =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_search_init(importer->trans, search_args, NULL,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen 0, NULL);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_search_args_unref(&search_args);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_search_next(importer->search_ctx, &importer->cur_mail))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->next_local_seq = importer->cur_mail->seq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this flag causes cur_guid to be looked up later */
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen importer->cur_mail_skip = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainenstatic void
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainendsync_mailbox_import_transaction_begin(struct dsync_mailbox_importer *importer)
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen{
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen const enum mailbox_transaction_flags ext_trans_flags =
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen importer->transaction_flags |
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen MAILBOX_TRANSACTION_FLAG_EXTERNAL |
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->trans = mailbox_transaction_begin(importer->box,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi importer->transaction_flags,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi "dsync import");
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->ext_trans = mailbox_transaction_begin(importer->box,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi ext_trans_flags,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi "dsync ext import");
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->mail = mail_alloc(importer->trans, 0, NULL);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen}
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstruct dsync_mailbox_importer *
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_init(struct mailbox *box,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mailbox *virtual_all_box,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_transaction_log_scan *log_scan,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t last_common_uid,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint64_t last_common_modseq,
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t last_common_pvt_modseq,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t remote_uid_next,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t remote_first_recent_uid,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint64_t remote_highest_modseq,
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t remote_highest_pvt_modseq,
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi time_t sync_since_timestamp,
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi time_t sync_until_timestamp,
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi uoff_t sync_max_size,
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen const char *sync_flag,
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen unsigned int commit_msgs_interval,
f3c24c2c92802cb773315eba1132254932d8709bTimo Sirainen enum dsync_mailbox_import_flags flags,
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi unsigned int hdr_hash_version,
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi const char *const *hashed_headers)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mailbox_importer *importer;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mailbox_status status;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool_t pool;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen 10240);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer = p_new(pool, struct dsync_mailbox_importer, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->pool = pool;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->box = box;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen importer->virtual_all_box = virtual_all_box;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->last_common_uid = last_common_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->last_common_modseq = last_common_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->last_common_pvt_modseq = last_common_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->last_common_uid_found =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last_common_uid != 0 || last_common_modseq != 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->remote_uid_next = remote_uid_next;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->remote_first_recent_uid = remote_first_recent_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->remote_highest_modseq = remote_highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen importer->sync_since_timestamp = sync_since_timestamp;
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi importer->sync_until_timestamp = sync_until_timestamp;
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen importer->sync_max_size = sync_max_size;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen importer->stateful_import = importer->last_common_uid_found;
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi importer->hashed_headers = hashed_headers;
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (sync_flag != NULL) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (sync_flag[0] == '-') {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen importer->sync_flag_dontwant = TRUE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen sync_flag++;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (sync_flag[0] == '\\')
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen importer->sync_flag = imap_parse_system_flag(sync_flag);
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen else
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen importer->sync_keyword = p_strdup(pool, sync_flag);
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen importer->commit_msgs_interval = commit_msgs_interval;
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen importer->transaction_flags = MAILBOX_TRANSACTION_FLAG_SYNC;
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY) != 0)
14b1d2a2634e75b988078baee1e8ad678de28a04Timo Sirainen importer->transaction_flags |= MAILBOX_TRANSACTION_FLAG_NO_NOTIFY;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create_direct(&importer->import_uids, pool, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_array_init(&importer->maybe_expunge_uids, 16);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_array_init(&importer->maybe_saves, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_array_init(&importer->newmails, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_array_init(&importer->wanted_uids, 128);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen i_array_init(&importer->saved_uids, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen dsync_mailbox_import_transaction_begin(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_array_init(&importer->mail_requests, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->want_mail_requests = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->master_brain =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen importer->revert_local_changes =
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0;
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen importer->mails_have_guids =
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0;
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen importer->mails_use_guid128 =
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0;
f3c24c2c92802cb773315eba1132254932d8709bTimo Sirainen importer->hdr_hash_version = hdr_hash_version;
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi importer->empty_hdr_workaround =
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen mailbox_get_open_status(importer->box, STATUS_UIDNEXT |
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen &status);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->local_uid_next = status.uidnext;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->local_initial_highestmodseq = status.highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_search_init(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen if (!importer->stateful_import)
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen ;
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen else if (importer->local_uid_next <= last_common_uid) {
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen "local UIDNEXT %u <= last common UID %u",
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen importer->local_uid_next, last_common_uid));
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen } else if (importer->local_initial_highestmodseq < last_common_modseq) {
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64,
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi importer->local_initial_highestmodseq,
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi last_common_modseq));
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen } else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) {
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64,
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi importer->local_initial_highestpvtmodseq,
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi last_common_pvt_modseq));
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen }
808fab19464062a665af85df5e147c6b64b1c348Timo Sirainen
08e9fec5ba9e1a26e658c4224207d666b6ced27dTimo Sirainen importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan);
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return importer;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainenstatic int
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainendsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer,
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen enum mail_attribute_type type, const char *key,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct dsync_mailbox_attribute **attr_r)
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen{
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen struct dsync_mailbox_attribute lookup_attr, *attr;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen const struct dsync_mailbox_attribute *attr_change;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen struct mail_attribute_value value;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen *attr_r = NULL;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
66c87722e0fd2a85cd59797326bad3d1c409dc3aAki Tuomi if (mailbox_attribute_get_stream(importer->box, type, key, &value) < 0) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen i_error("Mailbox %s: Failed to get attribute %s: %s",
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen mailbox_get_vname(importer->box), key,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen importer->failed = TRUE;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen return -1;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen }
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen lookup_attr.type = type;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen lookup_attr.key = key;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr_change = hash_table_lookup(importer->local_attr_changes,
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen &lookup_attr);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr_change == NULL &&
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen value.value == NULL && value.value_stream == NULL) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* we have no knowledge of this attribute */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen return 0;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen }
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr = t_new(struct dsync_mailbox_attribute, 1);
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->type = type;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->key = key;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->value = value.value;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->value_stream = value.value_stream;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->last_change = value.last_change;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen if (attr_change != NULL) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->deleted = attr_change->deleted &&
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen !DSYNC_ATTR_HAS_VALUE(attr);
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->modseq = attr_change->modseq;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen }
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen *attr_r = attr;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen return 0;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen}
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainenstatic int
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_istreams_cmp(struct istream *input1, struct istream *input2, int *cmp_r)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen{
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen const unsigned char *data1, *data2;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen size_t size1, size2, size;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
5e32f3d9e2c58b6db53cc3b063c9ee73949207caTimo Sirainen *cmp_r = -1; /* quiet gcc */
5e32f3d9e2c58b6db53cc3b063c9ee73949207caTimo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen for (;;) {
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody (void)i_stream_read_more(input1, &data1, &size1);
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody (void)i_stream_read_more(input2, &data2, &size2);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (size1 == 0 || size2 == 0)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen break;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen size = I_MIN(size1, size2);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen *cmp_r = memcmp(data1, data2, size);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (*cmp_r != 0)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_skip(input1, size);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_skip(input2, size);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (input1->stream_errno != 0) {
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen i_error("read(%s) failed: %s", i_stream_get_name(input1),
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen i_stream_get_error(input1));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return -1;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (input2->stream_errno != 0) {
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen i_error("read(%s) failed: %s", i_stream_get_name(input2),
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen i_stream_get_error(input2));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return -1;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (size1 == 0 && size2 == 0)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen *cmp_r = 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen else
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen *cmp_r = size1 == 0 ? -1 : 1;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen}
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainenstatic int
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen const struct dsync_mailbox_attribute *attr2,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen int *cmp_r)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen{
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct istream *input1, *input2;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen int ret;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
ea918d503ae61496e10103935028d3cc6419f1cbTimo Sirainen i_assert(attr1->value_stream != NULL || attr1->value != NULL);
ea918d503ae61496e10103935028d3cc6419f1cbTimo Sirainen i_assert(attr2->value_stream != NULL || attr2->value != NULL);
ea918d503ae61496e10103935028d3cc6419f1cbTimo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr1->value != NULL && attr2->value != NULL) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen *cmp_r = strcmp(attr1->value, attr2->value);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* at least one of them is a stream. make both of them streams. */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen input1 = attr1->value_stream != NULL ? attr1->value_stream :
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_create_from_data(attr1->value, strlen(attr1->value));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen input2 = attr2->value_stream != NULL ? attr2->value_stream :
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_create_from_data(attr2->value, strlen(attr2->value));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_seek(input1, 0);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_seek(input2, 0);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen ret = dsync_istreams_cmp(input1, input2, cmp_r);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr1->value_stream == NULL)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_unref(&input1);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr2->value_stream == NULL)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_unref(&input2);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return ret;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen}
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainenstatic int
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_attributes_cmp(const struct dsync_mailbox_attribute *attr,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen const struct dsync_mailbox_attribute *local_attr,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen int *cmp_r)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen{
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (DSYNC_ATTR_HAS_VALUE(attr) &&
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen !DSYNC_ATTR_HAS_VALUE(local_attr)) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* remote has a value and local doesn't -> use it */
ab3c52cff40218f248fac2bd7c93125cc2ae4c9dTimo Sirainen *cmp_r = 1;
ab3c52cff40218f248fac2bd7c93125cc2ae4c9dTimo Sirainen return 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen } else if (!DSYNC_ATTR_HAS_VALUE(attr) &&
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen DSYNC_ATTR_HAS_VALUE(local_attr)) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* remote doesn't have a value, bt local does -> skip */
ab3c52cff40218f248fac2bd7c93125cc2ae4c9dTimo Sirainen *cmp_r = -1;
ab3c52cff40218f248fac2bd7c93125cc2ae4c9dTimo Sirainen return 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return dsync_attributes_cmp_values(attr, local_attr, cmp_r);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen}
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainenstatic int
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainendsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer,
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen const struct dsync_mailbox_attribute *attr,
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen const struct dsync_mailbox_attribute *local_attr,
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen const char **result_r)
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen{
9307e4f91ea313a303b5473d9c53a3a2f383300fTimo Sirainen struct mail_attribute_value value;
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen int cmp;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen bool ignore = FALSE;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted);
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr->deleted &&
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* attribute doesn't exist on either side -> ignore */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Nonexistent in both sides";
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen if (local_attr == NULL) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* we haven't seen this locally -> use whatever remote has */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Nonexistent locally";
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (local_attr->modseq <= importer->last_common_modseq &&
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->modseq > importer->last_common_modseq &&
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen importer->last_common_modseq > 0) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* we're doing incremental syncing, and we can see that the
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attribute was changed remotely, but not locally -> use it */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Changed remotely";
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (local_attr->modseq > importer->last_common_modseq &&
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attr->modseq <= importer->last_common_modseq &&
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen importer->last_common_modseq > 0) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* we're doing incremental syncing, and we can see that the
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen attribute was changed locally, but not remotely -> ignore */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Changed locally";
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen ignore = TRUE;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (attr->last_change > local_attr->last_change) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* remote has a newer timestamp -> use it */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Remote has newer timestamp";
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (attr->last_change < local_attr->last_change) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* remote has an older timestamp -> ignore */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Local has newer timestamp";
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen ignore = TRUE;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* the timestamps are the same. now we're down to guessing
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen the right answer, unless the values are actually equal,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen so check that first. next try to use modseqs, but if even
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen they are the same, fallback to just picking one based on the
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen value. */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) {
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen importer->mail_error = MAIL_ERROR_TEMP;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen importer->failed = TRUE;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return -1;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (cmp == 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* identical scripts */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Unchanged value";
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen if (attr->modseq > local_attr->modseq) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* remote has a higher modseq -> use it */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Remote has newer modseq";
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else if (attr->modseq < local_attr->modseq) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen /* remote has an older modseq -> ignore */
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Local has newer modseq";
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen ignore = TRUE;
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen } else if (cmp < 0) {
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen ignore = TRUE;
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Value changed, but unknown which is newer - picking local";
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen } else {
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen *result_r = "Value changed, but unknown which is newer - picking remote";
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen }
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen }
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen if (ignore)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 0;
9307e4f91ea313a303b5473d9c53a3a2f383300fTimo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&value);
9307e4f91ea313a303b5473d9c53a3a2f383300fTimo Sirainen value.value = attr->value;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen value.value_stream = attr->value_stream;
9307e4f91ea313a303b5473d9c53a3a2f383300fTimo Sirainen value.last_change = attr->last_change;
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen if (mailbox_attribute_set(importer->trans, attr->type,
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen attr->key, &value) < 0) {
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen i_error("Mailbox %s: Failed to set attribute %s: %s",
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen mailbox_get_vname(importer->box), attr->key,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box, NULL));
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen /* the attributes aren't vital, don't fail everything just
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen because of them. */
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen }
719123a3ec5aeb45ef1c50c265039666c71981d7Timo Sirainen return 0;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen}
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainenint dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen const struct dsync_mailbox_attribute *attr)
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen{
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen struct dsync_mailbox_attribute *local_attr;
472f3cbaaa2ba84e2241da0747630d391e120c84Timo Sirainen const char *result = "";
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen int ret;
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen if (dsync_mailbox_import_lookup_attr(importer, attr->type,
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen attr->key, &local_attr) < 0)
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen ret = -1;
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen else {
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen ret = dsync_mailbox_import_attribute_real(importer, attr,
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen local_attr, &result);
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen if (local_attr != NULL && local_attr->value_stream != NULL)
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen i_stream_unref(&local_attr->value_stream);
eb729968680f4a36616ce8c60848c3b6c407f791Timo Sirainen }
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen imp_debug(importer, "Import attribute %s: %s", attr->key,
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen ret < 0 ? "failed" : result);
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen return ret;
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen}
a3e96a912e8a058a7372eda55ffc95d54269bd3cTimo Sirainen
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainenstatic void dsync_mail_error(struct dsync_mailbox_importer *importer,
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen struct mail *mail, const char *field)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *errstr;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen enum mail_error error;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi errstr = mailbox_get_last_internal_error(importer->box, &error);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (error == MAIL_ERROR_EXPUNGED)
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mailbox_get_vname(mail->box), field, mail->uid, errstr);
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen importer->mail_error = error;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->failed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainenstatic bool
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainendsync_mail_change_guid_equals(struct dsync_mailbox_importer *importer,
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen const struct dsync_mail_change *change,
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen const char *guid, const char **cmp_guid_r)
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen{
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen guid_128_t guid_128, change_guid_128;
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen if (guid_128_from_string(change->guid, change_guid_128) < 0)
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen i_unreached();
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen } else if (importer->mails_use_guid128) {
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen mail_generate_guid_128_hash(change->guid, change_guid_128);
8601cb6dafd4e8960c91b3c082af3095cfbebe74Timo Sirainen } else {
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen if (cmp_guid_r != NULL)
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen *cmp_guid_r = change->guid;
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen return strcmp(change->guid, guid) == 0;
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen }
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen mail_generate_guid_128_hash(guid, guid_128);
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) {
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen if (cmp_guid_r != NULL) {
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen *cmp_guid_r = t_strdup_printf("%s(guid128, orig=%s)",
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen binary_to_hex(change_guid_128, sizeof(change_guid_128)),
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen change->guid);
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen }
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen return FALSE;
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen }
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen return TRUE;
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen}
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainenstatic int
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainenimporter_try_next_mail(struct dsync_mailbox_importer *importer,
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen uint32_t wanted_uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen struct mail_private *pmail;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen const char *hdr_hash;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->cur_mail == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* end of search */
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (importer->cur_mail->seq < importer->next_local_seq ||
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_mail->uid < wanted_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!importer->cur_uid_has_change &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen !importer->last_common_uid_found) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this message exists locally, but remote didn't send
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen expunge-change for it. if the message's
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid <= last-common-uid, it should be deleted */
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&importer->maybe_expunge_uids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen importer->cur_mail_skip = FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!mailbox_search_next(importer->search_ctx,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen &importer->cur_mail)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_mail = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_guid = NULL;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen importer->cur_hdr_hash = NULL;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_uid_has_change = FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7be1a5530fcb414588fbe90eaed65eff83e84737Timo Sirainen importer->cur_uid_has_change = importer->cur_mail->uid == wanted_uid;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen if (importer->mails_have_guids) {
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen &importer->cur_guid) < 0) {
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen dsync_mail_error(importer, importer->cur_mail, "GUID");
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen return 0;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen }
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen } else {
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen if (dsync_mail_get_hdr_hash(importer->cur_mail,
45af47783693b3ba2768c5ad34eeff68132382d0Timo Sirainen importer->hdr_hash_version,
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi importer->hashed_headers,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen &hdr_hash) < 0) {
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen dsync_mail_error(importer, importer->cur_mail,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen "header hash");
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen return 0;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen }
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen pmail = (struct mail_private *)importer->cur_mail;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen importer->cur_hdr_hash = p_strdup(pmail->pool, hdr_hash);
a1893588a73d32a322153f5a2f7a16ef2c724979Timo Sirainen importer->cur_guid = "";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* make sure next_local_seq gets updated in case we came here
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen because of min_uid */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->next_local_seq = importer->cur_mail->seq;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen return 1;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen}
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainenstatic bool
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainenimporter_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid)
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen{
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen int ret;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen
3ef4aca8cae3e335e1fe53591049cb80b299c459Timo Sirainen for (;;) {
3ef4aca8cae3e335e1fe53591049cb80b299c459Timo Sirainen T_BEGIN {
3ef4aca8cae3e335e1fe53591049cb80b299c459Timo Sirainen ret = importer_try_next_mail(importer, wanted_uid);
3ef4aca8cae3e335e1fe53591049cb80b299c459Timo Sirainen } T_END;
3ef4aca8cae3e335e1fe53591049cb80b299c459Timo Sirainen if (ret != 0 || importer->failed)
3ef4aca8cae3e335e1fe53591049cb80b299c459Timo Sirainen break;
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen importer->next_local_seq = importer->cur_mail->seq + 1;
3ef4aca8cae3e335e1fe53591049cb80b299c459Timo Sirainen }
d52f5dcb05092e126058874772f2c367499e650aTimo Sirainen return ret > 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenimporter_mail_cmp(const struct importer_mail *m1,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct importer_mail *m2)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen int ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (m1->guid == NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (m2->guid == NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ret = strcmp(m1->guid, m2->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (ret != 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (m1->uid < m2->uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (m1->uid > m2->uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void newmail_link(struct dsync_mailbox_importer *importer,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen struct importer_new_mail *newmail, uint32_t remote_uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *first_mail, **last, *mail, *link = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (*newmail->guid != '\0') {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen first_mail = hash_table_lookup(importer->import_guids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (first_mail == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* first mail for this GUID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_insert(importer->import_guids,
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen newmail->guid, newmail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (remote_uid == 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* mail exists only locally. we don't want to request
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen it, and we'll assume it has no duplicate
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen instances. */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen first_mail = hash_table_lookup(importer->import_uids,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen POINTER_CAST(remote_uid));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (first_mail == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* first mail for this UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_insert(importer->import_uids,
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen POINTER_CAST(remote_uid), newmail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* 1) add the newmail to the end of the linked list
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen 2) find our link
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen FIXME: this loop is slow if the same GUID has a ton of instances.
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen Could it be improved in some way? */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last = &first_mail->next;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (mail = first_mail; mail != NULL; mail = mail->next) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (mail->final_uid == newmail->final_uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail->uid_is_usable = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (link == NULL && mail->link == NULL &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail->uid_in_local != newmail->uid_in_local)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen link = mail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last = &mail->next;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *last = newmail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (link != NULL && newmail->link == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen link->link = newmail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->link = link;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainenstatic void
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainendsync_mailbox_revert_existing_uid(struct dsync_mailbox_importer *importer,
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen uint32_t uid, const char *reason)
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen{
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen i_assert(importer->revert_local_changes);
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen /* UID either already exists or UIDNEXT is too high. we can't set the
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen wanted UID, so we'll need to delete the whole mailbox and resync */
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen i_warning("Deleting mailbox '%s': UID=%u already exists locally for a different mail: %s",
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen mailbox_get_vname(importer->box), uid, reason);
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen importer->delete_mailbox = TRUE;
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen importer->mail_error = MAIL_ERROR_TEMP;
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen importer->failed = TRUE;
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen}
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *save_change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_mail m1, m2;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *newmail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen int diff;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen bool remote_saved;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&m1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->cur_mail != NULL) {
8576eb5abb66178f251c00209e564c7673c0e4cfTimo Sirainen m1.guid = importer->mails_have_guids ?
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen importer->cur_guid : importer->cur_hdr_hash;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen m1.uid = importer->cur_mail->uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&m2);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (save_change != NULL) {
8576eb5abb66178f251c00209e564c7673c0e4cfTimo Sirainen m2.guid = importer->mails_have_guids ?
8576eb5abb66178f251c00209e564c7673c0e4cfTimo Sirainen save_change->guid : save_change->hdr_hash;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen m2.uid = save_change->uid;
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi if (importer->empty_hdr_workaround && !importer->mails_have_guids &&
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi importer->cur_mail != NULL && save_change != NULL &&
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi (dsync_mail_hdr_hash_is_empty(m1.guid) ||
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi dsync_mail_hdr_hash_is_empty(m2.guid))) {
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi /* one of the headers is empty. assume it's broken and that
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi the header matches what we have currently. */
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi diff = 0;
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi } else {
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi diff = importer_mail_cmp(&m1, &m2);
afd6d387ea65843b59fb6051fb567719d2a5279cAki Tuomi }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (diff < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* add a record for local mail */
3ea86ed7cf06ba04e4aa6cd1c4df9be336c06cd3Timo Sirainen i_assert(importer->cur_mail != NULL);
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen if (importer->revert_local_changes) {
8c8f6ed0a46db6589c08d8db1f4ed779ef5bc19bTimo Sirainen if (save_change == NULL &&
8c8f6ed0a46db6589c08d8db1f4ed779ef5bc19bTimo Sirainen importer->cur_mail->uid >= importer->remote_uid_next) {
79490ec1a58241d6011fa36713ca651d795855c3Timo Sirainen dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid,
8c8f6ed0a46db6589c08d8db1f4ed779ef5bc19bTimo Sirainen t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next));
79490ec1a58241d6011fa36713ca651d795855c3Timo Sirainen return TRUE;
79490ec1a58241d6011fa36713ca651d795855c3Timo Sirainen }
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen mail_expunge(importer->cur_mail);
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen importer->cur_mail_skip = TRUE;
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen importer->next_local_seq++;
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen return FALSE;
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen }
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->guid = p_strdup(importer->pool, importer->cur_guid);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->final_uid = importer->cur_mail->uid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->local_uid = importer->cur_mail->uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->uid_in_local = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->uid_is_usable =
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->final_uid >= importer->remote_uid_next;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_saved = FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (diff > 0) {
3ea86ed7cf06ba04e4aa6cd1c4df9be336c06cd3Timo Sirainen i_assert(save_change != NULL);
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->guid = save_change->guid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->final_uid = save_change->uid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->remote_uid = save_change->uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->uid_in_local = FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->uid_is_usable =
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->final_uid >= importer->local_uid_next;
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen if (!newmail->uid_is_usable && importer->revert_local_changes) {
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen dsync_mailbox_revert_existing_uid(importer, newmail->final_uid,
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen t_strdup_printf("UID >= local UIDNEXT=%u", importer->local_uid_next));
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen return TRUE;
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_saved = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* identical */
3ea86ed7cf06ba04e4aa6cd1c4df9be336c06cd3Timo Sirainen i_assert(importer->cur_mail != NULL);
3ea86ed7cf06ba04e4aa6cd1c4df9be336c06cd3Timo Sirainen i_assert(save_change != NULL);
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen newmail = p_new(importer->pool, struct importer_new_mail, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->guid = save_change->guid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->final_uid = importer->cur_mail->uid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->local_uid = importer->cur_mail->uid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->remote_uid = save_change->uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->uid_in_local = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->uid_is_usable = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->link = newmail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_saved = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (newmail->uid_in_local) {
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen importer->cur_mail_skip = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->next_local_seq++;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* NOTE: assumes save_change is allocated from importer pool */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen newmail->change = save_change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->newmails, &newmail, 1);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail_link(importer, newmail,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen save_change == NULL ? 0 : save_change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return remote_saved;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainenstatic bool ATTR_NULL(2)
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainendsync_mailbox_try_save(struct dsync_mailbox_importer *importer,
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen struct dsync_mail_change *save_change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
471a6b2b4e64eca5d5779ae20a477312b32c89eeTimo Sirainen if (importer->cur_mail_skip) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!importer_next_mail(importer, 0) && save_change == NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mailbox_try_save_cur(importer, save_change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void dsync_mailbox_save(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *save_change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (!dsync_mailbox_try_save(importer, save_change)) ;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic bool
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_import_set_mail(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen const char *guid, *cmp_guid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!mail_set_uid(importer->mail, change->uid))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->guid == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID is unknown */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (*change->guid == '\0') {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* backend doesn't support GUIDs. if hdr_hash is set, we could
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen verify it, but since this message really is supposed to
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen match, it's probably too much trouble. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* verify that GUID matches, just in case */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_error(importer, importer->mail, "GUID");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen if (!dsync_mail_change_guid_equals(importer, change, guid, &cmp_guid)) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen "Unexpected GUID mismatch for UID=%u: %s != %s",
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen change->uid, guid, cmp_guid));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainenstatic bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer,
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen const struct dsync_mail_change *change)
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen{
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen const char *cmp_guid;
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen
34b8b14ce06c0939932b60f22f61aea124198438Timo Sirainen if (change->guid == NULL || change->guid[0] == '\0' ||
34b8b14ce06c0939932b60f22f61aea124198438Timo Sirainen importer->cur_guid[0] == '\0')
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen return TRUE;
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen if (!dsync_mail_change_guid_equals(importer, change,
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen importer->cur_guid, &cmp_guid)) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen "Unexpected GUID mismatch (2) for UID=%u: %s != %s",
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen change->uid, importer->cur_guid, cmp_guid));
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen return FALSE;
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen }
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen return TRUE;
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen}
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenmerge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove,
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote,
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen uint32_t *change_add_r, uint32_t *change_remove_r,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen bool *remote_changed, bool *remote_pvt_changed)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t combined_add, combined_remove, conflict_flags;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint32_t local_wanted, remote_wanted, conflict_pvt_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* resolve conflicts */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen conflict_flags = local_add & remote_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (conflict_flags != 0) {
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_pvt_flags = conflict_flags & pvt_mask;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_flags &= ~pvt_mask;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (prefer_remote)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_add &= ~conflict_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen else
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_remove &= ~conflict_flags;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen if (prefer_pvt_remote)
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen local_add &= ~conflict_pvt_flags;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen remote_remove &= ~conflict_pvt_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
2e86ce0239b4bc88c94d7edb5813d38ab080627eTimo Sirainen conflict_flags = local_remove & remote_add;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (conflict_flags != 0) {
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_pvt_flags = conflict_flags & pvt_mask;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen conflict_flags &= ~pvt_mask;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (prefer_remote)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_remove &= ~conflict_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen else
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_add &= ~conflict_flags;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen if (prefer_pvt_remote)
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen local_remove &= ~conflict_pvt_flags;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen remote_add &= ~conflict_pvt_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen combined_add = local_add|remote_add;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen combined_remove = local_remove|remote_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert((combined_add & combined_remove) == 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
cf848255bf65a5e2382c59c093da72f877f7535aTimo Sirainen /* don't change flags that are currently identical in both sides */
cf848255bf65a5e2382c59c093da72f877f7535aTimo Sirainen conflict_flags = local_final ^ remote_final;
cf848255bf65a5e2382c59c093da72f877f7535aTimo Sirainen combined_add &= conflict_flags;
cf848255bf65a5e2382c59c093da72f877f7535aTimo Sirainen combined_remove &= conflict_flags;
cf848255bf65a5e2382c59c093da72f877f7535aTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* see if there are conflicting final flags */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_wanted = (local_final|combined_add) & ~combined_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_wanted = (remote_final|combined_add) & ~combined_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen conflict_flags = local_wanted ^ remote_wanted;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (conflict_flags != 0) {
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen if (prefer_remote && prefer_pvt_remote)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_wanted = remote_wanted;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else if (prefer_remote && !prefer_pvt_remote) {
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen local_wanted = (local_wanted & pvt_mask) |
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen (remote_wanted & ~pvt_mask);
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen } else if (!prefer_remote && prefer_pvt_remote) {
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen local_wanted = (local_wanted & ~pvt_mask) |
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen (remote_wanted & pvt_mask);
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *change_add_r = local_wanted & ~local_final;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *change_remove_r = local_final & ~local_wanted;
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen if ((local_wanted & ~pvt_mask) != (remote_final & ~pvt_mask))
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen *remote_changed = TRUE;
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen if ((local_wanted & pvt_mask) != (remote_final & pvt_mask))
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen *remote_pvt_changed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic bool
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenkeyword_find(ARRAY_TYPE(const_string) *keywords, const char *name,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int *idx_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *names;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen names = array_get(keywords, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (strcmp(names[i], name) == 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *idx_r = i;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void keywords_append(ARRAY_TYPE(const_string) *dest,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const ARRAY_TYPE(const_string) *keywords,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t bits, unsigned int start_idx)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *namep;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < 32; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((bits & (1U << i)) == 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen continue;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen namep = array_idx(keywords, start_idx+i);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(dest, namep, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenmerge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const ARRAY_TYPE(const_string) *remote_changes,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen bool prefer_remote,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen bool *remote_changed, bool *remote_pvt_changed)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* local_changes and remote_changes are assumed to have no
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen duplicates names */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *local_add, *local_remove, *local_final;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *remote_add, *remote_remove, *remote_final;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *change_add, *change_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *changes, *name, *const *local_keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_keywords *kw;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count, name_idx, array_size;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_keywords = mail_get_keywords(mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we'll assign a common index for each keyword name and place
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen the changes to separate bit arrays. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_is_created(remote_changes))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(remote_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen count = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_size = str_array_length(local_keywords) + count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_is_created(local_changes))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_size += array_count(local_changes);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_size == 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this message has no keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen t_array_init(&all_keywords, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen t_array_init(&add_keywords, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen t_array_init(&remove_keywords, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* @UNSAFE: create large enough arrays to fit all keyword indexes. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_size = (array_size+31)/32;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_add = t_new(uint32_t, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_remove = t_new(uint32_t, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_final = t_new(uint32_t, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_add = t_new(uint32_t, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_remove = t_new(uint32_t, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_final = t_new(uint32_t, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change_add = t_new(uint32_t, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change_remove = t_new(uint32_t, array_size);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get remote changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen name = changes[i]+1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen name_idx = array_count(&all_keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&all_keywords, &name, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen switch (changes[i][0]) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen case KEYWORD_CHANGE_ADD:
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_add[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen case KEYWORD_CHANGE_REMOVE:
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_remove[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen case KEYWORD_CHANGE_FINAL:
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen remote_final[name_idx/32] |= 1U << (name_idx%32);
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen break;
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen case KEYWORD_CHANGE_ADD_AND_FINAL:
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen remote_add[name_idx/32] |= 1U << (name_idx%32);
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen remote_final[name_idx/32] |= 1U << (name_idx%32);
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get local changes. use existing indexes for names when they exist. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_is_created(local_changes))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(local_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen count = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen name = changes[i]+1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!keyword_find(&all_keywords, name, &name_idx)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen name_idx = array_count(&all_keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&all_keywords, &name, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen switch (changes[i][0]) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen case KEYWORD_CHANGE_ADD:
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen case KEYWORD_CHANGE_ADD_AND_FINAL:
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_add[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen case KEYWORD_CHANGE_REMOVE:
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_remove[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen case KEYWORD_CHANGE_FINAL:
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; local_keywords[i] != NULL; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen name = local_keywords[i];
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!keyword_find(&all_keywords, name, &name_idx)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen name_idx = array_count(&all_keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&all_keywords, &name, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_final[name_idx/32] |= 1U << (name_idx%32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(array_count(&all_keywords) <= array_size*32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_size = (array_count(&all_keywords)+31) / 32;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < array_size; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_flags(local_final[i], local_add[i], local_remove[i],
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen remote_final[i], remote_add[i], remote_remove[i],
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen 0, prefer_remote, prefer_remote,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen &change_add[i], &change_remove[i],
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen remote_changed, remote_pvt_changed);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change_add[i] != 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keywords_append(&add_keywords, &all_keywords,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change_add[i], i*32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change_remove[i] != 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keywords_append(&remove_keywords, &all_keywords,
59bf1df59552b6898749a87ae28476debd9b77d8Timo Sirainen change_remove[i], i*32);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* apply changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_count(&add_keywords) > 0) {
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen array_append_zero(&add_keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen kw = mailbox_keywords_create_valid(mail->box,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_idx(&add_keywords, 0));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_keywords(mail, MODIFY_ADD, kw);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_keywords_unref(&kw);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_count(&remove_keywords) > 0) {
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen array_append_zero(&remove_keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen kw = mailbox_keywords_create_valid(mail->box,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_idx(&remove_keywords, 0));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_keywords(mail, MODIFY_REMOVE, kw);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_keywords_unref(&kw);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainenstatic void
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainendsync_mailbox_import_replace_flags(struct mail *mail,
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen const struct dsync_mail_change *change)
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen{
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen ARRAY_TYPE(const_string) keywords;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen struct mail_keywords *kw;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen const char *const *changes, *name;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen unsigned int i, count;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen if (array_is_created(&change->keyword_changes))
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen changes = array_get(&change->keyword_changes, &count);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen else {
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen changes = NULL;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen count = 0;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen }
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen t_array_init(&keywords, count+1);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen for (i = 0; i < count; i++) {
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen switch (changes[i][0]) {
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen case KEYWORD_CHANGE_ADD:
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen case KEYWORD_CHANGE_FINAL:
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen case KEYWORD_CHANGE_ADD_AND_FINAL:
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen name = changes[i]+1;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen array_append(&keywords, &name, 1);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen break;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen case KEYWORD_CHANGE_REMOVE:
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen break;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen }
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen }
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen array_append_zero(&keywords);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen kw = mailbox_keywords_create_valid(mail->box, array_idx(&keywords, 0));
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen mail_update_keywords(mail, MODIFY_REPLACE, kw);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen mailbox_keywords_unref(&kw);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen mail_update_flags(mail, MODIFY_REPLACE,
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen change->add_flags | change->final_flags);
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen if (mail_get_modseq(mail) < change->modseq)
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen mail_update_modseq(mail, change->modseq);
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
ccd7b4e0a5f09058a59cc4b3f878254e93e7cb0aTimo Sirainen mail_update_pvt_modseq(mail, change->pvt_modseq);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen}
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail_change *local_change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen enum mail_flags local_add, local_remove;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t change_add, change_remove;
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen uint64_t new_modseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail;
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen bool prefer_remote, prefer_pvt_remote;
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen bool remote_changed = FALSE, remote_pvt_changed = FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert((change->add_flags & change->remove_flags) == 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->cur_mail != NULL &&
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen importer->cur_mail->uid == change->uid) {
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen if (!dsync_check_cur_guid(importer, change))
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail = importer->cur_mail;
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!dsync_import_set_mail(importer, change))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail = importer->mail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen if (importer->revert_local_changes) {
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen /* dsync backup: just make the local look like remote. */
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen dsync_mailbox_import_replace_flags(mail, change);
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen return;
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen }
efeb13303798b47d2c4295468d233c1bcfd79c94Timo Sirainen
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen local_change = hash_table_lookup(importer->local_changes,
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen POINTER_CAST(change->uid));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (local_change == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_add = local_remove = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_add = local_change->add_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_remove = local_change->remove_flags;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local_keyword_changes = local_change->keyword_changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_get_modseq(mail) < change->modseq)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen prefer_remote = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen else if (mail_get_modseq(mail) > change->modseq)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen prefer_remote = FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* identical modseq, we'll just have to pick one.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen Note that both brains need to pick the same one, otherwise
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen they become unsynced. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen prefer_remote = !importer->master_brain;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen prefer_pvt_remote = TRUE;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else if (mail_get_pvt_modseq(mail) > change->pvt_modseq)
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen prefer_pvt_remote = FALSE;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen else
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen prefer_pvt_remote = !importer->master_brain;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge flags */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_flags(mail_get_flags(mail), local_add, local_remove,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->final_flags, change->add_flags, change->remove_flags,
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen mailbox_get_private_flags_mask(mail->box),
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen prefer_remote, prefer_pvt_remote,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen &change_add, &change_remove,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen &remote_changed, &remote_pvt_changed);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change_add != 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_flags(mail, MODIFY_ADD, change_add);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change_remove != 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_update_flags(mail, MODIFY_REMOVE, change_remove);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* merge keywords */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen merge_keywords(mail, &local_keyword_changes, &change->keyword_changes,
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen prefer_remote, &remote_changed, &remote_pvt_changed);
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen /* update modseqs. try to anticipate when we have to increase modseq
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen to get it closer to what remote has (although we can't guess it
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen exactly correctly) */
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen new_modseq = change->modseq;
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen if (remote_changed && new_modseq <= importer->remote_highest_modseq)
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen new_modseq = importer->remote_highest_modseq+1;
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen if (mail_get_modseq(mail) < new_modseq)
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen mail_update_modseq(mail, new_modseq);
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen new_modseq = change->pvt_modseq;
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen if (remote_pvt_changed && new_modseq <= importer->remote_highest_pvt_modseq)
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen new_modseq = importer->remote_highest_pvt_modseq+1;
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen if (mail_get_pvt_modseq(mail) < new_modseq)
53c384be5f8f27945fd61b8a0d731a93a261628fTimo Sirainen mail_update_pvt_modseq(mail, new_modseq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainenstatic bool
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainendsync_mail_change_have_keyword(const struct dsync_mail_change *change,
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen const char *keyword)
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen{
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen const char *const *strp;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen
845bac8afdbd5f521b8a6d4493f0581f3b1a2692Timo Sirainen if (!array_is_created(&change->keyword_changes))
845bac8afdbd5f521b8a6d4493f0581f3b1a2692Timo Sirainen return FALSE;
845bac8afdbd5f521b8a6d4493f0581f3b1a2692Timo Sirainen
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen array_foreach(&change->keyword_changes, strp) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen switch ((*strp)[0]) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen case KEYWORD_CHANGE_FINAL:
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen case KEYWORD_CHANGE_ADD_AND_FINAL:
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (strcasecmp((*strp)+1, keyword) == 0)
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return TRUE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen break;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen default:
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen break;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return FALSE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen}
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainenstatic bool
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainendsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer,
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen const struct dsync_mail_change *change,
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen const char **result_r)
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen{
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (importer->sync_since_timestamp > 0) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen i_assert(change->received_timestamp > 0);
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (change->received_timestamp < importer->sync_since_timestamp) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen /* mail has too old timestamp - skip it */
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail with too old timestamp";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return FALSE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi if (importer->sync_until_timestamp > 0) {
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi i_assert(change->received_timestamp > 0);
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi if (change->received_timestamp > importer->sync_until_timestamp) {
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi /* mail has too new timestamp - skip it */
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi *result_r = "Ignoring missing local mail with too new timestamp";
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi return FALSE;
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi }
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi }
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen if (importer->sync_max_size > 0) {
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen i_assert(change->virtual_size != (uoff_t)-1);
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen if (change->virtual_size < importer->sync_max_size) {
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen /* mail is too large - skip it */
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen *result_r = "Ignoring missing local mail with too large size";
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen return FALSE;
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen }
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (importer->sync_flag != 0) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen bool have_flag = (change->final_flags & importer->sync_flag) != 0;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (have_flag && importer->sync_flag_dontwant) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail that doesn't have wanted flags";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return FALSE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!have_flag && !importer->sync_flag_dontwant) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail that has unwanted flags";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return FALSE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (importer->sync_keyword != NULL) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen bool have_kw = dsync_mail_change_have_keyword(change, importer->sync_keyword);
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (have_kw && importer->sync_flag_dontwant) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail that doesn't have wanted keywords";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return FALSE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!have_kw && !importer->sync_flag_dontwant) {
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen *result_r = "Ignoring missing local mail that has unwanted keywords";
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return FALSE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return TRUE;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen}
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_save(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *save;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen const char *result;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->guid != NULL);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid == importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we've already verified that the GUID matches.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen apply flag changes if there are any. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(!importer->last_common_uid_found);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_flag_change(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!dsync_mailbox_import_want_change(importer, change, &result))
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save = p_new(importer->pool, struct dsync_mail_change, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_change_dup(importer->pool, change, save);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->last_common_uid_found) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this is a new mail. its UID may or may not conflict with
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen an existing local mail, we'll figure it out later. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_save(importer, save);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* the local mail is expunged. we'll decide later if we want
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen to save this mail locally or expunge it form remote. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen i_assert(importer->cur_mail == NULL ||
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen change->uid < importer->cur_mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&importer->maybe_saves, &save, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->last_common_uid_found) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* expunge the message, unless its GUID unexpectedly doesn't
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen match */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid <= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (dsync_import_set_mail(importer, change))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_expunge(importer->mail);
d8504a5ac72efb86a8da852c8d814ccaf0ae8625Timo Sirainen } else if (importer->cur_mail == NULL ||
d8504a5ac72efb86a8da852c8d814ccaf0ae8625Timo Sirainen change->uid < importer->cur_mail->uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already expunged locally, we can ignore this.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid=last_common_uid if we managed to verify from
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen transaction log that the GUIDs match */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid >= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (change->uid == importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already verified that the GUID matches */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(importer->cur_mail->uid == change->uid);
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen if (dsync_check_cur_guid(importer, change))
cd70f7aec3bf49147fa80b77dd7ede7d7697202eTimo Sirainen mail_expunge(importer->cur_mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we don't know yet if we should expunge this
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen message or not. queue it until we do. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid > importer->last_common_uid);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&importer->maybe_expunge_uids, change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* If there are local mails after last_common_uid which we skipped
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while trying to match the next message, we need to now go back */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->cur_mail != NULL &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_mail->uid <= importer->last_common_uid+1)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_mail = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->cur_guid = NULL;
975a784c2e02ecdcb56efb7a1db5e4769c7756d8Timo Sirainen importer->cur_hdr_hash = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->next_local_seq = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)mailbox_search_deinit(&importer->search_ctx);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_search_init(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *const *saves;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct seq_range_iter iter;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int n, i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen if (importer->debug) T_BEGIN {
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen string_t *expunges = t_str_new(64);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen imap_write_seq_range(expunges, &importer->maybe_expunge_uids);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen imp_debug(importer, "Last common UID=%u. Delayed expunges=%s",
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen importer->last_common_uid, str_c(expunges));
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen } T_END;
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->last_common_uid_found = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_rewind_search(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* expunge the messages whose expunge-decision we delayed previously */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (seq_range_array_iter_nth(&iter, n++, &uid)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (uid > importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we expunge messages only up to last_common_uid,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ignore the rest */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_set_uid(importer->mail, uid))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_expunge(importer->mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* handle pending saves */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen saves = array_get(&importer->maybe_saves, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen if (saves[i]->uid > importer->last_common_uid) {
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen imp_debug(importer, "Delayed save UID=%u: Save",
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen saves[i]->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_save(importer, saves[i]);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen } else {
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen imp_debug(importer, "Delayed save UID=%u: Ignore",
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen saves[i]->uid);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen const struct dsync_mail_change *change,
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen const char **result_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen const char *hdr_hash, *cmp_guid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (*change->guid != '\0' && *importer->cur_guid != '\0') {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we have GUIDs, verify them */
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen if (dsync_mail_change_guid_equals(importer, change,
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen importer->cur_guid, &cmp_guid)) {
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = "GUIDs match";
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen return 1;
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen } else {
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = t_strdup_printf("GUIDs don't match (%s vs %s)",
61f258ff61e4e210046e68acef9d10088f18d14eTimo Sirainen importer->cur_guid, cmp_guid);
9bbfe7f5ff821cac11d1d2550a91b148f389d82cTimo Sirainen return 0;
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* verify hdr_hash if it exists */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->hdr_hash == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(*importer->cur_guid == '\0');
173fc9ed37637e4609b1ecc9415a9b92067eeeb2Timo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
173fc9ed37637e4609b1ecc9415a9b92067eeeb2Timo Sirainen /* the message was already expunged, so we don't know
173fc9ed37637e4609b1ecc9415a9b92067eeeb2Timo Sirainen its header. return "unknown". */
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = "Unknown match for expunge";
173fc9ed37637e4609b1ecc9415a9b92067eeeb2Timo Sirainen return -1;
173fc9ed37637e4609b1ecc9415a9b92067eeeb2Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_error("Mailbox %s: GUIDs not supported, "
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "sync with header hashes instead",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_vname(importer->box));
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen importer->mail_error = MAIL_ERROR_TEMP;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->failed = TRUE;
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = "Error, invalid parameters";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
45af47783693b3ba2768c5ad34eeff68132382d0Timo Sirainen if (dsync_mail_get_hdr_hash(importer->cur_mail,
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi importer->hdr_hash_version,
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi importer->hashed_headers, &hdr_hash) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mail_error(importer, importer->cur_mail, "hdr-stream");
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = "Error fetching header stream";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen if (importer->empty_hdr_workaround &&
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen (dsync_mail_hdr_hash_is_empty(change->hdr_hash) ||
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen dsync_mail_hdr_hash_is_empty(hdr_hash))) {
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen *result_r = "Empty headers found with workaround enabled - assuming a match";
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen return 1;
94a1fedb51b3403bb879db9f3c4430ae34865e8eTimo Sirainen } else if (strcmp(change->hdr_hash, hdr_hash) == 0) {
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = "Headers hashes match";
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen return 1;
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen } else {
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen *result_r = t_strdup_printf("Headers hashes don't match (%s vs %s)",
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen change->hdr_hash, hdr_hash);
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen return 0;
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainenstatic bool
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainendsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer,
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen const struct dsync_mail_change *change,
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen const char **result_r)
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen{
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen const struct dsync_mail_change *local_change;
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen if (*change->guid == '\0') {
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* remote doesn't support GUIDs, can't verify expunge */
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "GUIDs not supported, can't verify expunge";
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen return FALSE;
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen }
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* local message is expunged. see if we can find its GUID from
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen transaction log and check if the GUIDs match. The GUID in
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen log is a 128bit GUID, so we may need to convert the remote's
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen GUID string to 128bit GUID first. */
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen local_change = hash_table_lookup(importer->local_changes,
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen POINTER_CAST(change->uid));
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen if (local_change == NULL || local_change->guid == NULL) {
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Expunged local mail's GUID not found";
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen return FALSE;
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen }
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen
03936179f87aebde358dbe1ca8c34e5b5551db45Timo Sirainen i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen if (dsync_mail_change_guid_equals(importer, local_change,
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen change->guid, NULL)) {
c3aebe3bd173bb339947d4fb6fa4f0e090c38e69Timo Sirainen importer->last_common_uid = change->uid;
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Expunged local mail's GUID matches remote";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen } else if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
c3aebe3bd173bb339947d4fb6fa4f0e090c38e69Timo Sirainen dsync_mailbox_common_uid_found(importer);
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Expunged local mail's GUID doesn't match remote GUID";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen } else {
d097ad375f7fc532ab5cb91020c206c0def55179Timo Sirainen /* GUID mismatch for two expunged mails. dsync can't update
d097ad375f7fc532ab5cb91020c206c0def55179Timo Sirainen GUIDs for already expunged messages, so we can't immediately
d097ad375f7fc532ab5cb91020c206c0def55179Timo Sirainen determine that the rest of the messages are a mismatch. so
d097ad375f7fc532ab5cb91020c206c0def55179Timo Sirainen for now we'll just skip over this pair. */
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Expunged mails' GUIDs don't match - delaying decision";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen /* NOTE: the return value here doesn't matter, because the only
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen caller that checks for it never reaches this code path */
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen }
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen return TRUE;
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen}
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainenstatic void
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainendsync_mailbox_revert_missing(struct dsync_mailbox_importer *importer,
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen const struct dsync_mail_change *change)
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen{
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen i_assert(importer->revert_local_changes);
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen /* mail exists on remote, but not locally. we'll need to
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen insert this mail back, which means deleting the whole
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen mailbox and resyncing. */
f502a549abbbb69c606243a8800cbe7d05f772f1Timo Sirainen i_warning("Deleting mailbox '%s': UID=%u GUID=%s is missing locally",
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen mailbox_get_vname(importer->box),
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen change->uid, change->guid);
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen importer->delete_mailbox = TRUE;
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen importer->mail_error = MAIL_ERROR_TEMP;
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen importer->failed = TRUE;
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen}
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen const struct dsync_mail_change *change,
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen const char **result_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen int ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi ((change->received_timestamp > 0 ||
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi (importer->sync_since_timestamp == 0 &&
ee8294dbc7bb549557f6ba1264d66b55fbef69b6Aki Tuomi importer->sync_until_timestamp == 0)) &&
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen (change->virtual_size != (uoff_t)-1 || importer->sync_max_size == 0)));
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* try to find the matching local mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!importer_next_mail(importer, change->uid)) {
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen /* no more local mails. we can still try to match
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen expunged mails though. */
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen /* mail doesn't exist remotely either, don't bother
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen looking it up locally. */
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen *result_r = "Expunged mail not found locally";
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen return;
a5f297ac8cb8fb168edabf9fe93a7061539a0afaTimo Sirainen }
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen i_assert(change->guid != NULL);
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!dsync_mailbox_import_want_change(importer, change, result_r))
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen ;
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen else if (importer->local_uid_next <= change->uid) {
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen dsync_mailbox_common_uid_found(importer);
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = "Mail's UID is above local UIDNEXT";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen } else if (importer->revert_local_changes) {
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen dsync_mailbox_revert_missing(importer, change);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen *result_r = "Reverting local change by deleting mailbox";
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen } else if (!dsync_mailbox_find_common_expunged_uid(importer, change, result_r)) {
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen /* it's unknown if this mail existed locally and was
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen expunged. since we don't want to lose any mails,
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen assume that we need to preserve the mail. use the
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen last message with a matching GUID as the last common
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen UID. */
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen dsync_mailbox_common_uid_found(importer);
f3053c3637f64633c8ef642e1b9b689b333ebf73Timo Sirainen }
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen *result_r = t_strdup_printf("%s - No more local mails found", *result_r);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->guid == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we can't know if this UID matches */
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen *result_r = "Expunged mail has no GUID, can't verify it";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->cur_mail->uid == change->uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we have a matching local UID. check GUID to see if it's
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen really the same mail or not */
de39002e8ede55284af93d616bbf5f126188debdTimo Sirainen if ((ret = dsync_mailbox_import_match_msg(importer, change, result_r)) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* unknown */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen if (ret > 0) {
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen importer->last_common_uid = change->uid;
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen } else if (!importer->revert_local_changes) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* mismatch - found the first non-common UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_common_uid_found(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen /* mismatch and we want to revert local changes -
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen need to delete the mailbox. */
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen dsync_mailbox_revert_existing_uid(importer, change->uid, *result_r);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen /* mail exists remotely, but doesn't exist locally. */
2e652d2651b2800f99a17dcb3014a009fe4660d3Timo Sirainen if (!dsync_mailbox_import_want_change(importer, change, result_r))
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen return;
f6a50ab94799b632c6fc95ec73bfed399ead4e19Timo Sirainen if (importer->revert_local_changes &&
f6a50ab94799b632c6fc95ec73bfed399ead4e19Timo Sirainen change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen dsync_mailbox_revert_missing(importer, change);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen *result_r = "Reverting local change by deleting mailbox";
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen } else {
34422d89f9e4adeb367fc85ff4f71db0137a7c20Timo Sirainen (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen }
bd78cb1d7e05956b08a2031b8618b8fb97c9991cTimo Sirainen *result_r = t_strdup_printf("%s (next local mail UID=%u)",
37312679c89109e8762c3ec33da3dfac4c4177b8Timo Sirainen *result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainenint dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen const char *result;
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(!importer->new_uids_assigned);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(importer->prev_uid < change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->prev_uid = change->uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (importer->failed)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return -1;
805d7834412465268486c50711962407ad13fbf6Timo Sirainen if (importer->require_full_resync)
805d7834412465268486c50711962407ad13fbf6Timo Sirainen return 0;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
e04e2ca2a536e2d1ea0c3d4dde68902ab14ea6c7Timo Sirainen if (!importer->last_common_uid_found) {
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen result = NULL;
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen dsync_mailbox_find_common_uid(importer, change, &result);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen i_assert(result != NULL);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen } else {
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen result = "New mail";
e04e2ca2a536e2d1ea0c3d4dde68902ab14ea6c7Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
589bc28b5489c76626f022adc74a5324ae02996aTimo Sirainen imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s",
589bc28b5489c76626f022adc74a5324ae02996aTimo Sirainen dsync_mail_change_type_names[change->type],
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen change->guid != NULL ? change->guid : "<unknown>", change->uid,
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen change->hdr_hash != NULL ? change->hdr_hash : "", result);
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen if (importer->failed)
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen return -1;
805d7834412465268486c50711962407ad13fbf6Timo Sirainen if (importer->require_full_resync)
805d7834412465268486c50711962407ad13fbf6Timo Sirainen return 0;
3df1930bcd45556e7f8031365ce505199c5dc4bdTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->last_common_uid_found) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* a) uid <= last_common_uid for flag changes and expunges.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen this happens only when last_common_uid was originally given
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen as parameter to importer.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen when we're finding the last_common_uid ourself,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uid>last_common_uid always in here, because
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last_common_uid_found=TRUE only after we find the first
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mismatch.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen b) uid > last_common_uid for i) new messages, ii) expunges
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen that were sent "just in case" */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid <= importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* ignore */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* a) uid < last_common_uid can never happen */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->uid >= importer->last_common_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* b) uid = last_common_uid if we've verified that the
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen messages' GUIDs match so far.
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen c) uid > last_common_uid: i) TYPE_EXPUNGE change has
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen GUID=NULL, so we couldn't verify yet if it matches our
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen local message, ii) local message is expunged and we couldn't
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen find its GUID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->uid > importer->last_common_uid) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen importer->cur_mail == NULL ||
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->uid < importer->cur_mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen switch (change->type) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen case DSYNC_MAIL_CHANGE_TYPE_SAVE:
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_save(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE:
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_expunge(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE:
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(importer->last_common_uid_found);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_flag_change(importer, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return importer->failed ? -1 : 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainenstatic int
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainenimporter_new_mail_final_uid_cmp(struct importer_new_mail *const *newmail1,
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen struct importer_new_mail *const *newmail2)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen{
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if ((*newmail1)->final_uid < (*newmail2)->final_uid)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen return -1;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if ((*newmail1)->final_uid > (*newmail2)->final_uid)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen return 1;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen return 0;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen}
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *newmail, *const *newmailp;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t common_uid_next, new_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen common_uid_next = I_MAX(importer->local_uid_next,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->remote_uid_next);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_foreach_modifiable(&importer->newmails, newmailp) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail = *newmailp;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (newmail->skip) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* already assigned */
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen i_assert(newmail->final_uid != 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen continue;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* figure out what UID to use for the mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (newmail->uid_is_usable) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* keep the UID */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen new_uid = newmail->final_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (newmail->link != NULL &&
c45baa023828a14f440eb184c24d186ffe8666edTimo Sirainen newmail->link->uid_is_usable) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* we can use the linked message's UID and expunge
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen this mail */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen new_uid = newmail->link->final_uid;
c45baa023828a14f440eb184c24d186ffe8666edTimo Sirainen } else {
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen i_assert(!importer->revert_local_changes);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen new_uid = common_uid_next++;
da215fe5cde683b7d1fe2339e7747c1b14bff04aTimo Sirainen imp_debug(importer, "UID %u isn't usable, assigning new UID %u",
da215fe5cde683b7d1fe2339e7747c1b14bff04aTimo Sirainen newmail->final_uid, new_uid);
c45baa023828a14f440eb184c24d186ffe8666edTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen newmail->final_uid = new_uid;
fadb1a39a4d7da6592b4cc11ae401d2763ba66b4Timo Sirainen if (newmail->link != NULL && newmail->link != newmail) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* skip processing the linked mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen newmail->link->skip = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen importer->last_common_uid = common_uid_next-1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->new_uids_assigned = TRUE;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen /* Sort the newmails by their final_uid. This is used for tracking
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen whether an intermediate commit is allowed. */
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen array_sort(&importer->newmails, importer_new_mail_final_uid_cmp);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainenstatic int
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainendsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mail *mail, uint32_t uid, const char *guid,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen struct dsync_mail *dmail_r)
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen{
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen const char *error_field, *errstr;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen enum mail_error error;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (!mail_set_uid(mail, uid))
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return 0;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen /* NOTE: Errors are logged, but they don't cause the entire import
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen to fail. */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mail_fill(mail, TRUE, dmail_r, &error_field) < 0) {
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi errstr = mailbox_get_last_internal_error(mail->box, &error);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (error == MAIL_ERROR_EXPUNGED)
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return 0;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mailbox_get_vname(importer->box),
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen error_field, uid, errstr);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return -1;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (*guid != '\0' && strcmp(guid, dmail_r->guid) != 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_import_unexpected_state(importer, t_strdup_printf(
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen "Unexpected GUID mismatch (3) for UID=%u: %s != %s",
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen uid, dmail_r->guid, guid));
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return -1;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return 1;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen}
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainenstatic void
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainendsync_mailbox_import_saved_uid(struct dsync_mailbox_importer *importer,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen uint32_t uid)
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen{
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen i_assert(importer->search_ctx == NULL);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen if (importer->highest_wanted_uid < uid)
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen importer->highest_wanted_uid = uid;
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen array_append(&importer->wanted_uids, &uid, 1);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen}
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainenstatic void
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainendsync_mailbox_import_update_first_saved(struct dsync_mailbox_importer *importer)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen{
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen struct importer_new_mail *const *newmails;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen unsigned int count;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen newmails = array_get(&importer->newmails, &count);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen while (importer->first_unsaved_idx < count) {
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if (!newmails[importer->first_unsaved_idx]->saved)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen break;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen importer->first_unsaved_idx++;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen }
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen}
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainenstatic void
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainendsync_mailbox_import_saved_newmail(struct dsync_mailbox_importer *importer,
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen struct importer_new_mail *newmail)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen{
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen dsync_mailbox_import_saved_uid(importer, newmail->final_uid);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen newmail->saved = TRUE;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen dsync_mailbox_import_update_first_saved(importer);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen importer->saves_since_commit++;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen /* we can commit only if all the upcoming mails will have UIDs that
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen are larger than we're committing.
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen Note that if any existing UIDs have been changed, the new UID is
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen usually higher than anything that is being saved so we can't do
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen an intermediate commit. It's too much extra work to try to handle
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen that situation. So here this never happens, because then
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen array_count(wanted_uids) is always higher than first_unsaved_idx. */
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if (importer->saves_since_commit >= importer->commit_msgs_interval &&
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen importer->first_unsaved_idx == array_count(&importer->wanted_uids)) {
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen if (dsync_mailbox_import_commit(importer, FALSE) < 0)
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen importer->failed = TRUE;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen importer->saves_since_commit = 0;
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen }
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen}
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainenstatic bool
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainendsync_msg_change_uid(struct dsync_mailbox_importer *importer,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen uint32_t old_uid, uint32_t new_uid)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen{
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen struct mail_save_context *save_ctx;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen IMPORTER_DEBUG_CHANGE(importer);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!mail_set_uid(importer->mail, old_uid))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return FALSE;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen save_ctx = mailbox_save_alloc(importer->ext_trans);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen mailbox_save_copy_flags(save_ctx, importer->mail);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen mailbox_save_set_uid(save_ctx, new_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (mailbox_move(&save_ctx, importer->mail) < 0)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return FALSE;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen dsync_mailbox_import_saved_uid(importer, new_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return TRUE;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen}
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainenstatic bool
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainendsync_mailbox_import_change_uid(struct dsync_mailbox_importer *importer,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen ARRAY_TYPE(seq_range) *unwanted_uids,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen uint32_t wanted_uid)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen{
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen const struct seq_range *range;
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen unsigned int count, n;
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen struct seq_range_iter iter;
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen uint32_t uid;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen /* optimize by first trying to use the latest UID */
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen range = array_get(unwanted_uids, &count);
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (count == 0)
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen return FALSE;
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (dsync_msg_change_uid(importer, range[count-1].seq2, wanted_uid)) {
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen seq_range_array_remove(unwanted_uids, range[count-1].seq2);
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen return TRUE;
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen }
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen seq_range_array_remove(unwanted_uids, range[count-1].seq2);
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen /* now try to use any of them by iterating through them. (would be
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen easier&faster to just iterate backwards, but probably too much
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen trouble to add such API) */
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen n = 0; seq_range_array_iter_init(&iter, unwanted_uids);
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen while (seq_range_array_iter_nth(&iter, n++, &uid)) {
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (dsync_msg_change_uid(importer, uid, wanted_uid)) {
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen seq_range_array_remove(unwanted_uids, uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return TRUE;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
add74aa767e4b6aaa08e3a389022883f0dd3f43dTimo Sirainen seq_range_array_remove(unwanted_uids, uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return FALSE;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen}
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainenstatic bool
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainendsync_mailbox_import_try_local(struct dsync_mailbox_importer *importer,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen struct importer_new_mail *all_newmails,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen ARRAY_TYPE(seq_range) *local_uids,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen ARRAY_TYPE(seq_range) *wanted_uids)
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen{
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen ARRAY_TYPE(seq_range) assigned_uids, unwanted_uids;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen struct seq_range_iter local_iter, wanted_iter;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen unsigned int local_n, wanted_n;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen uint32_t local_uid, wanted_uid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen struct importer_new_mail *mail;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen struct dsync_mail dmail;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (array_count(local_uids) == 0)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return FALSE;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen local_n = wanted_n = 0;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_iter_init(&local_iter, local_uids);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_iter_init(&wanted_iter, wanted_uids);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* wanted_uids contains UIDs that need to exist at the end. those that
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen don't already exist in local_uids have a higher UID than any
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen existing local UID */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen t_array_init(&assigned_uids, array_count(wanted_uids));
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen t_array_init(&unwanted_uids, 8);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (seq_range_array_iter_nth(&wanted_iter, wanted_n,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen &wanted_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (local_uid == wanted_uid) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* we have exactly the UID we want. keep it. */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&assigned_uids, wanted_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen wanted_n++;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen continue;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen i_assert(local_uid < wanted_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* we no longer want this local UID. */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&unwanted_uids, local_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* reuse as many existing messages as possible by changing their UIDs */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen while (seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!dsync_mailbox_import_change_uid(importer, &unwanted_uids,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen wanted_uid))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen break;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&assigned_uids, wanted_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen wanted_n++;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* expunge all unwanted messages */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen local_n = 0; seq_range_array_iter_init(&local_iter, &unwanted_uids);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen IMPORTER_DEBUG_CHANGE(importer);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (mail_set_uid(importer->mail, local_uid))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen mail_expunge(importer->mail);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* mark mails whose UIDs we got to be skipped over later */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen for (mail = all_newmails; mail != NULL; mail = mail->next) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!mail->skip &&
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_exists(&assigned_uids, mail->final_uid))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen mail->skip = TRUE;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* we've assigned all wanted UIDs */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return TRUE;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* try to find one existing message that we can use to copy to the
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen other instances */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen local_n = 0; seq_range_array_iter_init(&local_iter, local_uids);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mailbox_import_local_uid(importer, importer->mail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen local_uid, all_newmails->guid,
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen &dmail) > 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mailbox_save_newmails(importer, &dmail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen all_newmails, FALSE))
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return TRUE;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return FALSE;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen}
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainenstatic bool
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainendsync_mailbox_import_try_virtual_all(struct dsync_mailbox_importer *importer,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct importer_new_mail *all_newmails)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen{
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct dsync_mail dmail;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (all_newmails->virtual_all_uid == 0)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return FALSE;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mailbox_import_local_uid(importer, importer->virtual_mail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen all_newmails->virtual_all_uid,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen all_newmails->guid, &dmail) > 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (dsync_mailbox_save_newmails(importer, &dmail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen all_newmails, FALSE))
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return TRUE;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen }
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return FALSE;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen}
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainenstatic bool
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainendsync_mailbox_import_handle_mail(struct dsync_mailbox_importer *importer,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen struct importer_new_mail *all_newmails)
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen{
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen ARRAY_TYPE(seq_range) local_uids, wanted_uids;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen struct dsync_mail_request *request;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen struct importer_new_mail *mail;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen const char *request_guid = NULL;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen uint32_t request_uid = 0;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
791fb70b3255a11a91ce0c2dc3ae1460d4cf8459Timo Sirainen i_assert(all_newmails != NULL);
791fb70b3255a11a91ce0c2dc3ae1460d4cf8459Timo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* get the list of the current local UIDs and the wanted UIDs.
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen find the first remote instance that we can request in case there are
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen no local instances */
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen t_array_init(&local_uids, 8);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen t_array_init(&wanted_uids, 8);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen for (mail = all_newmails; mail != NULL; mail = mail->next) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (mail->uid_in_local)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&local_uids, mail->local_uid);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen else if (request_guid == NULL) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (*mail->guid != '\0')
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen request_guid = mail->guid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen request_uid = mail->remote_uid;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_assert(request_uid != 0);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!mail->skip)
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen seq_range_array_add(&wanted_uids, mail->final_uid);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen i_assert(array_count(&wanted_uids) > 0);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!dsync_mailbox_import_try_local(importer, all_newmails,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen &local_uids, &wanted_uids) &&
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen !dsync_mailbox_import_try_virtual_all(importer, all_newmails)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* no local instance. request from remote */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen IMPORTER_DEBUG_CHANGE(importer);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (importer->want_mail_requests) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen request = array_append_space(&importer->mail_requests);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen request->guid = request_guid;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen request->uid = request_uid;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return FALSE;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen /* successfully handled all the mails locally */
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen importer->import_pos++;
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen return TRUE;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen}
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainenstatic void
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainendsync_mailbox_import_find_virtual_uids(struct dsync_mailbox_importer *importer)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen{
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mail_search_context *search_ctx;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mail_search_args *search_args;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct importer_new_mail *newmail;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct mail *mail;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen const char *guid;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (mailbox_sync(importer->virtual_all_box, 0) < 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen i_error("Couldn't sync \\All mailbox '%s': %s",
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen mailbox_get_vname(importer->virtual_all_box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen }
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen search_args = mail_search_build_init();
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen mail_search_build_add_all(search_args);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen importer->virtual_trans =
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen mailbox_transaction_begin(importer->virtual_all_box,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi importer->transaction_flags,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi __func__);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen search_ctx = mailbox_search_init(importer->virtual_trans, search_args,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen NULL, MAIL_FETCH_GUID, NULL);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen mail_search_args_unref(&search_args);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen while (mailbox_search_next(search_ctx, &mail)) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen /* ignore errors */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen continue;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen }
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen newmail = hash_table_lookup(importer->import_guids, guid);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (newmail != NULL && newmail->virtual_all_uid == 0)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen newmail->virtual_all_uid = mail->uid;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen }
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (mailbox_search_deinit(&search_ctx) < 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen i_error("Couldn't search \\All mailbox '%s': %s",
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen mailbox_get_vname(importer->virtual_all_box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen }
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen importer->virtual_mail = mail_alloc(importer->virtual_trans, 0, NULL);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen}
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainenstatic void
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainendsync_mailbox_import_handle_local_mails(struct dsync_mailbox_importer *importer)
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen{
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen struct hash_iterate_context *iter;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen const char *key;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen void *key2;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen struct importer_new_mail *mail;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (importer->virtual_all_box != NULL &&
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen hash_table_count(importer->import_guids) > 0) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen /* find UIDs in \All mailbox for all wanted GUIDs. */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen dsync_mailbox_import_find_virtual_uids(importer);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen }
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen iter = hash_table_iterate_init(importer->import_guids);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen T_BEGIN {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (dsync_mailbox_import_handle_mail(importer, mail))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen hash_table_remove(importer->import_guids, key);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen } T_END;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen hash_table_iterate_deinit(&iter);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen iter = hash_table_iterate_init(importer->import_uids);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen while (hash_table_iterate(iter, importer->import_uids, &key2, &mail)) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen T_BEGIN {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (dsync_mailbox_import_handle_mail(importer, mail))
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen hash_table_remove(importer->import_uids, key2);
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen } T_END;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen hash_table_iterate_deinit(&iter);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen}
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainenint dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(!importer->new_uids_assigned);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!importer->last_common_uid_found) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* handle pending expunges and flag updates */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_common_uid_found(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* skip common local mails */
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)importer_next_mail(importer, importer->last_common_uid+1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* if there are any local mails left, add them to newmails list */
faca9429a68ada074390da8745cf5ee0c958b42bTimo Sirainen while (importer->cur_mail != NULL && !importer->failed)
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)dsync_mailbox_try_save(importer, NULL);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (importer->search_ctx != NULL) {
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (mailbox_search_deinit(&importer->search_ctx) < 0) {
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen i_error("Mailbox %s: Search failed: %s",
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->failed = TRUE;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen }
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen }
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen importer->import_count = hash_table_count(importer->import_guids) +
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen hash_table_count(importer->import_uids);
933e663cc1805e11d42d4c5ad69038c0866c56a8Timo Sirainen
933e663cc1805e11d42d4c5ad69038c0866c56a8Timo Sirainen dsync_mailbox_import_assign_new_uids(importer);
933e663cc1805e11d42d4c5ad69038c0866c56a8Timo Sirainen /* save mails from local sources where possible,
933e663cc1805e11d42d4c5ad69038c0866c56a8Timo Sirainen request the rest from remote */
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen if (!importer->failed)
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen dsync_mailbox_import_handle_local_mails(importer);
69f6407b20f623193981d672c26fa722ee75d941Timo Sirainen return importer->failed ? -1 : 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenconst struct dsync_mail_request *
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail_request *requests;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen requests = array_get(&importer->mail_requests, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->mail_request_idx == count)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return &requests[importer->mail_request_idx++];
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic const char *const *
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_get_final_keywords(const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(const_string) keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!array_is_created(&change->keyword_changes))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(&change->keyword_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen t_array_init(&keywords, count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (changes[i][0] == KEYWORD_CHANGE_ADD ||
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen changes[i][0] == KEYWORD_CHANGE_ADD_AND_FINAL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *name = changes[i]+1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&keywords, &name, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_count(&keywords) == 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen array_append_zero(&keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return array_idx(&keywords, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_save_context *save_ctx,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *keyword_names;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_keywords *keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keyword_names = dsync_mailbox_get_final_keywords(change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keywords = keyword_names == NULL ? NULL :
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_keywords_create_valid(importer->box,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keyword_names);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_flags(save_ctx, change->final_flags, keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (keywords != NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_keywords_unref(&keywords);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->modseq > 1) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_min_modseq(save_ctx, change->modseq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen /* FIXME: if there already are private flags, they get lost because
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen saving can't handle updating private index. they get added on the
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen next sync though. if this is fixed here, set min_pvt_modseq also. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_msg_try_copy(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_save_context **save_ctx_p,
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen struct importer_new_mail **all_newmails_forcopy)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *inst;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen for (inst = *all_newmails_forcopy; inst != NULL; inst = inst->next) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (inst->uid_in_local && !inst->copy_failed &&
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mail_set_uid(importer->mail, inst->local_uid)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_copy(save_ctx_p, importer->mail) < 0) {
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen enum mail_error error;
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen const char *errstr;
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen errstr = mailbox_get_last_internal_error(importer->box, &error);
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen if (error != MAIL_ERROR_EXPUNGED) {
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen i_warning("Failed to copy mail from UID=%u: "
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen "%s - falling back to other means",
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen inst->local_uid, errstr);
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen inst->copy_failed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen *all_newmails_forcopy = inst;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen *all_newmails_forcopy = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainenstatic void
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainendsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx,
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen const struct dsync_mail *mail)
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen{
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (mail->pop3_order > 0)
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen}
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic struct mail_save_context *
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail *mail,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct importer_new_mail *newmail)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_save_context *save_ctx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = mailbox_save_alloc(importer->ext_trans);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mailbox_save_set_uid(save_ctx, newmail->final_uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (*mail->guid != '\0')
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_set_guid(save_ctx, mail->guid);
8bc87e22fecd20797112b86778a961b08dc1f5c8Timo Sirainen if (mail->saved_date != 0)
8bc87e22fecd20797112b86778a961b08dc1f5c8Timo Sirainen mailbox_save_set_save_date(save_ctx, mail->saved_date);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (!mail->minimal_fields)
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen dsync_mailbox_save_set_nonminimal(save_ctx, mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return save_ctx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainenstatic bool
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainendsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen const struct dsync_mail *mail,
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen struct importer_new_mail *newmail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct importer_new_mail **all_newmails_forcopy,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen bool remote_mail)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_save_context *save_ctx;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen struct istream *input;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ssize_t ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen bool save_failed = FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* try to save the mail by copying an existing mail */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails_forcopy)) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (save_ctx == NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen if (ret <= 0 && mail->input_mail != NULL) {
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen /* copy using the source mail */
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen i_assert(mail->input_mail->uid == mail->input_mail_uid);
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen if (mailbox_copy(&save_ctx, mail->input_mail) == 0)
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen ret = 1;
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen else {
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen enum mail_error error;
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen const char *errstr;
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen errstr = mailbox_get_last_internal_error(importer->box, &error);
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen if (error != MAIL_ERROR_EXPUNGED) {
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen i_warning("Failed to copy source UID=%u mail: "
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen "%s - falling back to regular saving",
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen mail->input_mail->uid, errstr);
2da20ac5a4af910deebf4312469ba925bb88f432Timo Sirainen }
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen ret = -1;
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen }
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen
f5d3d7a0e1e517c6dff9bb1750334c8d9e4dce99Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (ret > 0) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen i_assert(save_ctx == NULL);
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen dsync_mailbox_import_saved_newmail(importer, newmail);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* fallback to saving from remote stream */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (!remote_mail) {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen /* the mail isn't remote yet. we were just trying to copy a
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen local mail to avoid downloading the remote mail. */
57c5e11941482d3603fc03482f78e07fa0f81ee5Timo Sirainen mailbox_save_cancel(&save_ctx);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return FALSE;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen }
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (mail->minimal_fields) {
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen struct dsync_mail mail2;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen const char *error_field;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen i_assert(mail->input_mail != NULL);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2,
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen &error_field) < 0) {
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen i_error("Mailbox %s: Failed to read mail %s uid=%u: %s",
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen mailbox_get_vname(importer->box),
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen error_field, mail->uid,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen importer->failed = TRUE;
ab841a241463160492bb4b8e41039752804b33b8Timo Sirainen mailbox_save_cancel(&save_ctx);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return TRUE;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen }
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen dsync_mailbox_save_set_nonminimal(save_ctx, &mail2);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen input = mail2.input;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen } else {
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen input = mail->input;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (input == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* it was just expunged in remote, skip it */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_cancel(&save_ctx);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen i_stream_seek(input, 0);
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (mailbox_save_begin(&save_ctx, input) < 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Saving failed: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->failed = TRUE;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen while ((ret = i_stream_read(input)) > 0 || ret == -2) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_save_continue(save_ctx) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_failed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ret = -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(ret == -1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (input->stream_errno != 0) {
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen i_error("Mailbox %s: read(msg input) failed: %s",
0afe75cd8ea1814ba5711196ef749f93a140c01cTimo Sirainen mailbox_get_vname(importer->box),
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen i_stream_get_error(input));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_cancel(&save_ctx);
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen importer->mail_error = MAIL_ERROR_TEMP;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->failed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (save_failed) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Saving failed: %s",
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_cancel(&save_ctx);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->failed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen i_assert(input->eof);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_save_finish(&save_ctx) < 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Saving failed: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->failed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
a76faea3eb26c4cd67886fbe02c604f74d54be8cTimo Sirainen dsync_mailbox_import_saved_newmail(importer, newmail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainenstatic bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen const struct dsync_mail *mail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen struct importer_new_mail *all_newmails,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen bool remote_mail)
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen{
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen struct importer_new_mail *newmail, *all_newmails_forcopy;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen bool ret = TRUE;
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen /* if all_newmails list is large, avoid scanning through the
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen uninteresting ones for each newmail */
bb4045cb95c33c3a888c50dce461554391129794Timo Sirainen all_newmails_forcopy = all_newmails;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen /* save all instances of the message */
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen for (newmail = all_newmails; newmail != NULL && ret; newmail = newmail->next) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (!newmail->skip) T_BEGIN {
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (!dsync_mailbox_save_body(importer, mail, newmail,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen &all_newmails_forcopy,
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen remote_mail))
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen ret = FALSE;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen } T_END;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen return ret;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen}
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
7ae6552482fe6e6a613f883335fdafdf9afbc7b7Timo Sirainenint dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
7ae6552482fe6e6a613f883335fdafdf9afbc7b7Timo Sirainen const struct dsync_mail *mail)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen struct importer_new_mail *all_newmails;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
89f0ab8c7daae308fbcb1c8537ee415d236816c0Timo Sirainen i_assert(mail->input == NULL || mail->input->seekable);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(importer->new_uids_assigned);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
b6d4436601d4371b4fbf5347bcaaad4f5e114671Timo Sirainen if (importer->failed)
7ae6552482fe6e6a613f883335fdafdf9afbc7b7Timo Sirainen return -1;
805d7834412465268486c50711962407ad13fbf6Timo Sirainen if (importer->require_full_resync)
805d7834412465268486c50711962407ad13fbf6Timo Sirainen return 0;
b6d4436601d4371b4fbf5347bcaaad4f5e114671Timo Sirainen
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Import mail body for GUID=%s UID=%u",
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen mail->guid, mail->uid);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen all_newmails = *mail->guid != '\0' ?
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_lookup(importer->import_guids, mail->guid) :
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid));
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (all_newmails == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (importer->want_mail_requests) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Remote sent unwanted message body for "
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "GUID=%s UID=%u",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_vname(importer->box),
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail->guid, mail->uid);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen } else {
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Skip unwanted mail body for "
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen "GUID=%s UID=%u", mail->guid, mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7ae6552482fe6e6a613f883335fdafdf9afbc7b7Timo Sirainen return 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (*mail->guid != '\0')
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_remove(importer->import_guids, mail->guid);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen else {
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_remove(importer->import_uids,
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen POINTER_CAST(mail->uid));
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen }
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen importer->import_pos++;
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (!dsync_mailbox_save_newmails(importer, mail, all_newmails, TRUE))
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen i_unreached();
7ae6552482fe6e6a613f883335fdafdf9afbc7b7Timo Sirainen return importer->failed ? -1 : 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int
ec047a9c54a02338e85fb1767120b0923f6d4148Timo Sirainenreassign_uids_in_seq_range(struct dsync_mailbox_importer *importer,
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen const ARRAY_TYPE(seq_range) *unwanted_uids)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
ec047a9c54a02338e85fb1767120b0923f6d4148Timo Sirainen struct mailbox *box = importer->box;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const enum mailbox_transaction_flags trans_flags =
b0ead8fffe8dfdee92fca875b91fadf3f3f2262bTimo Sirainen importer->transaction_flags |
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen MAILBOX_TRANSACTION_FLAG_EXTERNAL |
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mailbox_transaction_context *trans;
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen struct mail_search_args *search_args;
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen struct mail_search_arg *arg;
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen struct mail_search_context *search_ctx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_save_context *save_ctx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail;
ec047a9c54a02338e85fb1767120b0923f6d4148Timo Sirainen unsigned int renumber_count = 0;
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen int ret = 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen if (array_count(unwanted_uids) == 0)
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen if (importer->debug) T_BEGIN {
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen string_t *str = t_str_new(256);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imap_write_seq_range(str, unwanted_uids);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Reassign UIDs: %s", str_c(str));
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen } T_END;
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen search_args = mail_search_build_init();
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen arg = mail_search_build_add(search_args, SEARCH_UIDSET);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen p_array_init(&arg->value.seqset, search_args->pool,
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen array_count(unwanted_uids));
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen array_append_array(&arg->value.seqset, unwanted_uids);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi trans = mailbox_transaction_begin(box, trans_flags, __func__);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen mail_search_args_unref(&search_args);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen while (mailbox_search_next(search_ctx, &mail)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen save_ctx = mailbox_save_alloc(trans);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_save_copy_flags(save_ctx, mail);
3e6903afa5bd125e8bb14f996a558e032674dfbfTimo Sirainen if (mailbox_move(&save_ctx, mail) < 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Couldn't move mail within mailbox: %s",
3e6903afa5bd125e8bb14f996a558e032674dfbfTimo Sirainen mailbox_get_vname(box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(box, &importer->mail_error));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ret = -1;
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen } else if (ret > 0) {
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen ret = 0;
3e6903afa5bd125e8bb14f996a558e032674dfbfTimo Sirainen }
ec047a9c54a02338e85fb1767120b0923f6d4148Timo Sirainen renumber_count++;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen if (mailbox_search_deinit(&search_ctx) < 0) {
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen i_error("Mailbox %s: mail search failed: %s",
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen mailbox_get_vname(box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(box, &importer->mail_error));
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen ret = -1;
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_transaction_commit(&trans) < 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: UID reassign commit failed: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_vname(box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(box, &importer->mail_error));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ret = -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
311cbd9b1c81df050f5019600a2a809bc620be00Timo Sirainen if (ret == 0) {
311cbd9b1c81df050f5019600a2a809bc620be00Timo Sirainen imp_debug(importer, "Mailbox %s: Change during sync: "
311cbd9b1c81df050f5019600a2a809bc620be00Timo Sirainen "Renumbered %u of %u unwanted UIDs",
311cbd9b1c81df050f5019600a2a809bc620be00Timo Sirainen mailbox_get_vname(box),
311cbd9b1c81df050f5019600a2a809bc620be00Timo Sirainen renumber_count, array_count(unwanted_uids));
ec047a9c54a02338e85fb1767120b0923f6d4148Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
91ec3b77f0bb5365ff9601422f09c729a0e19897Timo Sirainenstatic int
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenreassign_unwanted_uids(struct dsync_mailbox_importer *importer,
4f7951e71128c120d8a502d6406cc603fcc8eb0bTimo Sirainen const char **changes_during_sync_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen ARRAY_TYPE(seq_range) unwanted_uids;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen const uint32_t *wanted_uids, *saved_uids;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen uint32_t highest_seen_uid;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen unsigned int i, wanted_count, saved_count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen int ret = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen wanted_uids = array_get(&importer->wanted_uids, &wanted_count);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen saved_uids = array_get(&importer->saved_uids, &saved_count);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen i_assert(wanted_count == saved_count);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (wanted_count == 0)
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen return 0;
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen /* wanted_uids contains the UIDs we tried to save mails with.
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen if nothing changed during dsync, we should have the expected UIDs
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen (saved_uids) and all is well.
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen if any new messages got inserted during dsync, we'll need to fix up
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen the UIDs and let the next dsync fix up the other side. for example:
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen remote uids = 5,7,9 = wanted_uids
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen remote uidnext = 12
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen locally added new uid=5 ->
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen saved_uids = 10,7,9
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen we'll now need to reassign UIDs 5 and 10. to be fully future-proof
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen we'll reassign all UIDs between [original local uidnext .. highest
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen UID we think we know] that aren't in saved_uids. */
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen /* create uidset for the list of UIDs we don't want to exist */
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen t_array_init(&unwanted_uids, 8);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen highest_seen_uid = I_MAX(importer->remote_uid_next-1,
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen importer->highest_wanted_uid);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen i_assert(importer->local_uid_next <= highest_seen_uid);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen seq_range_array_add_range(&unwanted_uids,
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen importer->local_uid_next, highest_seen_uid);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen for (i = 0; i < wanted_count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(i < wanted_count);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (saved_uids[i] == wanted_uids[i])
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen seq_range_array_remove(&unwanted_uids, saved_uids[i]);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
14460e21d9acf5812876832c53f7625b23a0bd69Timo Sirainen
ec047a9c54a02338e85fb1767120b0923f6d4148Timo Sirainen ret = reassign_uids_in_seq_range(importer, &unwanted_uids);
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen if (ret == 0) {
4f7951e71128c120d8a502d6406cc603fcc8eb0bTimo Sirainen *changes_during_sync_r = t_strdup_printf(
4f7951e71128c120d8a502d6406cc603fcc8eb0bTimo Sirainen "%u UIDs changed due to UID conflicts",
4f7951e71128c120d8a502d6406cc603fcc8eb0bTimo Sirainen seq_range_count(&unwanted_uids));
264c6bd26df5380dc52687f6ec0351354012a948Timo Sirainen /* conflicting changes during sync, revert our last-common-uid
264c6bd26df5380dc52687f6ec0351354012a948Timo Sirainen back to a safe value. */
264c6bd26df5380dc52687f6ec0351354012a948Timo Sirainen importer->last_common_uid = importer->local_uid_next - 1;
264c6bd26df5380dc52687f6ec0351354012a948Timo Sirainen }
8b3a4836da0b032673918941cb49c956d3b89b25Timo Sirainen return ret < 0 ? -1 : 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainenstatic int
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainendsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, bool final)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_transaction_commit_changes changes;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen struct seq_range_iter iter;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen uint32_t uid;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen unsigned int n;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen int ret = importer->failed ? -1 : 0;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen mail_free(&importer->mail);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen mail_free(&importer->ext_mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* commit saves */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_transaction_commit_get_changes(&importer->ext_trans,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen &changes) < 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Save commit failed: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box, &importer->mail_error));
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* removed wanted_uids that weren't actually saved */
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen array_delete(&importer->wanted_uids,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen array_count(&importer->saved_uids),
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen array_count(&importer->wanted_uids) -
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen array_count(&importer->saved_uids));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_transaction_rollback(&importer->trans);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen ret = -1;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen } else {
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* remember the UIDs that were successfully saved */
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen if (importer->debug) T_BEGIN {
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen string_t *str = t_str_new(256);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imap_write_seq_range(str, &changes.saved_uids);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Saved UIDs: %s", str_c(str));
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen } T_END;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen seq_range_array_iter_init(&iter, &changes.saved_uids); n = 0;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen while (seq_range_array_iter_nth(&iter, n++, &uid))
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen array_append(&importer->saved_uids, &uid, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool_unref(&changes.pool);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* commit flag changes and expunges */
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (mailbox_transaction_commit(&importer->trans) < 0) {
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen i_error("Mailbox %s: Commit failed: %s",
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen ret = -1;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (!final)
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen dsync_mailbox_import_transaction_begin(importer);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen return ret;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainenstatic int dsync_mailbox_import_finish(struct dsync_mailbox_importer *importer,
4f7951e71128c120d8a502d6406cc603fcc8eb0bTimo Sirainen const char **changes_during_sync_r)
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen{
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen struct mailbox_update update;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen int ret;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen ret = dsync_mailbox_import_commit(importer, TRUE);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (ret == 0) {
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* update mailbox metadata if we successfully saved
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen everything. */
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&update);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen update.min_next_uid = importer->remote_uid_next;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen update.min_first_recent_uid =
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen I_MIN(importer->last_common_uid+1,
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen importer->remote_first_recent_uid);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen update.min_highest_modseq = importer->remote_highest_modseq;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen imp_debug(importer, "Finish update: min_next_uid=%u "
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi "min_first_recent_uid=%u min_highest_modseq=%"PRIu64" "
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi "min_highest_pvt_modseq=%"PRIu64,
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen update.min_next_uid, update.min_first_recent_uid,
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi update.min_highest_modseq,
47a5a7e8296f3b8f2fac9a0659d4de3f2723ba4aMartti Rannanjärvi update.min_highest_pvt_modseq);
31d32d39dd09be0625a6d92ee715155f5d679515Timo Sirainen
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (mailbox_update(importer->box, &update) < 0) {
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen i_error("Mailbox %s: Update failed: %s",
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen ret = -1;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* sync mailbox to finish flag changes and expunges. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_sync(importer->box, 0) < 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Sync failed: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ret = -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (ret == 0) {
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen /* give new UIDs to messages that got saved with unwanted UIDs.
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen do it only if the whole transaction succeeded. */
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (reassign_unwanted_uids(importer, changes_during_sync_r) < 0)
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen ret = -1;
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainenstatic void
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainendsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer)
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen{
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen struct hash_iterate_context *iter;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen const char *key;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen struct importer_new_mail *mail;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen iter = hash_table_iterate_init(importer->import_guids);
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen for (; mail != NULL; mail = mail->next) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (mail->skip)
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen continue;
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)",
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen mailbox_get_vname(importer->box),
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mail->guid, mail->remote_uid);
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen importer->mail_error = MAIL_ERROR_TEMP;
9e992397c2d66eea80177357323a6ce3f042097dTimo Sirainen importer->failed = TRUE;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen }
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen }
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_iterate_deinit(&iter);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen}
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainenstatic void
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainendsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct hash_iterate_context *iter;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen void *key;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen struct importer_new_mail *mail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen iter = hash_table_iterate_init(importer->import_uids);
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (; mail != NULL; mail = mail->next) {
0c47c2096714b50880d48d00ce0bf28349eb4aceTimo Sirainen if (mail->skip)
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen continue;
307ec6c2c319e3335ddb1a7aca2d2884fe17fae0Timo Sirainen
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen i_error("Mailbox %s: Remote didn't send mail UID=%u",
1c95b8403d2d4dcf35e23fc0a1b51922f120a82aTimo Sirainen mailbox_get_vname(importer->box),
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mail->remote_uid);
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen importer->mail_error = MAIL_ERROR_TEMP;
9e992397c2d66eea80177357323a6ce3f042097dTimo Sirainen importer->failed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_iterate_deinit(&iter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenint dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen bool success,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t *last_common_uid_r,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint64_t *last_common_modseq_r,
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen uint64_t *last_common_pvt_modseq_r,
7d315281ae13a66e13da2b1ad006bdb883018278Timo Sirainen uint32_t *last_messages_count_r,
4f7951e71128c120d8a502d6406cc603fcc8eb0bTimo Sirainen const char **changes_during_sync_r,
805d7834412465268486c50711962407ad13fbf6Timo Sirainen bool *require_full_resync_r,
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen enum mail_error *error_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mailbox_importer *importer = *_importer;
7d315281ae13a66e13da2b1ad006bdb883018278Timo Sirainen struct mailbox_status status;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen int ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *_importer = NULL;
4f7951e71128c120d8a502d6406cc603fcc8eb0bTimo Sirainen *changes_during_sync_r = NULL;
805d7834412465268486c50711962407ad13fbf6Timo Sirainen *require_full_resync_r = importer->require_full_resync;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
805d7834412465268486c50711962407ad13fbf6Timo Sirainen if ((!success || importer->require_full_resync) && !importer->failed) {
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen importer->mail_error = MAIL_ERROR_TEMP;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen importer->failed = TRUE;
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen }
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (!importer->new_uids_assigned && !importer->failed)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_import_assign_new_uids(importer);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (!importer->failed) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_mailbox_import_check_missing_guid_imports(importer);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen dsync_mailbox_import_check_missing_uid_imports(importer);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen if (importer->search_ctx != NULL) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (mailbox_search_deinit(&importer->search_ctx) < 0) {
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen i_error("Mailbox %s: Search failed: %s",
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen importer->failed = TRUE;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen }
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen }
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen if (dsync_mailbox_import_finish(importer, changes_during_sync_r) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen importer->failed = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (importer->virtual_mail != NULL)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen mail_free(&importer->virtual_mail);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen if (importer->virtual_trans != NULL)
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen (void)mailbox_transaction_commit(&importer->virtual_trans);
70df8f39fb3db7c49b18c855178f8172176a037aTimo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_destroy(&importer->import_guids);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_destroy(&importer->import_uids);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_free(&importer->maybe_expunge_uids);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_free(&importer->maybe_saves);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_free(&importer->wanted_uids);
ec0cc8fa647794e44a1afaa448f495a713048dc4Timo Sirainen array_free(&importer->saved_uids);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_free(&importer->newmails);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_is_created(&importer->mail_requests))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_free(&importer->mail_requests);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *last_common_uid_r = importer->last_common_uid;
db40a9dd97fdd03b12bd6f793768a634e086f93cStephan Bosch if (*changes_during_sync_r == NULL) {
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen *last_common_modseq_r = importer->remote_highest_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen *last_common_pvt_modseq_r = importer->remote_highest_pvt_modseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* local changes occurred during dsync. we exported changes up
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen to local_initial_highestmodseq, so all of the changes have
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen happened after it. we want the next run to see those changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen so return it as the last common modseq */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *last_common_modseq_r = importer->local_initial_highestmodseq;
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen *last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen if (importer->delete_mailbox) {
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen if (mailbox_delete(importer->box) < 0) {
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen i_error("Couldn't delete mailbox %s: %s",
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen mailbox_get_vname(importer->box),
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(importer->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &importer->mail_error));
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen importer->failed = TRUE;
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen }
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen *last_messages_count_r = 0;
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen } else {
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status);
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen *last_messages_count_r = status.messages;
2faaca21fb49df0f1da859090246866b111cee1eTimo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen i_assert(importer->failed == (importer->mail_error != 0));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ret = importer->failed ? -1 : 0;
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen *error_r = importer->mail_error;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool_unref(&importer->pool);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainenconst char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer)
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen{
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen if (importer->search_ctx != NULL)
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen return "";
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen return t_strdup_printf("%u/%u", importer->import_pos,
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen importer->import_count);
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen}