5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "lib.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "array.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "hash.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "istream.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "seq-range-array.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "mail-storage-private.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "mail-search-build.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "dsync-transaction-log-scan.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "dsync-mail.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce#include "dsync-mailbox-import.h"

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestruct importer_mail {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce const char *guid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce};

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestruct importer_new_mail {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* linked list of mails for this GUID */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct importer_new_mail *next;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* if non-NULL, this mail exists in both local and remote. this link

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct importer_new_mail *link;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce const char *guid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct dsync_mail_change *change;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int uid_in_local:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int uid_is_usable:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int skip:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int copy_failed:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce};

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo SorceHASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo SorceHASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestruct dsync_mailbox_importer {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce pool_t pool;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mailbox *box;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t last_common_uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint64_t last_common_modseq;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t remote_uid_next;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t remote_first_recent_uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint64_t remote_highest_modseq;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mailbox_transaction_context *trans, *ext_trans;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mail_search_context *search_ctx;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mail *mail, *ext_mail;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mail *cur_mail;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce const char *cur_guid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* UID => struct dsync_mail_change */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce ARRAY_TYPE(seq_range) maybe_expunge_uids;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce ARRAY(struct dsync_mail_change *) maybe_saves;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* GUID => struct importer_new_mail */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce HASH_TABLE_TYPE(guid_new_mail) import_guids;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* UID => struct importer_new_mail */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce HASH_TABLE_TYPE(uid_new_mail) import_uids;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce ARRAY(struct importer_new_mail *) newmails;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce ARRAY_TYPE(uint32_t) wanted_uids;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce ARRAY(struct dsync_mail_request) mail_requests;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int mail_request_idx;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t prev_uid, next_local_seq, local_uid_next;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint64_t local_initial_highestmodseq;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int failed:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int last_common_uid_found:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int cur_uid_has_change:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int cur_mail_saved:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int local_expunged_guids_set:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int new_uids_assigned:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int want_mail_requests:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int mails_have_guids:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce unsigned int master_brain:1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce};

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestatic void

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcedsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce{

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mail_search_args *search_args;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mail_search_arg *sarg;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce search_args = mail_search_build_init();

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce sarg = mail_search_build_add(search_args, SEARCH_UIDSET);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce p_array_init(&sarg->value.seqset, search_args->pool, 128);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce seq_range_array_add_range(&sarg->value.seqset,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->last_common_uid+1, (uint32_t)-1);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->search_ctx =

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce mailbox_search_init(importer->trans, search_args, NULL,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce 0, NULL);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce mail_search_args_unref(&search_args);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (mailbox_search_next(importer->search_ctx, &importer->cur_mail))

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->next_local_seq = importer->cur_mail->seq;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* this flag causes cur_guid to be looked up later */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_mail_saved = TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce}

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestruct dsync_mailbox_importer *

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcedsync_mailbox_import_init(struct mailbox *box,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct dsync_transaction_log_scan *log_scan,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t last_common_uid,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint64_t last_common_modseq,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t remote_uid_next,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint32_t remote_first_recent_uid,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce uint64_t remote_highest_modseq,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce enum dsync_mailbox_import_flags flags)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce{

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce const enum mailbox_transaction_flags ext_trans_flags =

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce MAILBOX_TRANSACTION_FLAG_SYNC |

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce MAILBOX_TRANSACTION_FLAG_EXTERNAL |

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct dsync_mailbox_importer *importer;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mailbox_status status;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce pool_t pool;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer",

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce 10240);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer = p_new(pool, struct dsync_mailbox_importer, 1);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->pool = pool;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->box = box;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->last_common_uid = last_common_uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->last_common_modseq = last_common_modseq;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->last_common_uid_found =

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce last_common_uid != 0 || last_common_modseq != 0;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->remote_uid_next = remote_uid_next;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->remote_first_recent_uid = remote_first_recent_uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->remote_highest_modseq = remote_highest_modseq;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce hash_table_create_direct(&importer->import_uids, pool, 0);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_array_init(&importer->maybe_expunge_uids, 16);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_array_init(&importer->maybe_saves, 128);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_array_init(&importer->newmails, 128);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_array_init(&importer->wanted_uids, 128);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->trans = mailbox_transaction_begin(importer->box,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce MAILBOX_TRANSACTION_FLAG_SYNC);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->ext_trans = mailbox_transaction_begin(box, ext_trans_flags);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->mail = mail_alloc(importer->trans, 0, NULL);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_array_init(&importer->mail_requests, 128);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->want_mail_requests = TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->mails_have_guids =

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->master_brain =

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce mailbox_get_open_status(importer->box,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce &status);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->local_uid_next = status.uidnext;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->local_initial_highestmodseq = status.highest_modseq;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce dsync_mailbox_import_search_init(importer);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return importer;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce}

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestatic void dsync_mail_error(struct dsync_mailbox_importer *importer,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct mail *mail, const char *field)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce{

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce const char *errstr;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce enum mail_error error;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce errstr = mailbox_get_last_error(importer->box, &error);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (error == MAIL_ERROR_EXPUNGED)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_error("Can't lookup %s for UID=%u: %s", field, mail->uid, errstr);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->failed = TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce}

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestatic bool

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorceimporter_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce{

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (importer->cur_mail == NULL) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* end of search */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return FALSE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce while (importer->cur_mail->seq < importer->next_local_seq ||

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_mail->uid < wanted_uid) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (!importer->cur_uid_has_change &&

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce !importer->last_common_uid_found) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* this message exists locally, but remote didn't send

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce seq_range_array_add(&importer->maybe_expunge_uids,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_mail->uid);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_mail_saved = FALSE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (!mailbox_search_next(importer->search_ctx,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce &importer->cur_mail)) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_mail = NULL;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_guid = NULL;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return FALSE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_uid_has_change = FALSE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_uid_has_change = importer->cur_mail != NULL &&

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->cur_mail->uid == wanted_uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce &importer->cur_guid) < 0) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce dsync_mail_error(importer, importer->cur_mail, "GUID");

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return importer_next_mail(importer, wanted_uid);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* make sure next_local_seq gets updated in case we came here

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer->next_local_seq = importer->cur_mail->seq;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce}

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestatic int

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorceimporter_mail_cmp(const struct importer_mail *m1,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce const struct importer_mail *m2)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce{

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce int ret;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (m1->guid == NULL)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return 1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (m2->guid == NULL)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return -1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce ret = strcmp(m1->guid, m2->guid);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (ret != 0)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return ret;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (m1->uid < m2->uid)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return -1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (m1->uid > m2->uid)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return 1;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return 0;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce}

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestatic void importer_mail_request(struct dsync_mailbox_importer *importer,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct importer_new_mail *newmail)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce{

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct dsync_mail_request *request;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (importer->want_mail_requests && !newmail->uid_in_local) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce request = array_append_space(&importer->mail_requests);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce request->guid = newmail->guid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce request->uid = newmail->uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce}

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestatic void newmail_link(struct dsync_mailbox_importer *importer,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct importer_new_mail *newmail)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce{

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct importer_new_mail *first_mail, **last, *mail, *link = NULL;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (*newmail->guid != '\0') {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce first_mail = hash_table_lookup(importer->import_guids,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->guid);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (first_mail == NULL) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* first mail for this GUID */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce hash_table_insert(importer->import_guids,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->guid, newmail);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer_mail_request(importer, newmail);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce } else {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (!newmail->uid_in_local) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* FIXME: ? */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce first_mail = hash_table_lookup(importer->import_uids,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce POINTER_CAST(newmail->uid));

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (first_mail == NULL) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* first mail for this UID */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce hash_table_insert(importer->import_uids,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce POINTER_CAST(newmail->uid), newmail);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce importer_mail_request(importer, newmail);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce return;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* 1) add the newmail to the end of the linked list

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce last = &first_mail->next;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce for (mail = first_mail; mail != NULL; mail = mail->next) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (mail->uid == newmail->uid)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce mail->uid_is_usable = TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (link == NULL && mail->link == NULL &&

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce mail->uid_in_local != newmail->uid_in_local)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce link = mail;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce last = &mail->next;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce *last = newmail;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (link != NULL && newmail->link == NULL) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce link->link = newmail;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->link = link;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce}

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorcestatic bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct dsync_mail_change *save_change)

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce{

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct importer_mail m1, m2;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce struct importer_new_mail *newmail;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce int diff;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce bool remote_saved;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce memset(&m1, 0, sizeof(m1));

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (importer->cur_mail != NULL) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce m1.guid = importer->cur_guid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce m1.uid = importer->cur_mail->uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce memset(&m2, 0, sizeof(m2));

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (save_change != NULL) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce m2.guid = save_change->guid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce m2.uid = save_change->uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce }

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail = p_new(importer->pool, struct importer_new_mail, 1);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce diff = importer_mail_cmp(&m1, &m2);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce if (diff < 0) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* add a record for local mail */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_assert(importer->cur_mail != NULL);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->guid = p_strdup(importer->pool, importer->cur_guid);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid = importer->cur_mail->uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid_in_local = TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid_is_usable =

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid >= importer->remote_uid_next;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce remote_saved = FALSE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce } else if (diff > 0) {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_assert(save_change != NULL);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->guid = save_change->guid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid = save_change->uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid_in_local = FALSE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid_is_usable =

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid >= importer->local_uid_next;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce remote_saved = TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce } else {

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce /* identical */

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_assert(importer->cur_mail != NULL);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce i_assert(save_change != NULL);

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->guid = save_change->guid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid = importer->cur_mail->uid;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid_in_local = TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->uid_is_usable = TRUE;

5dbf360f2d6b0281c32f1bba6ebf5cc834c1716eSimo Sorce newmail->link = newmail;

remote_saved = TRUE;

}

if (newmail->uid_in_local) {

importer->cur_mail_saved = TRUE;

importer->next_local_seq++;

} else {

/* NOTE: assumes save_change is allocated from importer pool */

newmail->change = save_change;

}

array_append(&importer->newmails, &newmail, 1);

newmail_link(importer, newmail);

return remote_saved;

}

static bool ATTR_NULL(2)

dsync_mailbox_try_save(struct dsync_mailbox_importer *importer,

struct dsync_mail_change *save_change)

{

if (importer->cur_mail_saved) {

if (!importer_next_mail(importer, 0) && save_change == NULL)

return FALSE;

}

return dsync_mailbox_try_save_cur(importer, save_change);

}

static void dsync_mailbox_save(struct dsync_mailbox_importer *importer,

struct dsync_mail_change *save_change)

{

while (!dsync_mailbox_try_save(importer, save_change)) ;

}

static bool

dsync_import_set_mail(struct dsync_mailbox_importer *importer,

const struct dsync_mail_change *change)

{

const char *guid;

if (!mail_set_uid(importer->mail, change->uid))

return FALSE;

if (change->guid == NULL) {

/* GUID is unknown */

return TRUE;

}

if (*change->guid == '\0') {

/* backend doesn't support GUIDs. if hdr_hash is set, we could

return TRUE;

}

/* verify that GUID matches, just in case */

if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) {

dsync_mail_error(importer, importer->mail, "GUID");

return FALSE;

}

if (strcmp(guid, change->guid) != 0) {

i_error("Mailbox %s: Unexpected GUID mismatch for "

"UID=%u: %s != %s", mailbox_get_vname(importer->box),

change->uid, guid, change->guid);

importer->last_common_uid = 1;

importer->failed = TRUE;

return FALSE;

}

return TRUE;

}

static void

merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove,

uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove,

bool prefer_remote,

uint32_t *change_add_r, uint32_t *change_remove_r)

{

uint32_t combined_add, combined_remove, conflict_flags;

uint32_t local_wanted, remote_wanted;

/* resolve conflicts */

conflict_flags = local_add & remote_remove;

if (conflict_flags != 0) {

if (prefer_remote)

local_add &= ~conflict_flags;

else

remote_remove &= ~conflict_flags;

}

conflict_flags = local_remove & remote_add;

if (conflict_flags != 0) {

if (prefer_remote)

local_remove &= ~conflict_flags;

else

remote_add &= ~conflict_flags;

}

combined_add = local_add|remote_add;

combined_remove = local_remove|remote_remove;

i_assert((combined_add & combined_remove) == 0);

/* see if there are conflicting final flags */

local_wanted = (local_final|combined_add) & ~combined_remove;

remote_wanted = (remote_final|combined_add) & ~combined_remove;

conflict_flags = local_wanted ^ remote_wanted;

if (conflict_flags != 0) {

if (prefer_remote)

local_wanted = remote_wanted;

/*else

}

*change_add_r = local_wanted & ~local_final;

*change_remove_r = local_final & ~local_wanted;

}

static bool

keyword_find(ARRAY_TYPE(const_string) *keywords, const char *name,

unsigned int *idx_r)

{

const char *const *names;

unsigned int i, count;

names = array_get(keywords, &count);

for (i = 0; i < count; i++) {

if (strcmp(names[i], name) == 0) {

*idx_r = i;

return TRUE;

}

}

return FALSE;

}

static void keywords_append(ARRAY_TYPE(const_string) *dest,

const ARRAY_TYPE(const_string) *keywords,

uint32_t bits, unsigned int start_idx)

{

const char *const *namep;

unsigned int i;

for (i = 0; i < 32; i++) {

if ((bits & (1U << i)) == 0)

continue;

namep = array_idx(keywords, start_idx+i);

array_append(dest, namep, 1);

}

}

static void

merge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes,

const ARRAY_TYPE(const_string) *remote_changes,

bool prefer_remote)

{

/* local_changes and remote_changes are assumed to have no

uint32_t *local_add, *local_remove, *local_final;

uint32_t *remote_add, *remote_remove, *remote_final;

uint32_t *change_add, *change_remove;

ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords;

const char *const *changes, *name, *const *local_keywords;

struct mail_keywords *kw;

unsigned int i, count, name_idx, array_size;

local_keywords = mail_get_keywords(mail);

/* we'll assign a common index for each keyword name and place

if (array_is_created(remote_changes))

changes = array_get(remote_changes, &count);

else {

changes = NULL;

count = 0;

}

array_size = str_array_length(local_keywords) + count;

if (array_is_created(local_changes))

array_size += array_count(local_changes);

if (array_size == 0) {

/* this message has no keywords */

return;

}

t_array_init(&all_keywords, array_size);

t_array_init(&add_keywords, array_size);

t_array_init(&remove_keywords, array_size);

/* @UNSAFE: create large enough arrays to fit all keyword indexes. */

array_size = (array_size+31)/32;

local_add = t_new(uint32_t, array_size);

local_remove = t_new(uint32_t, array_size);

local_final = t_new(uint32_t, array_size);

remote_add = t_new(uint32_t, array_size);

remote_remove = t_new(uint32_t, array_size);

remote_final = t_new(uint32_t, array_size);

change_add = t_new(uint32_t, array_size);

change_remove = t_new(uint32_t, array_size);

/* get remote changes */

for (i = 0; i < count; i++) {

name = changes[i]+1;

name_idx = array_count(&all_keywords);

array_append(&all_keywords, &name, 1);

switch (changes[i][0]) {

case KEYWORD_CHANGE_ADD:

remote_add[name_idx/32] |= 1U << (name_idx%32);

/* fall through */

case KEYWORD_CHANGE_FINAL:

remote_final[name_idx/32] |= 1U << (name_idx%32);

break;

case KEYWORD_CHANGE_REMOVE:

remote_remove[name_idx/32] |= 1U << (name_idx%32);

break;

}

}

/* get local changes. use existing indexes for names when they exist. */

if (array_is_created(local_changes))

changes = array_get(local_changes, &count);

else {

changes = NULL;

count = 0;

}

for (i = 0; i < count; i++) {

name = changes[i]+1;

if (!keyword_find(&all_keywords, name, &name_idx)) {

name_idx = array_count(&all_keywords);

array_append(&all_keywords, &name, 1);

}

switch (changes[i][0]) {

case KEYWORD_CHANGE_ADD:

local_add[name_idx/32] |= 1U << (name_idx%32);

break;

case KEYWORD_CHANGE_REMOVE:

local_remove[name_idx/32] |= 1U << (name_idx%32);

break;

case KEYWORD_CHANGE_FINAL:

i_unreached();

}

}

for (i = 0; local_keywords[i] != NULL; i++) {

name = local_keywords[i];

if (!keyword_find(&all_keywords, name, &name_idx)) {

name_idx = array_count(&all_keywords);

array_append(&all_keywords, &name, 1);

}

local_final[name_idx/32] |= 1U << (name_idx%32);

}

i_assert(array_count(&all_keywords) <= array_size*32);

array_size = (array_count(&all_keywords)+31) / 32;

/* merge keywords */

for (i = 0; i < array_size; i++) {

merge_flags(local_final[i], local_add[i], local_remove[i],

remote_final[i], remote_add[i], remote_remove[i],

prefer_remote, &change_add[i], &change_remove[i]);

if (change_add[i] != 0) {

keywords_append(&add_keywords, &all_keywords,

change_add[i], i*32);

}

if (change_remove[i] != 0) {

keywords_append(&remove_keywords, &all_keywords,

change_add[i], i*32);

}

}

/* apply changes */

if (array_count(&add_keywords) > 0) {

array_append_zero(&add_keywords);

kw = mailbox_keywords_create_valid(mail->box,

array_idx(&add_keywords, 0));

mail_update_keywords(mail, MODIFY_ADD, kw);

mailbox_keywords_unref(&kw);

}

if (array_count(&remove_keywords) > 0) {

array_append_zero(&remove_keywords);

kw = mailbox_keywords_create_valid(mail->box,

array_idx(&remove_keywords, 0));

mail_update_keywords(mail, MODIFY_REMOVE, kw);

mailbox_keywords_unref(&kw);

}

}

static void

dsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer,

const struct dsync_mail_change *change)

{

const struct dsync_mail_change *local_change;

enum mail_flags local_add, local_remove;

uint32_t change_add, change_remove;

ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT;

struct mail *mail;

bool prefer_remote;

i_assert((change->add_flags & change->remove_flags) == 0);

if (importer->cur_mail != NULL &&

importer->cur_mail->uid == change->uid)

mail = importer->cur_mail;

else {

if (!dsync_import_set_mail(importer, change))

return;

mail = importer->mail;

}

local_change = hash_table_lookup(importer->local_changes,

POINTER_CAST(change->uid));

if (local_change == NULL) {

local_add = local_remove = 0;

} else {

local_add = local_change->add_flags;

local_remove = local_change->remove_flags;

local_keyword_changes = local_change->keyword_changes;

}

if (mail_get_modseq(mail) < change->modseq)

prefer_remote = TRUE;

else if (mail_get_modseq(mail) > change->modseq)

prefer_remote = FALSE;

else {

/* identical modseq, we'll just have to pick one.

prefer_remote = !importer->master_brain;

}

/* merge flags */

merge_flags(mail_get_flags(mail), local_add, local_remove,

change->final_flags, change->add_flags, change->remove_flags,

prefer_remote, &change_add, &change_remove);

if (change_add != 0)

mail_update_flags(mail, MODIFY_ADD, change_add);

if (change_remove != 0)

mail_update_flags(mail, MODIFY_REMOVE, change_remove);

/* merge keywords */

merge_keywords(mail, &local_keyword_changes, &change->keyword_changes,

prefer_remote);

mail_update_modseq(mail, change->modseq);

}

static void

dsync_mailbox_import_save(struct dsync_mailbox_importer *importer,

const struct dsync_mail_change *change)

{

struct dsync_mail_change *save;

i_assert(change->guid != NULL);

if (change->uid == importer->last_common_uid) {

/* we've already verified that the GUID matches.

i_assert(!importer->last_common_uid_found);

dsync_mailbox_import_flag_change(importer, change);

return;

}

save = p_new(importer->pool, struct dsync_mail_change, 1);

dsync_mail_change_dup(importer->pool, change, save);

if (importer->last_common_uid_found) {

/* this is a new mail. its UID may or may not conflict with

i_assert(change->uid > importer->last_common_uid);

dsync_mailbox_save(importer, save);

} else {

/* the local mail is expunged. we'll decide later if we want

i_assert(change->uid > importer->last_common_uid);

i_assert(change->uid < importer->cur_mail->uid);

array_append(&importer->maybe_saves, &save, 1);

}

}

static void

dsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer,

const struct dsync_mail_change *change)

{

if (importer->last_common_uid_found) {

/* expunge the message, unless its GUID unexpectedly doesn't

i_assert(change->uid <= importer->last_common_uid);

if (dsync_import_set_mail(importer, change))

mail_expunge(importer->mail);

} else if (change->uid < importer->cur_mail->uid) {

/* already expunged locally, we can ignore this.

i_assert(change->uid >= importer->last_common_uid);

} else if (change->uid == importer->last_common_uid) {

/* already verified that the GUID matches */

i_assert(importer->cur_mail->uid == change->uid);

mail_expunge(importer->cur_mail);

} else {

/* we don't know yet if we should expunge this

i_assert(change->uid > importer->last_common_uid);

seq_range_array_add(&importer->maybe_expunge_uids, change->uid);

}

}

static void

dsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer)

{

/* If there are local mails after last_common_uid which we skipped

if (importer->cur_mail != NULL &&

importer->cur_mail->uid <= importer->last_common_uid+1)

return;

importer->cur_mail = NULL;

importer->cur_guid = NULL;

importer->next_local_seq = 0;

(void)mailbox_search_deinit(&importer->search_ctx);

dsync_mailbox_import_search_init(importer);

}

static void

dsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer)

{

struct dsync_mail_change *const *saves;

struct seq_range_iter iter;

unsigned int n, i, count;

uint32_t uid;

importer->last_common_uid_found = TRUE;

dsync_mailbox_rewind_search(importer);

/* expunge the messages whose expunge-decision we delayed previously */

seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0;

while (seq_range_array_iter_nth(&iter, n++, &uid)) {

if (uid > importer->last_common_uid) {

/* we expunge messages only up to last_common_uid,

break;

}

if (mail_set_uid(importer->mail, uid))

mail_expunge(importer->mail);

}

/* handle pending saves */

saves = array_get(&importer->maybe_saves, &count);

for (i = 0; i < count; i++) {

if (saves[i]->uid > importer->last_common_uid)

dsync_mailbox_save(importer, saves[i]);

}

}

static int

dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,

const struct dsync_mail_change *change)

{

const char *hdr_hash;

if (*change->guid != '\0' && *importer->cur_guid != '\0') {

/* we have GUIDs, verify them */

return strcmp(change->guid, importer->cur_guid) == 0 ? 1 : 0;

}

/* verify hdr_hash if it exists */

if (change->hdr_hash == NULL) {

i_assert(*importer->cur_guid == '\0');

i_error("Mailbox %s: GUIDs not supported, "

"sync with header hashes instead",

mailbox_get_vname(importer->box));

importer->failed = TRUE;

return -1;

}

if (dsync_mail_get_hdr_hash(importer->cur_mail, &hdr_hash) < 0) {

dsync_mail_error(importer, importer->cur_mail, "hdr-stream");

return -1;

}

return strcmp(change->hdr_hash, hdr_hash) == 0 ? 1 : 0;

}

static void

dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,

const struct dsync_mail_change *change)

{

const struct dsync_mail_change *local_change;

guid_128_t guid_128, change_guid_128;

int ret;

/* try to find the matching local mail */

if (!importer_next_mail(importer, change->uid)) {

/* no more local mails. use the last message with a matching

dsync_mailbox_common_uid_found(importer);

return;

}

if (change->guid == NULL) {

/* we can't know if this UID matches */

return;

}

if (importer->cur_mail->uid == change->uid) {

/* we have a matching local UID. check GUID to see if it's

if ((ret = dsync_mailbox_import_match_msg(importer, change)) < 0) {

/* unknown */

return;

}

if (ret == 0) {

/* mismatch - found the first non-common UID */

dsync_mailbox_common_uid_found(importer);

} else {

importer->last_common_uid = change->uid;

}

return;

}

if (*change->guid == '\0') {

/* remote doesn't support GUIDs, can't verify expunge */

return;

}

/* local message is expunged. see if we can find its GUID from

local_change = hash_table_lookup(importer->local_changes,

POINTER_CAST(change->uid));

if (local_change == NULL || local_change->guid == NULL)

return;

if (guid_128_from_string(local_change->guid, guid_128) < 0)

i_unreached();

mail_generate_guid_128_hash(change->guid, change_guid_128);

if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) {

/* mismatch - found the first non-common UID */

dsync_mailbox_common_uid_found(importer);

} else {

importer->last_common_uid = change->uid;

}

return;

}

void dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,

const struct dsync_mail_change *change)

{

i_assert(!importer->new_uids_assigned);

i_assert(importer->prev_uid < change->uid);

importer->prev_uid = change->uid;

if (!importer->last_common_uid_found)

dsync_mailbox_find_common_uid(importer, change);

if (importer->last_common_uid_found) {

/* a) uid <= last_common_uid for flag changes and expunges.

if (change->uid <= importer->last_common_uid) {

i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE);

} else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {

/* ignore */

return;

} else {

i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE);

}

} else {

/* a) uid < last_common_uid can never happen */

i_assert(change->uid >= importer->last_common_uid);

/* b) uid = last_common_uid if we've verified that the

if (change->uid > importer->last_common_uid) {

i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||

change->uid < importer->cur_mail->uid);

}

}

switch (change->type) {

case DSYNC_MAIL_CHANGE_TYPE_SAVE:

dsync_mailbox_import_save(importer, change);

break;

case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE:

dsync_mailbox_import_expunge(importer, change);

break;

case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE:

i_assert(importer->last_common_uid_found);

dsync_mailbox_import_flag_change(importer, change);

break;

}

}

static void

dsync_msg_update_uid(struct dsync_mailbox_importer *importer,

uint32_t old_uid, uint32_t new_uid)

{

struct mail_save_context *save_ctx;

if (!mail_set_uid(importer->mail, old_uid))

return;

save_ctx = mailbox_save_alloc(importer->ext_trans);

mailbox_save_copy_flags(save_ctx, importer->mail);

mailbox_save_set_uid(save_ctx, new_uid);

if (mailbox_copy(&save_ctx, importer->mail) == 0) {

array_append(&importer->wanted_uids, &new_uid, 1);

mail_expunge(importer->mail);

}

}

static void

dsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer)

{

struct importer_new_mail *newmail, *const *newmailp;

uint32_t common_uid_next, new_uid;

common_uid_next = I_MAX(importer->local_uid_next,

importer->remote_uid_next);

array_foreach_modifiable(&importer->newmails, newmailp) {

newmail = *newmailp;

if (newmail->skip) {

/* already assigned */

if (newmail->uid_in_local) {

if (mail_set_uid(importer->mail, newmail->uid))

mail_expunge(importer->mail);

}

continue;

}

/* figure out what UID to use for the mail */

if (newmail->uid_is_usable) {

/* keep the UID */

new_uid = newmail->uid;

} else if (newmail->link != NULL &&

newmail->link->uid_is_usable)

new_uid = newmail->link->uid;

else

new_uid = common_uid_next++;

if (newmail->uid_in_local && newmail->uid != new_uid) {

/* local UID changed, reassign it by copying */

dsync_msg_update_uid(importer, newmail->uid, new_uid);

}

newmail->uid = new_uid;

if (newmail->link != NULL) {

/* skip the linked mail */

newmail->link->skip = TRUE;

}

}

importer->last_common_uid = common_uid_next;

importer->new_uids_assigned = TRUE;

}

void dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer)

{

i_assert(!importer->new_uids_assigned);

if (!importer->last_common_uid_found) {

/* handle pending expunges and flag updates */

dsync_mailbox_common_uid_found(importer);

}

/* skip common local mails */

(void)importer_next_mail(importer, importer->last_common_uid+1);

/* if there are any local mails left, add them to newmails list */

while (importer->cur_mail != NULL)

(void)dsync_mailbox_try_save(importer, NULL);

dsync_mailbox_import_assign_new_uids(importer);

}

const struct dsync_mail_request *

dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer)

{

const struct dsync_mail_request *requests;

unsigned int count;

requests = array_get(&importer->mail_requests, &count);

if (importer->mail_request_idx == count)

return NULL;

return &requests[importer->mail_request_idx++];

}

static const char *const *

dsync_mailbox_get_final_keywords(const struct dsync_mail_change *change)

{

ARRAY_TYPE(const_string) keywords;

const char *const *changes;

unsigned int i, count;

if (!array_is_created(&change->keyword_changes))

return NULL;

changes = array_get(&change->keyword_changes, &count);

t_array_init(&keywords, count);

for (i = 0; i < count; i++) {

if (changes[i][0] == KEYWORD_CHANGE_ADD ||

changes[i][0] == KEYWORD_CHANGE_FINAL) {

const char *name = changes[i]+1;

array_append(&keywords, &name, 1);

}

}

if (array_count(&keywords) == 0)

return NULL;

array_append_zero(&keywords);

return array_idx(&keywords, 0);

}

static void

dsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer,

struct mail_save_context *save_ctx,

const struct dsync_mail_change *change)

{

const char *const *keyword_names;

struct mail_keywords *keywords;

keyword_names = dsync_mailbox_get_final_keywords(change);

keywords = keyword_names == NULL ? NULL :

mailbox_keywords_create_valid(importer->box,

keyword_names);

mailbox_save_set_flags(save_ctx, change->final_flags, keywords);

if (keywords != NULL)

mailbox_keywords_unref(&keywords);

mailbox_save_set_save_date(save_ctx, change->save_timestamp);

if (change->modseq > 1) {

(void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE);

mailbox_save_set_min_modseq(save_ctx, change->modseq);

}

}

static int

dsync_msg_try_copy(struct dsync_mailbox_importer *importer,

struct mail_save_context **save_ctx_p,

struct importer_new_mail *all_newmails)

{

struct importer_new_mail *inst;

for (inst = all_newmails; inst != NULL; inst = inst->next) {

if (inst->uid_in_local && !inst->copy_failed &&

mail_set_uid(importer->mail, inst->uid)) {

if (mailbox_copy(save_ctx_p, importer->mail) < 0) {

inst->copy_failed = TRUE;

return -1;

}

return 1;

}

}

return 0;

}

static struct mail_save_context *

dsync_mailbox_save_init(struct dsync_mailbox_importer *importer,

const struct dsync_mail *mail,

struct importer_new_mail *newmail)

{

struct mail_save_context *save_ctx;

save_ctx = mailbox_save_alloc(importer->ext_trans);

mailbox_save_set_uid(save_ctx, newmail->uid);

if (*mail->guid != '\0')

mailbox_save_set_guid(save_ctx, mail->guid);

dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);

if (*mail->pop3_uidl != '\0')

mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);

if (mail->pop3_order > 0)

mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);

mailbox_save_set_received_date(save_ctx, mail->received_date, 0);

return save_ctx;

}

static void dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,

const struct dsync_mail *mail,

struct importer_new_mail *newmail,

struct importer_new_mail *all_newmails)

{

struct mail_save_context *save_ctx;

ssize_t ret;

bool save_failed = FALSE;

/* try to save the mail by copying an existing mail */

save_ctx = dsync_mailbox_save_init(importer, mail, newmail);

if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails)) < 0) {

if (save_ctx == NULL)

save_ctx = dsync_mailbox_save_init(importer, mail, newmail);

}

if (ret > 0) {

array_append(&importer->wanted_uids, &newmail->uid, 1);

return;

}

/* fallback to saving from remote stream */

if (mail->input == NULL) {

/* it was just expunged in remote, skip it */

mailbox_save_cancel(&save_ctx);

return;

}

i_stream_seek(mail->input, 0);

if (mailbox_save_begin(&save_ctx, mail->input) < 0) {

i_error("Can't save message to mailbox %s: %s",

mailbox_get_vname(importer->box),

mailbox_get_last_error(importer->box, NULL));

importer->failed = TRUE;

return;

}

while ((ret = i_stream_read(mail->input)) > 0 || ret == -2) {

if (mailbox_save_continue(save_ctx) < 0) {

save_failed = TRUE;

ret = -1;

break;

}

}

i_assert(ret == -1);

if (mail->input->stream_errno != 0) {

errno = mail->input->stream_errno;

i_error("read(msg input) failed: %m");

mailbox_save_cancel(&save_ctx);

importer->failed = TRUE;

} else if (save_failed) {

mailbox_save_cancel(&save_ctx);

importer->failed = TRUE;

} else {

i_assert(mail->input->eof);

if (mailbox_save_finish(&save_ctx) < 0) {

i_error("Can't save message to mailbox %s: %s",

mailbox_get_vname(importer->box),

mailbox_get_last_error(importer->box, NULL));

importer->failed = TRUE;

} else {

array_append(&importer->wanted_uids, &newmail->uid, 1);

}

}

}

void dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,

const struct dsync_mail *mail)

{

struct importer_new_mail *newmail, *allmails;

i_assert(mail->input->seekable);

i_assert(importer->new_uids_assigned);

newmail = *mail->guid != '\0' ?

hash_table_lookup(importer->import_guids, mail->guid) :

hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid));

if (newmail == NULL) {

if (importer->want_mail_requests) {

i_error("%s: Remote sent unwanted message body for "

"GUID=%s UID=%u",

mailbox_get_vname(importer->box),

mail->guid, mail->uid);

}

return;

}

if (*mail->guid != '\0')

hash_table_remove(importer->import_guids, mail->guid);

else {

hash_table_remove(importer->import_uids,

POINTER_CAST(mail->uid));

}

/* save all instances of the message */

allmails = newmail;

for (; newmail != NULL; newmail = newmail->next) {

if (newmail->skip) {

/* no need to do anything for this mail */

continue;

}

if (newmail->uid_in_local) {

/* we already handled this by copying the mail */

continue;

}

T_BEGIN {

dsync_mailbox_save_body(importer, mail, newmail,

allmails);

} T_END;

}

}

static int

reassign_uids_in_seq_range(struct mailbox *box, uint32_t seq1, uint32_t seq2)

{

const enum mailbox_transaction_flags trans_flags =

MAILBOX_TRANSACTION_FLAG_EXTERNAL |

MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;

struct mailbox_transaction_context *trans;

struct mail_save_context *save_ctx;

struct mail *mail;

uint32_t seq;

int ret = 0;

trans = mailbox_transaction_begin(box, trans_flags);

mail = mail_alloc(trans, 0, NULL);

for (seq = seq1; seq <= seq2; seq++) {

mail_set_seq(mail, seq);

save_ctx = mailbox_save_alloc(trans);

mailbox_save_copy_flags(save_ctx, mail);