dsync-mailbox-export.c revision 791fb70b3255a11a91ce0c2dc3ae1460d4cf8459
5a580c3a38ced62d4bcc95b8ac7c4f2935b5d294Timo Sirainen/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID => instances */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen /* uint32_t UID => struct dsync_mail_change */
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen HASH_TABLE(void *, struct dsync_mail_change *) changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* changes sorted by UID */
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct dsync_mail_change *) sorted_changes;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct hash_iterate_context *attr_change_iter;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen errstr = mailbox_get_last_error(exporter->box, &error);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "Can't lookup %s for UID=%u: %s",
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainenfinal_keyword_check(struct dsync_mail_change *change, const char *name,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(&change->keyword_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen switch (changes[i][0]) {
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen /* replace with ADD_AND_FINAL */
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen /* a final keyword is marked as removed.
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen this shouldn't normally happen. */
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen /* no change */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainensearch_update_flag_changes(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail, struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert((change->add_flags & change->remove_flags) == 0);
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen change->pvt_modseq = mail_get_pvt_modseq(mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!array_is_created(&change->keyword_changes) &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&change->keyword_changes, exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* add the final keyword if it's not already there
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen as +keyword */
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen if (!final_keyword_check(change, keywords[i], &type)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenexporter_get_guids(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* always try to get GUID, even if we're also getting header hash */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_get_special(mail, MAIL_FETCH_GUID, guid_r) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mail_error(exporter, mail, "GUID");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get header hash also */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (dsync_mail_get_hdr_hash(mail, hdr_hash_r) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mail_error(exporter, mail, "hdr-stream");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = "Backend doesn't support GUIDs, "
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "sync with header hashes instead";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUIDs are required, we don't need header hash */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainensearch_update_flag_change_guid(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *change, *log_change;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen change = hash_table_lookup(exporter->changes, POINTER_CAST(mail->uid));
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen change = p_new(exporter->pool, struct dsync_mail_change, 1);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* the message was expunged during export */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* find its GUID from log if possible */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen log_change = dsync_transaction_log_scan_find_new_expunge(
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_update_flag_changes(exporter, mail, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenexport_save_change_get(struct dsync_mailbox_exporter *exporter, uint32_t uid)
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen change = hash_table_lookup(exporter->changes, POINTER_CAST(uid));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change = p_new(exporter->pool, struct dsync_mail_change, 1);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_insert(exporter->changes, POINTER_CAST(uid), change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* move flag changes into a save. this happens only when
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last_common_uid isn't known */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenexport_add_mail_instance(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *change, uint32_t seq)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->auto_export_mails && !exporter->mails_have_guids) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUIDs not supported, mail is requested by UIDs */
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&exporter->requested_uids, change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* mail UIDs are manually requested */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances = hash_table_lookup(exporter->export_guids, change->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&instances->seqs, exporter->pool, 2);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainensearch_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* If message is already expunged here, just skip it */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_get_save_date(mail, &save_timestamp) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mail_error(exporter, mail, "save-date");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change = export_save_change_get(exporter, mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_update_flag_changes(exporter, mail, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen export_add_mail_instance(exporter, change, mail->seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(exporter->changes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* any flag changes for UIDs above last_common_uid weren't found by
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox search, which means they were already expunged. for some
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen reason the log scanner found flag changes for the message, but not
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen the expunge. just remove these. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(exporter->changes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_search(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&sarg->value.seqset, search_args->pool, 1);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen if (exporter->return_all_mails || exporter->last_common_uid == 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we want to know about all mails */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset, 1, (uint32_t)-1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* lookup GUIDs for messages with flag changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_add_flagchange_uids(exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* lookup new messages */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->trans = mailbox_transaction_begin(exporter->box, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_ctx = mailbox_search_init(exporter->trans, search_args, NULL,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (mailbox_search_next(search_ctx, &mail)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ret = search_update_flag_change_guid(exporter, mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_drop_expunged_flag_changes(exporter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "Mail search failed: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int dsync_mail_change_p_uid_cmp(struct dsync_mail_change *const *c1,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&exporter->sorted_changes, exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(exporter->changes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, exporter->changes, &key, &change))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&exporter->sorted_changes, &change, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_sort(&exporter->sorted_changes, dsync_mail_change_p_uid_cmp);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen mailbox_attribute_iter_init(exporter->box, type, "");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_log_scan(struct dsync_mailbox_exporter *exporter,
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE_TYPE(dsync_uid_mail_change) log_changes;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen struct dsync_mail_change *change, *dup_change;
08e9fec5ba9e1a26e658c4224207d666b6ced27dTimo Sirainen log_changes = dsync_transaction_log_scan_get_hash(log_scan);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen if (dsync_transaction_log_scan_has_all_changes(log_scan)) {
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen /* we tried to access too old/invalid modseqs. to make sure
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen no changes get lost, we need to send all of the messages */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* clone the hash table, since we're changing it. */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create_direct(&exporter->changes, exporter->pool,
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, log_changes, &key, &change)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dup_change = p_new(exporter->pool, struct dsync_mail_change, 1);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_insert(exporter->changes, key, dup_change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->highest_changed_uid < change->uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox export",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter = p_new(pool, struct dsync_mailbox_exporter, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&exporter->requested_uids, pool, 16);
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen p_array_init(&exporter->search_uids, pool, 16);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&exporter->expunged_seqs, pool, 16);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&exporter->expunged_guids, pool, 16);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* first scan transaction log and save any expunges and flag changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_log_scan(exporter, log_scan);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get saves and also find GUIDs for flag changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get the changes sorted by UID */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter)
791fb70b3255a11a91ce0c2dc3ae1460d4cf8459Timo Sirainen while (hash_table_iterate(exporter->attr_change_iter,
791fb70b3255a11a91ce0c2dc3ae1460d4cf8459Timo Sirainen dsync_transaction_log_scan_get_attr_hash(exporter->log_scan),
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* lookup the value mainly to get its last_change value. */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (mailbox_attribute_get_stream(exporter->trans, attr->type,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen "Mailbox attribute %s lookup failed: %s", attr->key,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (value.value != NULL || value.value_stream != NULL) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->value = p_strdup(exporter->pool, value.value);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen hash_table_iterate_deinit(&exporter->attr_change_iter);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct dsync_mailbox_attribute lookup_attr, *attr;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen const char *key;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen export_all_attrs = exporter->return_all_mails ||
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* note that the order of processing may be important for some
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attributes. for example sieve can't set a script active until it's
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen first been created */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr_change = hash_table_lookup(attr_changes, &lookup_attr);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (mailbox_attribute_get_stream(exporter->trans,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen "Mailbox attribute %s lookup failed: %s", key,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* readonly attributes can't be changed,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen no point in exporting them */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (value.value == NULL && value.value_stream == NULL &&
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen (attr_change == NULL || !attr_change->deleted)) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* the attribute was just deleted?
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen skip for this sync. */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->value = p_strdup(exporter->pool, value.value);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen "Mailbox attribute iteration failed: %s",
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* export shared attributes */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return dsync_mailbox_export_iter_next_attr(exporter);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->attr_change_iter = hash_table_iterate_init(attr_changes);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainendsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (dsync_mailbox_export_iter_next_attr(exporter) <= 0)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (dsync_mailbox_export_iter_next_nonexistent_attr(exporter) <= 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(&exporter->sorted_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen sarg = mail_search_build_add(search_args, SEARCH_SEQSET);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&sarg->value.seqset, search_args->pool, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get a list of messages we want to fetch. if there are more than one
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instance for a GUID, use the first one. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(exporter->export_guids);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, exporter->export_guids,
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&sarg->value.seqset, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (seq_range_exists(&exporter->expunged_seqs, seq)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we're on a second round, refetching expunged
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_remove(&instances->seqs, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_remove(&exporter->expunged_seqs, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* no instances left */
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&sarg->value.seqset, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* add requested UIDs */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen range = array_get(&exporter->requested_uids, &count);
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset,
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen array_append_array(&exporter->search_uids, &exporter->requested_uids);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_search_init(exporter->trans, search_args, NULL,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return array_count(&sarg->value.seqset) > 0 ? 1 : 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_search_deinit(&exporter->search_ctx) < 0 &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "Mail search failed: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen if (dsync_mail_fill(mail, &exporter->dsync_mail, &error_field) < 0)
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return dsync_mail_error(exporter, mail, error_field);
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen instances = *exporter->dsync_mail.guid == '\0' ? NULL :
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID found */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* mail requested by UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "GUID unexpectedly changed for UID=%u GUID=%s",
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen if (!seq_range_exists(&exporter->search_uids, mail->uid))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this message was successfully returned, don't try retrying it */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenvoid dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter,
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&exporter->requested_uids, request->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances = hash_table_lookup(exporter->export_guids, request->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "Remote requested unexpected GUID %s", request->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *guids;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (dsync_mailbox_export_body_search_init(exporter) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (mailbox_search_next(exporter->search_ctx, &mail)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = dsync_mailbox_export_mail(exporter, mail)) > 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* the message was expunged. if the GUID has another instance,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen try sending it later. */
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&exporter->expunged_seqs, mail->seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* if some instances of messages were expunged, retry fetching them
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen with other instances */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_body_search_deinit(exporter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = dsync_mailbox_export_body_search_init(exporter)) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* not finished yet */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mailbox_export_next_mail(exporter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* finished with messages. if there are any expunged messages,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return them */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen guids = array_get(&exporter->expunged_guids, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen memset(&exporter->dsync_mail, 0, sizeof(exporter->dsync_mail));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenint dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **_exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char **error_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mailbox_exporter *exporter = *_exporter;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_body_search_deinit(exporter);