bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "lib.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "array.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "hash.h"
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen#include "istream.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "mail-index-modseq.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "mail-storage-private.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "mail-search-build.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "dsync-transaction-log-scan.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "dsync-mail.h"
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen#include "dsync-mailbox.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen#include "dsync-mailbox-export.h"
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstruct dsync_mail_guid_instances {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(seq_range) seqs;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen bool requested;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen bool searched;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen};
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstruct dsync_mailbox_exporter {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool_t pool;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mailbox *box;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_transaction_log_scan *log_scan;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t last_common_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
f97c983e7a742e1d54ec60e502093abbb3ad2907Timo Sirainen struct mailbox_header_lookup_ctx *wanted_headers;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mailbox_transaction_context *trans;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_context *search_ctx;
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen unsigned int search_pos, search_count;
45af47783693b3ba2768c5ad34eeff68132382d0Timo Sirainen unsigned int hdr_hash_version;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi const char *const *hashed_headers;
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID => instances */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(seq_range) requested_uids;
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen ARRAY_TYPE(seq_range) search_uids;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(seq_range) expunged_seqs;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(const_string) expunged_guids;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int expunged_guid_idx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen /* uint32_t UID => struct dsync_mail_change */
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen HASH_TABLE(void *, struct dsync_mail_change *) changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* changes sorted by UID */
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct dsync_mail_change *) sorted_changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int change_idx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t highest_changed_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct mailbox_attribute_iter *attr_iter;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct hash_iterate_context *attr_change_iter;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen enum mail_attribute_type attr_type;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct dsync_mailbox_attribute attr;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail dsync_mail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *error;
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen enum mail_error mail_error;
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool body_search_initialized:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool auto_export_mails:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool mails_have_guids:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool minimal_dmail_fill:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool return_all_mails:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool export_received_timestamps:1;
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen bool export_virtual_sizes:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool no_hdr_hashes:1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen};
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail, const char *field)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *errstr;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen enum mail_error error;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi errstr = mailbox_get_last_internal_error(exporter->box, &error);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (error == MAIL_ERROR_EXPUNGED)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen exporter->mail_error = error;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "Can't lookup %s for UID=%u: %s",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen field, mail->uid, errstr);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic bool
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainenfinal_keyword_check(struct dsync_mail_change *change, const char *name,
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen char *type_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
5446576156fbe26e07a5cb964a900281d283f387Timo Sirainen *type_r = KEYWORD_CHANGE_FINAL;
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(&change->keyword_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; i < count; i++) {
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen if (strcmp(changes[i]+1, name) != 0)
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen continue;
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen switch (changes[i][0]) {
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen case KEYWORD_CHANGE_ADD:
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen /* replace with ADD_AND_FINAL */
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen array_delete(&change->keyword_changes, i, 1);
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen *type_r = KEYWORD_CHANGE_ADD_AND_FINAL;
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen return FALSE;
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen case KEYWORD_CHANGE_REMOVE:
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen /* a final keyword is marked as removed.
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen this shouldn't normally happen. */
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen array_delete(&change->keyword_changes, i, 1);
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen return FALSE;
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen case KEYWORD_CHANGE_ADD_AND_FINAL:
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen case KEYWORD_CHANGE_FINAL:
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen /* no change */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return FALSE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainensearch_update_flag_changes(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail, struct dsync_mail_change *change)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *keywords;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i;
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen char type;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert((change->add_flags & change->remove_flags) == 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->modseq = mail_get_modseq(mail);
6abf66a3731d52889517bd644595c540e3a9b3ecTimo Sirainen change->pvt_modseq = mail_get_pvt_modseq(mail);
113ad88e9506a02dfa6fc7f4927c47272cf1b461Timo Sirainen change->final_flags = mail_get_flags(mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keywords = mail_get_keywords(mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!array_is_created(&change->keyword_changes) &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen keywords[0] != NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&change->keyword_changes, exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen str_array_length(keywords));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen for (i = 0; keywords[i] != NULL; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* add the final keyword if it's not already there
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen as +keyword */
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen if (!final_keyword_check(change, keywords[i], &type)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *keyword_change =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_strdup_printf(exporter->pool, "%c%s",
252f5c6a63878e7a8a7ffb5847eecbad7f8737e8Timo Sirainen type, keywords[i]);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&change->keyword_changes,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen &keyword_change, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenexporter_get_guids(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail, const char **guid_r,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char **hdr_hash_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *guid_r = "";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *hdr_hash_r = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* always try to get GUID, even if we're also getting header hash */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mail_get_special(mail, MAIL_FETCH_GUID, guid_r) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mail_error(exporter, mail, "GUID");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!exporter->mails_have_guids) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get header hash also */
03af8e5325a7b4fec36414ac35949457bc426c0bTimo Sirainen if (exporter->no_hdr_hashes) {
03af8e5325a7b4fec36414ac35949457bc426c0bTimo Sirainen *hdr_hash_r = "";
03af8e5325a7b4fec36414ac35949457bc426c0bTimo Sirainen return 1;
03af8e5325a7b4fec36414ac35949457bc426c0bTimo Sirainen }
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version,
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi exporter->hashed_headers, hdr_hash_r) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return dsync_mail_error(exporter, mail, "hdr-stream");
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (**guid_r == '\0') {
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen exporter->mail_error = MAIL_ERROR_TEMP;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = "Backend doesn't support GUIDs, "
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "sync with header hashes instead";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUIDs are required, we don't need header hash */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainensearch_update_flag_change_guid(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *change, *log_change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *guid, *hdr_hash;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen int ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen change = hash_table_lookup(exporter->changes, POINTER_CAST(mail->uid));
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen if (change != NULL) {
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen } else {
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen i_assert(exporter->return_all_mails);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen change = p_new(exporter->pool, struct dsync_mail_change, 1);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen change->uid = mail->uid;
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE;
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen hash_table_insert(exporter->changes,
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen POINTER_CAST(mail->uid), change);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (ret == 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* the message was expunged during export */
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->uid = mail->uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* find its GUID from log if possible */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen log_change = dsync_transaction_log_scan_find_new_expunge(
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->log_scan, mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (log_change != NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->guid = log_change->guid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->guid = *guid == '\0' ? "" :
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_strdup(exporter->pool, guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_update_flag_changes(exporter, mail, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic struct dsync_mail_change *
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenexport_save_change_get(struct dsync_mailbox_exporter *exporter, uint32_t uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen change = hash_table_lookup(exporter->changes, POINTER_CAST(uid));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change = p_new(exporter->pool, struct dsync_mail_change, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->uid = uid;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_insert(exporter->changes, POINTER_CAST(uid), change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* move flag changes into a save. this happens only when
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen last_common_uid isn't known */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(exporter->last_common_uid == 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenexport_add_mail_instance(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *change, uint32_t seq)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_guid_instances *instances;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->auto_export_mails && !exporter->mails_have_guids) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUIDs not supported, mail is requested by UIDs */
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&exporter->requested_uids, change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (*change->guid == '\0') {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* mail UIDs are manually requested */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(!exporter->mails_have_guids);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances = hash_table_lookup(exporter->export_guids, change->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (instances == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances = p_new(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_guid_instances, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&instances->seqs, exporter->pool, 2);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_insert(exporter->export_guids,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_strdup(exporter->pool, change->guid),
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->auto_export_mails)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances->requested = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&instances->seqs, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainensearch_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *guid, *hdr_hash;
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen time_t received_timestamp = 0;
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen uoff_t virtual_size = (uoff_t)-1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen int ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen /* update wanted fields in case we didn't already set them for the
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen search */
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen if (exporter->export_received_timestamps)
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen if (exporter->export_virtual_sizes)
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE;
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen mail_add_temp_wanted_fields(mail, wanted_fields,
f97c983e7a742e1d54ec60e502093abbb3ad2907Timo Sirainen exporter->wanted_headers);
f97c983e7a742e1d54ec60e502093abbb3ad2907Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* If message is already expunged here, just skip it */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen if (exporter->export_received_timestamps) {
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen if (mail_get_received_date(mail, &received_timestamp) < 0)
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen return dsync_mail_error(exporter, mail, "received-time");
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen if (received_timestamp == 0) {
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen /* don't allow timestamps to be zero. we want to have
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen asserts verify that the timestamp is set properly. */
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen received_timestamp = 1;
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen }
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen }
93a4387a19cdb45cec27eb951c9fcdb31fe5b1d5Timo Sirainen if (exporter->export_virtual_sizes) {
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen if (mail_get_virtual_size(mail, &virtual_size) < 0)
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen return dsync_mail_error(exporter, mail, "virtual-size");
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen i_assert(virtual_size != (uoff_t)-1);
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen }
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change = export_save_change_get(exporter, mail->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->guid = *guid == '\0' ? "" :
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_strdup(exporter->pool, guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen change->received_timestamp = received_timestamp;
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen change->virtual_size = virtual_size;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_update_flag_changes(exporter, mail, change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen export_add_mail_instance(exporter, change, mail->seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen ARRAY_TYPE(seq_range) *uids)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct hash_iterate_context *iter;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen void *key;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen struct dsync_mail_change *change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(exporter->changes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE)
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(uids, change->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_iterate_deinit(&iter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct hash_iterate_context *iter;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen void *key;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen struct dsync_mail_change *change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* any flag changes for UIDs above last_common_uid weren't found by
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox search, which means they were already expunged. for some
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen reason the log scanner found flag changes for the message, but not
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen the expunge. just remove these. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(exporter->changes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen change->uid > exporter->last_common_uid)
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_remove(exporter->changes, key);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_iterate_deinit(&iter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_search(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_context *search_ctx;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_args *search_args;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_arg *sarg;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail;
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen enum mail_fetch_field wanted_fields = 0;
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen struct mailbox_header_lookup_ctx *wanted_headers = NULL;
b0a06fd1ca37f4626037a62be5c7f0ce12395013Timo Sirainen int ret = 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_args = mail_search_build_init();
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&sarg->value.seqset, search_args->pool, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen if (exporter->return_all_mails || exporter->last_common_uid == 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we want to know about all mails */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset, 1, (uint32_t)-1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* lookup GUIDs for messages with flag changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_add_flagchange_uids(exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen &sarg->value.seqset);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* lookup new messages */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->last_common_uid + 1,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (uint32_t)-1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen if (exporter->last_common_uid == 0) {
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen /* we're syncing all mails, so we can request the wanted
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen fields for all the mails */
8bc87e22fecd20797112b86778a961b08dc1f5c8Timo Sirainen wanted_fields = MAIL_FETCH_GUID;
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen wanted_headers = exporter->wanted_headers;
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen }
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen
ac51b12850a6740c2152e4e78cf2abe3aa620391Timo Sirainen exporter->trans = mailbox_transaction_begin(exporter->box,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi MAILBOX_TRANSACTION_FLAG_SYNC,
0dab9cb35a976c49b28a11e28d5570f5191f1a7aMartti Rannanjärvi __func__);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_ctx = mailbox_search_init(exporter->trans, search_args, NULL,
95de57450afe1e92a6dd57538fba4db0925fa43bTimo Sirainen wanted_fields, wanted_headers);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_search_args_unref(&search_args);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (mailbox_search_next(search_ctx, &mail)) {
07c8b1b48eba7ad2693dbfeaa9916d837f006262Timo Sirainen T_BEGIN {
07c8b1b48eba7ad2693dbfeaa9916d837f006262Timo Sirainen if (mail->uid <= exporter->last_common_uid)
07c8b1b48eba7ad2693dbfeaa9916d837f006262Timo Sirainen ret = search_update_flag_change_guid(exporter, mail);
07c8b1b48eba7ad2693dbfeaa9916d837f006262Timo Sirainen else
07c8b1b48eba7ad2693dbfeaa9916d837f006262Timo Sirainen ret = search_add_save(exporter, mail);
07c8b1b48eba7ad2693dbfeaa9916d837f006262Timo Sirainen } T_END;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (ret < 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen break;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
b0a06fd1ca37f4626037a62be5c7f0ce12395013Timo Sirainen i_assert(ret >= 0 || exporter->error != NULL);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_drop_expunged_flag_changes(exporter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_search_deinit(&search_ctx) < 0 &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "Mail search failed: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(exporter->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &exporter->mail_error));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int dsync_mail_change_p_uid_cmp(struct dsync_mail_change *const *c1,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *const *c2)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((*c1)->uid < (*c2)->uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((*c1)->uid > (*c2)->uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct hash_iterate_context *iter;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen void *key;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen struct dsync_mail_change *change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&exporter->sorted_changes, exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_count(exporter->changes));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(exporter->changes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, exporter->changes, &key, &change))
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&exporter->sorted_changes, &change, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_iterate_deinit(&iter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_sort(&exporter->sorted_changes, dsync_mail_change_p_uid_cmp);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainenstatic void
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen enum mail_attribute_type type)
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen{
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->attr_iter =
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen mailbox_attribute_iter_init(exporter->box, type, "");
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->attr_type = type;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen}
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_log_scan(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_transaction_log_scan *log_scan)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE_TYPE(dsync_uid_mail_change) log_changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct hash_iterate_context *iter;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen void *key;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen struct dsync_mail_change *change, *dup_change;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
08e9fec5ba9e1a26e658c4224207d666b6ced27dTimo Sirainen log_changes = dsync_transaction_log_scan_get_hash(log_scan);
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen if (dsync_transaction_log_scan_has_all_changes(log_scan)) {
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen /* we tried to access too old/invalid modseqs. to make sure
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen no changes get lost, we need to send all of the messages */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->return_all_mails = TRUE;
c307328f59c963eba21091ecd36c9435d42b47d8Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* clone the hash table, since we're changing it. */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create_direct(&exporter->changes, exporter->pool,
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_count(log_changes));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(log_changes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, log_changes, &key, &change)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dup_change = p_new(exporter->pool, struct dsync_mail_change, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *dup_change = *change;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen hash_table_insert(exporter->changes, key, dup_change);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->highest_changed_uid < change->uid)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->highest_changed_uid = change->uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_iterate_deinit(&iter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstruct dsync_mailbox_exporter *
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_init(struct mailbox *box,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_transaction_log_scan *log_scan,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t last_common_uid,
f3c24c2c92802cb773315eba1132254932d8709bTimo Sirainen enum dsync_mailbox_exporter_flags flags,
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi unsigned int hdr_hash_version,
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi const char *const *hashed_headers)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mailbox_exporter *exporter;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool_t pool;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox export",
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen 4096);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter = p_new(pool, struct dsync_mailbox_exporter, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->pool = pool;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->box = box;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->log_scan = log_scan;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->last_common_uid = last_common_uid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->auto_export_mails =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->mails_have_guids =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen exporter->minimal_dmail_fill =
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen exporter->export_received_timestamps =
3561c7bb472a78af74d755219cc0fc71c85ff5c2Timo Sirainen (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen exporter->export_virtual_sizes =
ae949831f1f668b5501b4b125e7f7b1767fb109bTimo Sirainen (flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0;
f3c24c2c92802cb773315eba1132254932d8709bTimo Sirainen exporter->hdr_hash_version = hdr_hash_version;
03af8e5325a7b4fec36414ac35949457bc426c0bTimo Sirainen exporter->no_hdr_hashes =
03af8e5325a7b4fec36414ac35949457bc426c0bTimo Sirainen (flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0;
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi exporter->hashed_headers = hashed_headers;
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&exporter->requested_uids, pool, 16);
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen p_array_init(&exporter->search_uids, pool, 16);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&exporter->expunged_seqs, pool, 16);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&exporter->expunged_guids, pool, 16);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
03af8e5325a7b4fec36414ac35949457bc426c0bTimo Sirainen if (!exporter->mails_have_guids && !exporter->no_hdr_hashes)
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi exporter->wanted_headers =
69a71891361b2b27ff68ed84b29278486628464aAki Tuomi dsync_mail_get_hash_headers(box, exporter->hashed_headers);
f97c983e7a742e1d54ec60e502093abbb3ad2907Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* first scan transaction log and save any expunges and flag changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_log_scan(exporter, log_scan);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get saves and also find GUIDs for flag changes */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_search(exporter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get the changes sorted by UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_sort_changes(exporter);
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return exporter;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainenstatic int
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen{
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct dsync_mailbox_attribute *attr;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct mail_attribute_value value;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
791fb70b3255a11a91ce0c2dc3ae1460d4cf8459Timo Sirainen while (hash_table_iterate(exporter->attr_change_iter,
791fb70b3255a11a91ce0c2dc3ae1460d4cf8459Timo Sirainen dsync_transaction_log_scan_get_attr_hash(exporter->log_scan),
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen &attr, &attr)) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr->exported || !attr->deleted)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen continue;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* lookup the value mainly to get its last_change value. */
66c87722e0fd2a85cd59797326bad3d1c409dc3aAki Tuomi if (mailbox_attribute_get_stream(exporter->box, attr->type,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->key, &value) < 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen "Mailbox attribute %s lookup failed: %s", attr->key,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(exporter->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &exporter->mail_error));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen break;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek i_stream_unref(&value.value_stream);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen continue;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->last_change = value.last_change;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (value.value != NULL || value.value_stream != NULL) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->value = p_strdup(exporter->pool, value.value);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->value_stream = value.value_stream;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->deleted = FALSE;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->exported = TRUE;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->attr = *attr;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 1;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen hash_table_iterate_deinit(&exporter->attr_change_iter);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen}
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainenstatic int
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainendsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen{
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct dsync_mailbox_attribute lookup_attr, *attr;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct dsync_mailbox_attribute *attr_change;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen const char *key;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen struct mail_attribute_value value;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen bool export_all_attrs;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen export_all_attrs = exporter->return_all_mails ||
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->last_common_uid == 0;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen lookup_attr.type = exporter->attr_type;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* note that the order of processing may be important for some
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attributes. for example sieve can't set a script active until it's
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen first been created */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen lookup_attr.key = key;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr_change = hash_table_lookup(attr_changes, &lookup_attr);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr_change == NULL && !export_all_attrs)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen continue;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
66c87722e0fd2a85cd59797326bad3d1c409dc3aAki Tuomi if (mailbox_attribute_get_stream(exporter->box,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->attr_type, key,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen &value) < 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen "Mailbox attribute %s lookup failed: %s", key,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(exporter->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &exporter->mail_error));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return -1;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* readonly attributes can't be changed,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen no point in exporting them */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (value.value_stream != NULL)
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen i_stream_unref(&value.value_stream);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen continue;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (value.value == NULL && value.value_stream == NULL &&
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen (attr_change == NULL || !attr_change->deleted)) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* the attribute was just deleted?
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen skip for this sync. */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen continue;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
2da176a0b105bc75a3289facb4b467e98bc04829Timo Sirainen if (attr_change != NULL && attr_change->exported) {
2da176a0b105bc75a3289facb4b467e98bc04829Timo Sirainen /* duplicate attribute returned.
2da176a0b105bc75a3289facb4b467e98bc04829Timo Sirainen shouldn't normally happen, but don't crash. */
2da176a0b105bc75a3289facb4b467e98bc04829Timo Sirainen i_warning("Ignoring duplicate attributes '%s'", key);
2da176a0b105bc75a3289facb4b467e98bc04829Timo Sirainen continue;
2da176a0b105bc75a3289facb4b467e98bc04829Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr = &exporter->attr;
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(attr);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->type = exporter->attr_type;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->value = p_strdup(exporter->pool, value.value);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->value_stream = value.value_stream;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->last_change = value.last_change;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (attr_change != NULL) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr_change->exported = TRUE;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->key = attr_change->key;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->deleted = attr_change->deleted &&
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen !DSYNC_ATTR_HAS_VALUE(attr);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->modseq = attr_change->modseq;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen } else {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen attr->key = p_strdup(exporter->pool, key);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return 1;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen "Mailbox attribute iteration failed: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(exporter->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &exporter->mail_error));
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return -1;
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) {
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen /* export shared attributes */
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen dsync_mailbox_export_attr_init(exporter,
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen MAIL_ATTRIBUTE_TYPE_SHARED);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return dsync_mailbox_export_iter_next_attr(exporter);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen exporter->attr_change_iter = hash_table_iterate_init(attr_changes);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen return dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen}
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainenint dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter,
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen const struct dsync_mailbox_attribute **attr_r)
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen{
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen int ret;
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen if (exporter->error != NULL)
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return -1;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek i_stream_unref(&exporter->attr.value_stream);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen if (exporter->attr_iter != NULL) {
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen ret = dsync_mailbox_export_iter_next_attr(exporter);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen } else {
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen ret = dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
41e51b972f02e8b16c19fab9160294ea0a07c343Timo Sirainen }
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen if (ret > 0)
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen *attr_r = &exporter->attr;
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return ret;
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen}
55d33f807765482eb47374aaaced1fe714e0b256Timo Sirainen
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainenint dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter,
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen const struct dsync_mail_change **change_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_change *const *changes;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->error != NULL)
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen changes = array_get(&exporter->sorted_changes, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->change_idx == count)
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return 0;
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen *change_r = changes[exporter->change_idx++];
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_args *search_args;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail_search_arg *sarg;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct hash_iterate_context *iter;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct seq_range *uids;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen char *guid;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen const char *const_guid;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen enum mail_fetch_field wanted_fields;
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen struct dsync_mail_guid_instances *instances;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct seq_range *range;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int i, count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uint32_t seq, seq1, seq2;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(exporter->search_ctx == NULL);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen search_args = mail_search_build_init();
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen sarg = mail_search_build_add(search_args, SEARCH_SEQSET);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen p_array_init(&sarg->value.seqset, search_args->pool, 128);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* get a list of messages we want to fetch. if there are more than one
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instance for a GUID, use the first one. */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen iter = hash_table_iterate_init(exporter->export_guids);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, exporter->export_guids,
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen &guid, &instances)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!instances->requested ||
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_count(&instances->seqs) == 0)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen continue;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uids = array_idx(&instances->seqs, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq = uids[0].seq1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!instances->searched) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances->searched = TRUE;
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&sarg->value.seqset, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else if (seq_range_exists(&exporter->expunged_seqs, seq)) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* we're on a second round, refetching expunged
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen messages */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_remove(&instances->seqs, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_remove(&exporter->expunged_seqs, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (array_count(&instances->seqs) == 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* no instances left */
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen const_guid = guid;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_append(&exporter->expunged_guids,
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen &const_guid, 1);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen continue;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen uids = array_idx(&instances->seqs, 0);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq = uids[0].seq1;
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&sarg->value.seqset, seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_iterate_deinit(&iter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* add requested UIDs */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen range = array_get(&exporter->requested_uids, &count);
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen for (i = 0; i < count; i++) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_get_seq_range(exporter->box,
789cb161ef9cb3126e6370c87a21a5a844d7dd0bTimo Sirainen range[i].seq1, range[i].seq2,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen &seq1, &seq2);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq_range_array_add_range(&sarg->value.seqset,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen seq1, seq2);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen array_clear(&exporter->search_uids);
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen array_append_array(&exporter->search_uids, &exporter->requested_uids);
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen array_clear(&exporter->requested_uids);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (!exporter->minimal_dmail_fill) {
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen wanted_fields |= MAIL_FETCH_RECEIVED_DATE |
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER |
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY;
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen }
723f7be99e6de0f3905394283679acde0f1f2cceTimo Sirainen exporter->search_count += seq_range_count(&sarg->value.seqset);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->search_ctx =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mailbox_search_init(exporter->trans, search_args, NULL,
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen wanted_fields, NULL);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen mail_search_args_unref(&search_args);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return array_count(&sarg->value.seqset) > 0 ? 1 : 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic void
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainendsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter *exporter)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->search_ctx == NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (mailbox_search_deinit(&exporter->search_ctx) < 0 &&
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "Mail search failed: %s",
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi mailbox_get_last_internal_error(exporter->box,
bf7dc750b95039981c0e9d728f313d50cf38a156Martti Rannanjärvi &exporter->mail_error));
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenstatic int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_guid_instances *instances;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen const char *error_field;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen if (dsync_mail_fill(mail, exporter->minimal_dmail_fill,
d519a0449d0e536a32db93305516fdbd7db6773dTimo Sirainen &exporter->dsync_mail, &error_field) < 0)
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen return dsync_mail_error(exporter, mail, error_field);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen instances = *exporter->dsync_mail.guid == '\0' ? NULL :
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen hash_table_lookup(exporter->export_guids,
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen exporter->dsync_mail.guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (instances != NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* GUID found */
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen } else if (exporter->dsync_mail.uid != 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* mail requested by UID */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen } else {
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen exporter->mail_error = MAIL_ERROR_TEMP;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "GUID unexpectedly changed for UID=%u GUID=%s",
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen mail->uid, exporter->dsync_mail.guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
3d651cf1a2171471bf6f0a41dcf7388d43206636Timo Sirainen if (!seq_range_exists(&exporter->search_uids, mail->uid))
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen exporter->dsync_mail.uid = 0;
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen else
d9b9687bf8cae9cfb070b1b7aadefa683220269fTimo Sirainen exporter->dsync_mail.guid = "";
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* this message was successfully returned, don't try retrying it */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (instances != NULL)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen array_clear(&instances->seqs);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenvoid dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const struct dsync_mail_request *request)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mail_guid_instances *instances;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(!exporter->auto_export_mails);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
d8bdf558c7ba173fc47a194633d9bd97af1b9c74Timo Sirainen if (request->guid == NULL) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(request->uid > 0);
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&exporter->requested_uids, request->uid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances = hash_table_lookup(exporter->export_guids, request->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (instances == NULL) {
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen exporter->mail_error = MAIL_ERROR_TEMP;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->error = p_strdup_printf(exporter->pool,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen "Remote requested unexpected GUID %s", request->guid);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen instances->requested = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainenint dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter,
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen const struct dsync_mail **mail_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct mail *mail;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen const char *const *guids;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen unsigned int count;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen int ret;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->error != NULL)
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (!exporter->body_search_initialized) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->body_search_initialized = TRUE;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (dsync_mailbox_export_body_search_init(exporter) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(exporter->error != NULL);
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen while (mailbox_search_next(exporter->search_ctx, &mail)) {
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen exporter->search_pos++;
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen if ((ret = dsync_mailbox_export_mail(exporter, mail)) > 0) {
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen *mail_r = &exporter->dsync_mail;
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return 1;
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (ret < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(exporter->error != NULL);
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* the message was expunged. if the GUID has another instance,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen try sending it later. */
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen seq_range_array_add(&exporter->expunged_seqs, mail->seq);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* if some instances of messages were expunged, retry fetching them
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen with other instances */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_body_search_deinit(exporter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if ((ret = dsync_mailbox_export_body_search_init(exporter)) < 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen i_assert(exporter->error != NULL);
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return -1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (ret > 0) {
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* not finished yet */
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return dsync_mailbox_export_next_mail(exporter, mail_r);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen /* finished with messages. if there are any expunged messages,
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen return them */
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen guids = array_get(&exporter->expunged_guids, &count);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen if (exporter->expunged_guid_idx < count) {
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&exporter->dsync_mail);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen exporter->dsync_mail.guid =
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen guids[exporter->expunged_guid_idx++];
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen *mail_r = &exporter->dsync_mail;
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return 1;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen }
e1bef591ed45c5baf6b5bdc69fb7fa5ff05df2b6Timo Sirainen return 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainenint dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **_exporter,
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen const char **errstr_r, enum mail_error *error_r)
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen{
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen struct dsync_mailbox_exporter *exporter = *_exporter;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen *_exporter = NULL;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
df74814b12ef891a90522b1ced231350afaf1232Timo Sirainen if (exporter->attr_iter != NULL)
df74814b12ef891a90522b1ced231350afaf1232Timo Sirainen (void)mailbox_attribute_iter_deinit(&exporter->attr_iter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen dsync_mailbox_export_body_search_deinit(exporter);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen (void)mailbox_transaction_commit(&exporter->trans);
ceb8c97c6c9fe0ee7eb544645c6bdb74dfcb519dJosef 'Jeff' Sipek mailbox_header_lookup_unref(&exporter->wanted_headers);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek i_stream_unref(&exporter->attr.value_stream);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen hash_table_destroy(&exporter->export_guids);
237a6211c7fc4d6dbb58dd0467da6dba1b8f21f6Timo Sirainen hash_table_destroy(&exporter->changes);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen i_assert((exporter->error != NULL) == (exporter->mail_error != 0));
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen *error_r = exporter->mail_error;
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen *errstr_r = t_strdup(exporter->error);
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen pool_unref(&exporter->pool);
ce0e25f26d6e67480ee39b5ca0ad634fa60c4605Timo Sirainen return *errstr_r != NULL ? -1 : 0;
7bd5b1c64cc987715bdaf8cc4907c3c37d5d7b29Timo Sirainen}
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainenconst char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter)
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen{
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen if (exporter->search_ctx == NULL)
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen return "";
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen return t_strdup_printf("%u/%u", exporter->search_pos,
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen exporter->search_count);
47255691575e06a1c95ce78ff0a1b502199de3abTimo Sirainen}