24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "lib.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "array.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "bsearch-insert-pos.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "ioloop.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "str.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "mail-index-modseq.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "mail-search-build.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "mailbox-search-result-private.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "index-sync-private.h"

7a7d2aa11e46195e2d92d6c337d7e78052a5ce67Timo Sirainen#include "index-search-result.h"

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include "virtual-storage.h"

c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include <stdlib.h>

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstruct virtual_add_record {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct virtual_mail_index_record rec;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen time_t received_date;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen};

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstruct virtual_sync_mail {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uint32_t vseq;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct virtual_mail_index_record vrec;

499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen};

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainenstruct virtual_sync_context {

f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen struct virtual_mailbox *mbox;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct mail_index_sync_ctx *index_sync_ctx;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen struct mail_index *index;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen struct mail_index_view *sync_view;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen struct mail_index_transaction *trans;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen const char *const *kw_all;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen /* messages expunged within this sync */

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen ARRAY_TYPE(seq_range) sync_expunges;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen ARRAY_DEFINE(all_adds, struct virtual_add_record);

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen enum mailbox_sync_flags flags;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen uint32_t uid_validity;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen unsigned int ext_header_changed:1;

c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen unsigned int ext_header_rewrite:1;

c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen unsigned int expunge_removed:1;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen unsigned int index_broken:1;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen};

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen

942302b0247403645394d848b3c620ead262a2a5Timo Sirainenstatic void virtual_sync_set_uidvalidity(struct virtual_sync_context *ctx)

942302b0247403645394d848b3c620ead262a2a5Timo Sirainen{

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen uint32_t uid_validity = ioloop_time;

942302b0247403645394d848b3c620ead262a2a5Timo Sirainen

942302b0247403645394d848b3c620ead262a2a5Timo Sirainen mail_index_update_header(ctx->trans,

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen offsetof(struct mail_index_header, uid_validity),

11352dc3e4b29f3d2763c82f8ea4f99e8daf4fa3Timo Sirainen &uid_validity, sizeof(uid_validity), TRUE);

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen ctx->uid_validity = uid_validity;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen}

942302b0247403645394d848b3c620ead262a2a5Timo Sirainen

942302b0247403645394d848b3c620ead262a2a5Timo Sirainenstatic void virtual_sync_external_flags(struct virtual_sync_context *ctx,

942302b0247403645394d848b3c620ead262a2a5Timo Sirainen struct virtual_backend_box *bbox,

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen uint32_t vseq, uint32_t real_uid)

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen{

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen enum mail_flags flags;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen const char *const *kw_names;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct mail_keywords *keywords;

cd8507179823de33d6e8242e10dbc15d136245b5Timo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (!mail_set_uid(bbox->sync_mail, real_uid)) {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_panic("UID %u lost unexpectedly from %s",

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen real_uid, bbox->box->name);

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen }

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* copy flags */

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen flags = mail_get_flags(bbox->sync_mail);

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_update_flags(ctx->trans, vseq, MODIFY_REPLACE, flags);

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* copy keywords */

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen kw_names = mail_get_keywords(bbox->sync_mail);

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen keywords = mail_index_keywords_create(ctx->index, kw_names);

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_update_keywords(ctx->trans, vseq, MODIFY_REPLACE, keywords);

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_keywords_unref(&keywords);

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen}

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainenstatic int virtual_sync_mail_cmp(const void *p1, const void *p2)

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen{

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen const struct virtual_sync_mail *m1 = p1, *m2 = p2;

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (m1->vrec.mailbox_id < m2->vrec.mailbox_id)

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen return -1;

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (m1->vrec.mailbox_id > m2->vrec.mailbox_id)

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen return 1;

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (m1->vrec.real_uid < m2->vrec.real_uid)

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen return -1;

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (m1->vrec.real_uid > m2->vrec.real_uid)

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen return 1;

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen /* broken */

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen return 0;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen}

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenvirtual_backend_box_sync_mail_set(struct virtual_backend_box *bbox)

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen{

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct mailbox_transaction_context *trans;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (bbox->sync_mail == NULL) {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen trans = mailbox_transaction_begin(bbox->box, 0);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen bbox->sync_mail = mail_alloc(trans, 0, NULL);

4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen }

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen}

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenvirtual_backend_box_sync_mail_unset(struct virtual_backend_box *bbox)

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen{

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct mailbox_transaction_context *trans;

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen if (bbox->sync_mail != NULL) {

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen trans = bbox->sync_mail->transaction;

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen mail_free(&bbox->sync_mail);

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen (void)mailbox_transaction_commit(&trans);

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen }

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen}

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainenstatic int bbox_mailbox_id_cmp(struct virtual_backend_box *const *b1,

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen struct virtual_backend_box *const *b2)

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen{

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen if ((*b1)->mailbox_id < (*b2)->mailbox_id)

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen return -1;

c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen if ((*b1)->mailbox_id > (*b2)->mailbox_id)

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen return 1;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen return 0;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen}

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainenstatic int

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainenvirtual_sync_get_backend_box(struct virtual_sync_context *ctx, const char *name,

11352dc3e4b29f3d2763c82f8ea4f99e8daf4fa3Timo Sirainen struct virtual_backend_box **bbox_r)

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen{

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen *bbox_r = virtual_backend_box_lookup_name(ctx->mbox, name);

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen if (*bbox_r != NULL || !ctx->mbox->sync_initialized)

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen return 0;

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen /* another process just added a new mailbox.

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen ctx->mbox->inconsistent = TRUE;

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen mail_storage_set_error(ctx->mbox->ibox.box.storage, MAIL_ERROR_TEMP,

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen "Backend mailbox added by another session. "

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen "Reopen the virtual mailbox.");

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen return -1;

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen}

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen

755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainenstatic int virtual_sync_ext_header_read(struct virtual_sync_context *ctx)

8305127d1074cf9a1e25dec9be2735276462079dTimo Sirainen{

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen const struct virtual_mail_index_header *ext_hdr;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen const struct mail_index_header *hdr;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen const struct virtual_mail_index_mailbox_record *mailboxes;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen struct virtual_backend_box *bbox, **bboxes;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen const void *ext_data;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen size_t ext_size;

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen unsigned int i, count, ext_name_offset, ext_mailbox_count;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen uint32_t prev_mailbox_id;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen int ret = 1;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen hdr = mail_index_get_header(ctx->sync_view);

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_get_header_ext(ctx->sync_view, ctx->mbox->virtual_ext_id,

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen &ext_data, &ext_size);

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen ext_hdr = ext_data;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (ctx->mbox->sync_initialized &&

596433ccbca59ce2328dc1d029586154cd937155Timo Sirainen ctx->mbox->prev_uid_validity == hdr->uid_validity &&

596433ccbca59ce2328dc1d029586154cd937155Timo Sirainen ext_size >= sizeof(*ext_hdr) &&

4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen ctx->mbox->prev_change_counter == ext_hdr->change_counter) {

4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen /* fully refreshed */

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return TRUE;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen }

c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen

e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainen ctx->mbox->prev_uid_validity = hdr->uid_validity;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen if (ext_hdr == NULL ||

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen ctx->mbox->search_args_crc32 != ext_hdr->search_args_crc32) {

132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen mailboxes = NULL;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ext_name_offset = 0;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ext_mailbox_count = 0;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen } else {

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ctx->mbox->prev_change_counter = ext_hdr->change_counter;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen mailboxes = (const void *)(ext_hdr + 1);

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ext_name_offset = sizeof(*ext_hdr) +

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ext_hdr->mailbox_count * sizeof(*mailboxes);

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen if (ext_name_offset >= ext_size ||

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ext_hdr->mailbox_count > INT_MAX/sizeof(*mailboxes)) {

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen i_error("virtual index %s: Broken mailbox_count header",

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ctx->mbox->ibox.box.path);

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ctx->index_broken = TRUE;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ext_mailbox_count = 0;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen ret = 0;

7f3b826a89bcb7a72759912e99f574b28309fe1bTimo Sirainen } else {

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen ext_mailbox_count = ext_hdr->mailbox_count;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen }

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen }

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen /* update mailbox backends */

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen prev_mailbox_id = 0;

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen for (i = 0; i < ext_mailbox_count; i++) {

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen if (mailboxes[i].id > ext_hdr->highest_mailbox_id ||

6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen mailboxes[i].id <= prev_mailbox_id) {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_error("virtual index %s: Broken mailbox id",

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen ctx->mbox->ibox.box.path);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen break;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (mailboxes[i].name_len == 0 ||

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mailboxes[i].name_len > ext_size) {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_error("virtual index %s: Broken mailbox name_len",

e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen ctx->mbox->ibox.box.path);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen break;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (ext_name_offset + mailboxes[i].name_len > ext_size) {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_error("virtual index %s: Broken mailbox list",

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen ctx->mbox->ibox.box.path);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen break;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen T_BEGIN {

c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen const unsigned char *nameptr;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen const char *name;

18ccd19c244f49665fe03cda785efa066d2c38dfTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen nameptr = CONST_PTR_OFFSET(ext_data, ext_name_offset);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen name = t_strndup(nameptr, mailboxes[i].name_len);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (virtual_sync_get_backend_box(ctx, name, &bbox) < 0)

c529313e1cbc22244d4528e80aa3e485f8806cd3Timo Sirainen ret = -1;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen } T_END;

d22301419109ed4a38351715e6760011421dadecTimo Sirainen

d22301419109ed4a38351715e6760011421dadecTimo Sirainen if (bbox == NULL) {

d22301419109ed4a38351715e6760011421dadecTimo Sirainen if (ret < 0)

d22301419109ed4a38351715e6760011421dadecTimo Sirainen return -1;

d22301419109ed4a38351715e6760011421dadecTimo Sirainen /* mailbox no longer exists. */

499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen ret = 0;

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen } else {

c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen bbox->mailbox_id = mailboxes[i].id;

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen bbox->sync_uid_validity = mailboxes[i].uid_validity;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen bbox->sync_highest_modseq = mailboxes[i].highest_modseq;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen bbox->sync_next_uid = mailboxes[i].next_uid;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen bbox->sync_mailbox_idx = i;

d22301419109ed4a38351715e6760011421dadecTimo Sirainen }

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen ext_name_offset += mailboxes[i].name_len;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen prev_mailbox_id = mailboxes[i].id;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen }

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen if (i < ext_mailbox_count) {

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ctx->index_broken = TRUE;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ret = 0;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen }

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ctx->mbox->highest_mailbox_id = ext_hdr == NULL ? 0 :

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ext_hdr->highest_mailbox_id;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ctx->mbox->sync_initialized = TRUE;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen /* assign new mailbox IDs if any are missing */

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen bboxes = array_get_modifiable(&ctx->mbox->backend_boxes, &count);

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen for (i = 0; i < count; i++) {

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen if (bboxes[i]->mailbox_id == 0) {

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen bboxes[i]->mailbox_id = ++ctx->mbox->highest_mailbox_id;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ret = 0;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen }

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen }

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen /* sort the backend mailboxes by mailbox_id. */

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen array_sort(&ctx->mbox->backend_boxes, bbox_mailbox_id_cmp);

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen return ret;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen}

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen

c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainenstatic void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx)

c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen{

c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen struct virtual_mail_index_header ext_hdr;

047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen struct virtual_mail_index_mailbox_record mailbox;

c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen struct virtual_backend_box **bboxes;

047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen buffer_t *buf;

047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen const void *ext_data;

047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen size_t ext_size;

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen unsigned int i, mailbox_pos, name_pos, count;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen bboxes = array_get_modifiable(&ctx->mbox->backend_boxes, &count);

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen mailbox_pos = sizeof(ext_hdr);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen name_pos = mailbox_pos + sizeof(mailbox) * count;

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen memset(&ext_hdr, 0, sizeof(ext_hdr));

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen memset(&mailbox, 0, sizeof(mailbox));

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen ext_hdr.change_counter = ++ctx->mbox->prev_change_counter;

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen ext_hdr.mailbox_count = count;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ext_hdr.highest_mailbox_id = ctx->mbox->highest_mailbox_id;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ext_hdr.search_args_crc32 = ctx->mbox->search_args_crc32;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), name_pos + 256);

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen buffer_append(buf, &ext_hdr, sizeof(ext_hdr));

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen for (i = 0; i < count; i++) {

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen i_assert(i == 0 ||

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen bboxes[i]->mailbox_id > bboxes[i-1]->mailbox_id);

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen bboxes[i]->sync_mailbox_idx = i;

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen mailbox.id = bboxes[i]->mailbox_id;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mailbox.name_len = strlen(bboxes[i]->name);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mailbox.uid_validity = bboxes[i]->sync_uid_validity;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mailbox.highest_modseq = bboxes[i]->sync_highest_modseq;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mailbox.next_uid = bboxes[i]->sync_next_uid;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen buffer_write(buf, mailbox_pos, &mailbox, sizeof(mailbox));

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen buffer_write(buf, name_pos, bboxes[i]->name, mailbox.name_len);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mailbox_pos += sizeof(mailbox);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen name_pos += mailbox.name_len;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen i_assert(buf->used == name_pos);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_index_get_header_ext(ctx->sync_view, ctx->mbox->virtual_ext_id,

942302b0247403645394d848b3c620ead262a2a5Timo Sirainen &ext_data, &ext_size);

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen if (ext_size < name_pos) {

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen mail_index_ext_resize(ctx->trans, ctx->mbox->virtual_ext_id,

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen name_pos,

18ccd19c244f49665fe03cda785efa066d2c38dfTimo Sirainen sizeof(struct virtual_mail_index_record),

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen sizeof(uint32_t));

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen }

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,

db0735f9b388c5bcfb781b1b25015e898d63d953Timo Sirainen 0, buf->data, name_pos);

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen}

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void virtual_sync_ext_header_update(struct virtual_sync_context *ctx)

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen{

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen struct virtual_mail_index_header ext_hdr;

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen if (!ctx->ext_header_changed)

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen return;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen /* we changed something - update the change counter in header */

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen ext_hdr.change_counter = ++ctx->mbox->prev_change_counter;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen offsetof(struct virtual_mail_index_header, change_counter),

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen &ext_hdr.change_counter, sizeof(ext_hdr.change_counter));

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen}

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainenstatic void virtual_sync_index_rec(struct virtual_sync_context *ctx,

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen const struct mail_index_sync_rec *sync_rec)

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen{

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen struct virtual_backend_box *bbox;

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen const struct virtual_mail_index_record *vrec;

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen const void *data;

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen enum mail_flags flags;

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen struct mail_keywords *keywords;

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen enum modify_type modify_type;

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen const char *kw_names[2];

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen uint32_t vseq, seq1, seq2;

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen bool expunged;

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen switch (sync_rec->type) {

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen case MAIL_INDEX_SYNC_TYPE_APPEND:

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen /* don't care */

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen return;

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen case MAIL_INDEX_SYNC_TYPE_EXPUNGE:

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen case MAIL_INDEX_SYNC_TYPE_FLAGS:

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen break;

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen }

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen if (!mail_index_lookup_seq_range(ctx->sync_view,

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen sync_rec->uid1, sync_rec->uid2,

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen &seq1, &seq2)) {

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen /* already expunged, nothing to do. */

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen return;

d859478e8b106de6cea54f26861bd4232c92f62cTimo Sirainen }

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen for (vseq = seq1; vseq <= seq2; vseq++) {

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen mail_index_lookup_ext(ctx->sync_view, vseq, virtual_ext_id,

d859478e8b106de6cea54f26861bd4232c92f62cTimo Sirainen &data, &expunged);

206ed2f6fa3a6fb291498627b2da626581c07a18Timo Sirainen vrec = data;

206ed2f6fa3a6fb291498627b2da626581c07a18Timo Sirainen

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen bbox = virtual_backend_box_lookup(ctx->mbox, vrec->mailbox_id);

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen if (bbox == NULL)

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen continue;

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen virtual_backend_box_sync_mail_set(bbox);

206ed2f6fa3a6fb291498627b2da626581c07a18Timo Sirainen if (!mail_set_uid(bbox->sync_mail, vrec->real_uid))

d859478e8b106de6cea54f26861bd4232c92f62cTimo Sirainen i_panic("UID lost unexpectedly");

d859478e8b106de6cea54f26861bd4232c92f62cTimo Sirainen

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen switch (sync_rec->type) {

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen case MAIL_INDEX_SYNC_TYPE_EXPUNGE:

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen mail_expunge(bbox->sync_mail);

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen break;

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen case MAIL_INDEX_SYNC_TYPE_FLAGS:

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen flags = sync_rec->add_flags & MAIL_FLAGS_NONRECENT;

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen if (flags != 0) {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_update_flags(bbox->sync_mail,

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen MODIFY_ADD, flags);

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen }

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen flags = sync_rec->remove_flags & MAIL_FLAGS_NONRECENT;

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen if (flags != 0) {

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen mail_update_flags(bbox->sync_mail,

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen MODIFY_REMOVE, flags);

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen }

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen break;

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:

9438ecaf1caee1bb33c8d7f638742875ac42c4e5Timo Sirainen kw_names[0] = ctx->kw_all[sync_rec->keyword_idx];

9438ecaf1caee1bb33c8d7f638742875ac42c4e5Timo Sirainen kw_names[1] = NULL;

77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen keywords = mailbox_keywords_create_valid(bbox->box,

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen kw_names);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen modify_type = sync_rec->type ==

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD ?

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen MODIFY_ADD : MODIFY_REMOVE;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mail_update_keywords(bbox->sync_mail,

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen modify_type, keywords);

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen mailbox_keywords_unref(bbox->box, &keywords);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen break;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen kw_names[0] = NULL;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen keywords = mailbox_keywords_create_valid(bbox->box,

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen kw_names);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mail_update_keywords(bbox->sync_mail, MODIFY_REPLACE,

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen keywords);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mailbox_keywords_unref(bbox->box, &keywords);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen break;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen case MAIL_INDEX_SYNC_TYPE_APPEND:

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i_unreached();

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic void virtual_sync_index_changes(struct virtual_sync_context *ctx)

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen{

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen const ARRAY_TYPE(keywords) *keywords;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct mail_index_sync_rec sync_rec;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen keywords = mail_index_get_keywords(ctx->index);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen ctx->kw_all = array_count(keywords) == 0 ? NULL :

86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen array_idx(keywords, 0);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec))

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen virtual_sync_index_rec(ctx, &sync_rec);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainenstatic void virtual_sync_index_finish(struct virtual_sync_context *ctx)

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen{

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct mailbox *box = &ctx->mbox->ibox.box;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen const struct mail_index_header *hdr;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen uint32_t seq1, seq2;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen hdr = mail_index_get_header(ctx->sync_view);

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen if (hdr->uid_validity != 0)

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen ctx->uid_validity = hdr->uid_validity;

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen else

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen virtual_sync_set_uidvalidity(ctx);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* mark the newly seen messages as recent */

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen hdr->next_uid, &seq1, &seq2)) {

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen index_mailbox_set_recent_seq(&ctx->mbox->ibox, ctx->sync_view,

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen seq1, seq2);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen if (ctx->ext_header_rewrite) {

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* entire mailbox list needs to be rewritten */

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen virtual_sync_ext_header_rewrite(ctx);

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen } else {

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen /* update only changed parts in the header */

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen virtual_sync_ext_header_update(ctx);

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen }

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (box->v.sync_notify != NULL)

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen box->v.sync_notify(box, 0, 0);

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic int virtual_sync_backend_box_init(struct virtual_backend_box *bbox)

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen{

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct mailbox_transaction_context *trans;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct mail_search_context *search_ctx;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct mail *mail;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct virtual_backend_uidmap uidmap;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen enum mailbox_search_result_flags result_flags;

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen int ret;

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen

ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen trans = mailbox_transaction_begin(bbox->box, 0);

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen mail = mail_alloc(trans, 0, NULL);

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen search_ctx = mailbox_search_init(trans, bbox->search_args, NULL);

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* save the result and keep it updated */

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen result_flags = MAILBOX_SEARCH_RESULT_FLAG_UPDATE |

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC;

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen bbox->search_result =

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen mailbox_search_result_save(search_ctx, result_flags);

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen /* add the found UIDs to uidmap. virtual_uid gets assigned later. */

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen memset(&uidmap, 0, sizeof(uidmap));

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen array_clear(&bbox->uids);

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen while (mailbox_search_next(search_ctx, mail) > 0) {

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen uidmap.real_uid = mail->uid;

3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen array_append(&bbox->uids, &uidmap, 1);

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen }

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen ret = mailbox_search_deinit(&search_ctx);

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen mail_free(&mail);

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen (void)mailbox_transaction_commit(&trans);

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen return ret;

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen}

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainenstatic int

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenvirtual_backend_uidmap_bsearch_cmp(const uint32_t *uidp,

7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen const struct virtual_backend_uidmap *uidmap)

4ba9a1d3facc515b3feb5238a16bcf91f76fac61Timo Sirainen{

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return *uidp < uidmap->real_uid ? -1 :

7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen (*uidp > uidmap->real_uid ? 1 : 0);

a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen

563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainenstatic void

c6afd726060aae56b6622c6c52aec10231c4bf1cTimo Sirainenvirtual_sync_mailbox_box_remove(struct virtual_sync_context *ctx,

047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen struct virtual_backend_box *bbox,

2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen const ARRAY_TYPE(seq_range) *removed_uids)

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen{

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen const struct seq_range *uids;

34f4c7610b846a945779b6be78d1ef575c7d0ca8Timo Sirainen struct virtual_backend_uidmap *uidmap;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen unsigned int i, src, dest, uid_count, rec_count;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uint32_t uid, vseq;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uids = array_get(removed_uids, &uid_count);

7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen if (uid_count == 0)

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return;

db0735f9b388c5bcfb781b1b25015e898d63d953Timo Sirainen

047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen /* everything in removed_uids should exist in bbox->uids */

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen uidmap = array_get_modifiable(&bbox->uids, &rec_count);

1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen i_assert(rec_count >= uid_count);

eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen /* find the first uidmap record to be removed */

17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen if (!array_bsearch_insert_pos(&bbox->uids, &uids[0].seq1,

61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen virtual_backend_uidmap_bsearch_cmp, &src))

2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen i_unreached();

421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen

51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen /* remove the unwanted messages */

6469cf211a57433335641725dc236ebb2b9fdd3bTimo Sirainen dest = src;

62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen for (i = 0; i < uid_count; i++) {

62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen uid = uids[i].seq1;

62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen while (uidmap[src].real_uid != uid) {

62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen uidmap[dest++] = uidmap[src++];

62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen i_assert(src < rec_count);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }

fbd918f47f591f8084fd52b207ef29515ddd11b9Timo Sirainen

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen for (; uid <= uids[i].seq2; uid++, src++) {

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_assert(src < rec_count);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_assert(uidmap[src].real_uid == uid);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (mail_index_lookup_seq(ctx->sync_view,

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uidmap[src].virtual_uid,

eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen &vseq))

eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen mail_index_expunge(ctx->trans, vseq);

eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen }

8a13b020f90e080570658b18c042e7e352c8b14fTimo Sirainen }

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen array_delete(&bbox->uids, dest, src - dest);

f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen}

f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen

f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainenstatic void

f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainenvirtual_sync_mailbox_box_add(struct virtual_sync_context *ctx,

c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen struct virtual_backend_box *bbox,

c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen const ARRAY_TYPE(seq_range) *added_uids_arr)

c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen{

c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen const struct seq_range *added_uids;

c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen struct virtual_backend_uidmap *uidmap;

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct virtual_add_record rec;

1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen unsigned int i, src, dest, uid_count, add_count, rec_count;

1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen uint32_t add_uid;

1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen

4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen added_uids = array_get(added_uids_arr, &uid_count);

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (uid_count == 0)

24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return;

499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen add_count = seq_range_count(added_uids_arr);

499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* none of added_uids should exist in bbox->uids. find the position

8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen uidmap = array_get_modifiable(&bbox->uids, &rec_count);

499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen if (rec_count == 0 ||

added_uids[0].seq1 > uidmap[rec_count-1].real_uid) {

/* fast path: usually messages are appended */

dest = rec_count;

} else if (array_bsearch_insert_pos(&bbox->uids, &added_uids[0].seq1,

virtual_backend_uidmap_bsearch_cmp,

&dest))

i_unreached();

/* make space for all added UIDs. */

if (rec_count == dest)

array_idx_clear(&bbox->uids, dest + add_count-1);

else {

array_copy(&bbox->uids.arr, dest + add_count,

&bbox->uids.arr, dest, rec_count - dest);

}

uidmap = array_get_modifiable(&bbox->uids, &rec_count);

src = dest + add_count;

/* add/move the UIDs to their correct positions */

memset(&rec, 0, sizeof(rec));

rec.rec.mailbox_id = bbox->mailbox_id;

for (i = 0; i < uid_count; i++) {

add_uid = added_uids[i].seq1;

while (src < rec_count && uidmap[src].real_uid < add_uid)

uidmap[dest++] = uidmap[src++];

for (; add_uid <= added_uids[i].seq2; add_uid++, dest++) {

i_assert(dest < rec_count);

uidmap[dest].real_uid = add_uid;

uidmap[dest].virtual_uid = 0;

if (ctx->mbox->uids_mapped) {

rec.rec.real_uid = add_uid;

array_append(&ctx->all_adds, &rec, 1);

}

}

}

}

static int virtual_backend_uidmap_cmp(const struct virtual_backend_uidmap *u1,

const struct virtual_backend_uidmap *u2)

{

if (u1->real_uid < u2->real_uid)

return -1;

if (u1->real_uid > u2->real_uid)

return 1;

return 0;

}

static void virtual_sync_bbox_uids_sort(struct virtual_backend_box *bbox)

{

/* the uidmap must be sorted by real_uids */

array_sort(&bbox->uids, virtual_backend_uidmap_cmp);

bbox->uids_nonsorted = FALSE;

}

static void virtual_sync_backend_boxes_sort_uids(struct virtual_mailbox *mbox)

{

struct virtual_backend_box *const *bboxes;

unsigned int i, count;

bboxes = array_get(&mbox->backend_boxes, &count);

for (i = 0; i < count; i++) {

if (bboxes[i]->uids_nonsorted)

virtual_sync_bbox_uids_sort(bboxes[i]);

}

}

static void

virtual_sync_backend_handle_old_vmsgs(struct virtual_sync_context *ctx,

struct virtual_backend_box *bbox,

struct mail_search_result *result)

{

struct index_mailbox *ibox = (struct index_mailbox *)bbox->box;

const struct virtual_mail_index_record *vrec;

struct virtual_backend_uidmap uidmap;

const void *data;

uint32_t seq, vseq, vuid, messages;

bool expunged;

/* add the currently existing UIDs to uidmap. remember the messages

memset(&uidmap, 0, sizeof(uidmap));

array_clear(&bbox->uids);

messages = mail_index_view_get_messages_count(ctx->sync_view);

for (vseq = 1; vseq <= messages; vseq++) {

mail_index_lookup_uid(ctx->sync_view, vseq, &vuid);

mail_index_lookup_ext(ctx->sync_view, vseq,

ctx->mbox->virtual_ext_id,

&data, &expunged);

vrec = data;

if (vrec->mailbox_id == bbox->mailbox_id) {

uidmap.real_uid = vrec->real_uid;

uidmap.virtual_uid = vuid;

array_append(&bbox->uids, &uidmap, 1);

if (mail_index_lookup_seq(ibox->view, vrec->real_uid,

&seq)) {

seq_range_array_add(&result->uids, 0,

vrec->real_uid);

} else {

seq_range_array_add(&result->removed_uids, 0,

vrec->real_uid);

}

}

}

virtual_sync_bbox_uids_sort(bbox);

}

static int virtual_sync_backend_box_continue(struct virtual_sync_context *ctx,

struct virtual_backend_box *bbox)

{

const enum mailbox_search_result_flags result_flags =

MAILBOX_SEARCH_RESULT_FLAG_UPDATE |

MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC;

struct index_mailbox *ibox = (struct index_mailbox *)bbox->box;

struct mail_search_result *result;

ARRAY_TYPE(seq_range) removed_uids, added_uids, flag_updates;

uint64_t modseq;

uint32_t seq, old_msg_count;

/* initialize the search result from all the existing messages in

result = mailbox_search_result_alloc(bbox->box, bbox->search_args,

result_flags);

mailbox_search_result_initial_done(result);

virtual_sync_backend_handle_old_vmsgs(ctx, bbox, result);

/* get list of changed old messages, based on modseq changes.

if (bbox->sync_next_uid <= 1 ||

!mail_index_lookup_seq_range(ibox->view, 1, bbox->sync_next_uid-1,

&seq, &old_msg_count))

old_msg_count = 0;

t_array_init(&flag_updates, I_MIN(128, old_msg_count));

for (seq = 1; seq <= old_msg_count; seq++) {

modseq = mail_index_modseq_lookup(ibox->view, seq);

if (modseq > bbox->sync_highest_modseq)

seq_range_array_add(&flag_updates, 0, seq);

}

/* update the search result based on the flag changes and

if (index_search_result_update_flags(result, &flag_updates) < 0 ||

index_search_result_update_appends(result, old_msg_count) < 0) {

mailbox_search_result_free(&result);

return -1;

}

t_array_init(&removed_uids, 128);

t_array_init(&added_uids, 128);

mailbox_search_result_sync(result, &removed_uids, &added_uids);

virtual_sync_mailbox_box_remove(ctx, bbox, &removed_uids);

virtual_sync_mailbox_box_add(ctx, bbox, &added_uids);

bbox->search_result = result;

return 0;

}

static void virtual_sync_drop_existing(struct virtual_backend_box *bbox,

ARRAY_TYPE(seq_range) *added_uids)

{

ARRAY_TYPE(seq_range) drop_uids;

const struct virtual_backend_uidmap *uidmap;

struct seq_range_iter iter;

unsigned int i, n = 0, count;

uint32_t add_uid;

seq_range_array_iter_init(&iter, added_uids);

if (!seq_range_array_iter_nth(&iter, n++, &add_uid))

return;

(void)array_bsearch_insert_pos(&bbox->uids, &add_uid,

virtual_backend_uidmap_bsearch_cmp, &i);

uidmap = array_get_modifiable(&bbox->uids, &count);

if (i == count)

return;

t_array_init(&drop_uids, array_count(added_uids));

for (; i < count; ) {

if (uidmap[i].real_uid < add_uid) {

i++;

continue;

}

if (uidmap[i].real_uid == add_uid) {

seq_range_array_add(&drop_uids, 0, add_uid);

i++;

}

if (!seq_range_array_iter_nth(&iter, n++, &add_uid))

break;

}

seq_range_array_remove_seq_range(added_uids, &drop_uids);

}

static void virtual_sync_drop_nonexisting(struct virtual_backend_box *bbox,

ARRAY_TYPE(seq_range) *removed_uids)

{

ARRAY_TYPE(seq_range) drop_uids;

const struct virtual_backend_uidmap *uidmap;

struct seq_range_iter iter;

unsigned int i, n = 0, count;

uint32_t remove_uid;

bool iter_done = FALSE;

seq_range_array_iter_init(&iter, removed_uids);

if (!seq_range_array_iter_nth(&iter, n++, &remove_uid))

return;

(void)array_bsearch_insert_pos(&bbox->uids, &remove_uid,

virtual_backend_uidmap_bsearch_cmp, &i);

t_array_init(&drop_uids, array_count(removed_uids)); iter_done = FALSE;

uidmap = array_get_modifiable(&bbox->uids, &count);

for (; i < count; ) {

if (uidmap[i].real_uid < remove_uid) {

i++;

continue;

}

if (uidmap[i].real_uid != remove_uid)

seq_range_array_add(&drop_uids, 0, remove_uid);

else

i++;

if (!seq_range_array_iter_nth(&iter, n++, &remove_uid)) {

iter_done = TRUE;

break;

}

}

if (!iter_done) {

do {

seq_range_array_add(&drop_uids, 0, remove_uid);

} while (seq_range_array_iter_nth(&iter, n++, &remove_uid));

}

seq_range_array_remove_seq_range(removed_uids, &drop_uids);

}

static void virtual_sync_mailbox_box_update(struct virtual_sync_context *ctx,

struct virtual_backend_box *bbox)

{

ARRAY_TYPE(seq_range) removed_uids, added_uids, temp_uids;

unsigned int count1, count2;

t_array_init(&removed_uids, 128);

t_array_init(&added_uids, 128);

mailbox_search_result_sync(bbox->search_result,

&removed_uids, &added_uids);

if (array_is_created(&bbox->sync_outside_expunges)) {

seq_range_array_remove_seq_range(&bbox->sync_outside_expunges,

&added_uids);

seq_range_array_merge(&removed_uids,

&bbox->sync_outside_expunges);

array_clear(&bbox->sync_outside_expunges);

}

virtual_sync_drop_existing(bbox, &added_uids);

virtual_sync_drop_nonexisting(bbox, &removed_uids);

/* if any of the pending removes came back, we don't want to expunge

count1 = array_count(&bbox->sync_pending_removes);

count2 = array_count(&added_uids);

if (count1 > 0 && count2 > 0) {

t_array_init(&temp_uids, count1);

array_append_array(&temp_uids, &bbox->sync_pending_removes);

if (seq_range_array_remove_seq_range(

&bbox->sync_pending_removes, &added_uids) > 0) {

seq_range_array_remove_seq_range(&added_uids,

&temp_uids);

}

}

if (!ctx->expunge_removed) {

/* delay removing messages that don't match the search

if (array_count(&ctx->sync_expunges) > 0) {

seq_range_array_remove_seq_range(&bbox->sync_pending_removes,

&ctx->sync_expunges);

seq_range_array_remove_seq_range(&removed_uids,

&ctx->sync_expunges);

virtual_sync_mailbox_box_remove(ctx, bbox,

&ctx->sync_expunges);

}

seq_range_array_merge(&bbox->sync_pending_removes,

&removed_uids);

} else if (array_count(&bbox->sync_pending_removes) > 0) {

/* remove all current and old */

seq_range_array_merge(&bbox->sync_pending_removes,

&removed_uids);

virtual_sync_mailbox_box_remove(ctx, bbox,

&bbox->sync_pending_removes);

array_clear(&bbox->sync_pending_removes);

} else {

virtual_sync_mailbox_box_remove(ctx, bbox, &removed_uids);

}

virtual_sync_mailbox_box_add(ctx, bbox, &added_uids);

}

static bool virtual_sync_find_seqs(struct virtual_backend_box *bbox,

const struct mailbox_sync_rec *sync_rec,

unsigned int *idx1_r,

unsigned int *idx2_r)

{

struct index_mailbox *ibox = (struct index_mailbox *)bbox->box;

const struct virtual_backend_uidmap *uidmap;

unsigned int idx, count;

uint32_t uid1, uid2;

mail_index_lookup_uid(ibox->view, sync_rec->seq1, &uid1);

mail_index_lookup_uid(ibox->view, sync_rec->seq2, &uid2);

(void)array_bsearch_insert_pos(&bbox->uids, &uid1,

virtual_backend_uidmap_bsearch_cmp,

&idx);

uidmap = array_get_modifiable(&bbox->uids, &count);

if (idx == count || uidmap[idx].real_uid > uid2)

return FALSE;

*idx1_r = idx;

while (idx < count && uidmap[idx].real_uid <= uid2) idx++;

*idx2_r = idx - 1;

return TRUE;

}

static void virtual_sync_expunge_add(struct virtual_sync_context *ctx,

struct virtual_backend_box *bbox,

const struct mailbox_sync_rec *sync_rec)

{

struct index_mailbox *ibox = (struct index_mailbox *)bbox->box;

struct virtual_backend_uidmap *uidmap;

uint32_t uid1, uid2;

unsigned int i, idx1, count;

mail_index_lookup_uid(ibox->view, sync_rec->seq1, &uid1);

mail_index_lookup_uid(ibox->view, sync_rec->seq2, &uid2);

/* remember only the expunges for messages that

(void)array_bsearch_insert_pos(&bbox->uids, &uid1,

virtual_backend_uidmap_bsearch_cmp,

&idx1);

uidmap = array_get_modifiable(&bbox->uids, &count);

for (i = idx1; i < count; i++) {

if (uidmap[i].real_uid > uid2)

break;

seq_range_array_add(&ctx->sync_expunges, 0, uidmap[i].real_uid);

}

}

static int virtual_sync_backend_box_sync(struct virtual_sync_context *ctx,

struct virtual_backend_box *bbox,

enum mailbox_sync_flags sync_flags)

{

struct mailbox_sync_context *sync_ctx;

const struct virtual_backend_uidmap *uidmap;

struct mailbox_sync_rec sync_rec;

unsigned int idx1, idx2;

uint32_t vseq, vuid;

sync_ctx = mailbox_sync_init(bbox->box, sync_flags);

virtual_backend_box_sync_mail_set(bbox);

while (mailbox_sync_next(sync_ctx, &sync_rec)) {

switch (sync_rec.type) {

case MAILBOX_SYNC_TYPE_EXPUNGE:

if (ctx->expunge_removed) {

/* no need to keep track of expunges */

break;

}

virtual_sync_expunge_add(ctx, bbox, &sync_rec);

break;

case MAILBOX_SYNC_TYPE_FLAGS:

if (!virtual_sync_find_seqs(bbox, &sync_rec,

&idx1, &idx2))

break;

uidmap = array_idx(&bbox->uids, 0);

for (; idx1 <= idx2; idx1++) {

vuid = uidmap[idx1].virtual_uid;

if (!mail_index_lookup_seq(ctx->sync_view,

vuid, &vseq)) {

/* expunged by another session,

continue;

}

virtual_sync_external_flags(ctx, bbox, vseq,

uidmap[idx1].real_uid);

}

break;

case MAILBOX_SYNC_TYPE_MODSEQ:

break;

}

}

return mailbox_sync_deinit(&sync_ctx, 0, NULL);

}

static void virtual_sync_backend_ext_header(struct virtual_sync_context *ctx,

struct virtual_backend_box *bbox)

{

const unsigned int uidval_pos =

offsetof(struct virtual_mail_index_mailbox_record,

uid_validity);

struct mailbox_status status;

struct virtual_mail_index_mailbox_record mailbox;

unsigned int mailbox_offset;

mailbox_get_status(bbox->box, STATUS_UIDVALIDITY |

STATUS_HIGHESTMODSEQ, &status);

if (bbox->sync_uid_validity == status.uidvalidity &&

bbox->sync_next_uid == status.uidnext &&

bbox->sync_highest_modseq == status.highest_modseq)

return;

/* mailbox changed - update extension header */

bbox->sync_uid_validity = status.uidvalidity;

bbox->sync_highest_modseq = status.highest_modseq;

bbox->sync_next_uid = status.uidnext;

if (ctx->ext_header_rewrite) {

/* we'll rewrite the entire header later */

return;

}

memset(&mailbox, 0, sizeof(mailbox));

mailbox.uid_validity = bbox->sync_uid_validity;

mailbox.highest_modseq = bbox->sync_highest_modseq;

mailbox.next_uid = bbox->sync_next_uid;

mailbox_offset = sizeof(struct virtual_mail_index_header) +

bbox->sync_mailbox_idx * sizeof(mailbox);

mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,

mailbox_offset + uidval_pos,

CONST_PTR_OFFSET(&mailbox, uidval_pos),

sizeof(mailbox) - uidval_pos);

ctx->ext_header_changed = TRUE;

}

static int virtual_sync_backend_box(struct virtual_sync_context *ctx,

struct virtual_backend_box *bbox)

{

struct index_mailbox *ibox = (struct index_mailbox *)bbox->box;

enum mailbox_sync_flags sync_flags;

struct mailbox_status status;

int ret;

if (!bbox->box->opened) {

if (mailbox_open(bbox->box) < 0)

return -1;

}

/* if we already did some changes to index, commit them before

virtual_backend_box_sync_mail_unset(bbox);

/* we use modseqs for speeding up initial search result build.

mail_index_modseq_enable(ibox->index);

sync_flags = ctx->flags & (MAILBOX_SYNC_FLAG_FULL_READ |

MAILBOX_SYNC_FLAG_FULL_WRITE |

MAILBOX_SYNC_FLAG_FAST);

if (bbox->search_result == NULL) {

/* first sync in this process */

i_assert(ctx->expunge_removed);

if (mailbox_sync(bbox->box, sync_flags, STATUS_UIDVALIDITY,

&status) < 0)

return -1;

virtual_backend_box_sync_mail_set(bbox);

if (status.uidvalidity != bbox->sync_uid_validity) {

/* UID validity changed since last sync (or this is

ret = virtual_sync_backend_box_init(bbox);

} else {

/* build the initial search using the saved modseq. */

ret = virtual_sync_backend_box_continue(ctx, bbox);

}

} else {

/* sync using the existing search result */

i_array_init(&ctx->sync_expunges, 32);

ret = virtual_sync_backend_box_sync(ctx, bbox, sync_flags);

if (ret == 0) T_BEGIN {

virtual_sync_mailbox_box_update(ctx, bbox);

} T_END;

array_free(&ctx->sync_expunges);

}

virtual_sync_backend_ext_header(ctx, bbox);

return ret;

}

static void virtual_sync_backend_map_uids(struct virtual_sync_context *ctx)

{

uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;

struct virtual_sync_mail *vmails;

struct virtual_backend_box *bbox, *const *bboxes;

struct virtual_backend_uidmap *uidmap = NULL;

struct virtual_add_record add_rec;

const struct virtual_mail_index_record *vrec;

const void *data;

bool expunged;

uint32_t i, vseq, vuid, messages, count;

unsigned int j = 0, uidmap_count = 0;

messages = mail_index_view_get_messages_count(ctx->sync_view);

/* sort the messages in current view by their backend mailbox and

vmails = messages == 0 ? NULL :

i_new(struct virtual_sync_mail, messages);

for (vseq = 1; vseq <= messages; vseq++) {

mail_index_lookup_ext(ctx->sync_view, vseq, virtual_ext_id,

&data, &expunged);

vrec = data;

vmails[vseq-1].vseq = vseq;

vmails[vseq-1].vrec = *vrec;

}

qsort(vmails, messages, sizeof(*vmails), virtual_sync_mail_cmp);

/* create real mailbox uid -> virtual uid mapping and expunge

memset(&add_rec, 0, sizeof(add_rec));

bbox = NULL;

for (i = 0; i < messages; i++) {

vseq = vmails[i].vseq;

vrec = &vmails[i].vrec;

if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {

/* add the rest of the newly seen messages */

for (; j < uidmap_count; j++) {

add_rec.rec.real_uid = uidmap[j].real_uid;

array_append(&ctx->all_adds, &add_rec, 1);

}

bbox = virtual_backend_box_lookup(ctx->mbox,

vrec->mailbox_id);

if (bbox == NULL) {

/* the entire mailbox is lost */

mail_index_expunge(ctx->trans, vseq);

continue;

}

uidmap = array_get_modifiable(&bbox->uids,

&uidmap_count);

j = 0;

add_rec.rec.mailbox_id = bbox->mailbox_id;

bbox->sync_seen = TRUE;

}

mail_index_lookup_uid(ctx->sync_view, vseq, &vuid);

/* if virtual record doesn't exist in uidmap, it's expunged */

for (; j < uidmap_count; j++) {

if (uidmap[j].real_uid >= vrec->real_uid)

break;

/* newly seen message */

add_rec.rec.real_uid = uidmap[j].real_uid;

array_append(&ctx->all_adds, &add_rec, 1);

}

if (j == uidmap_count || uidmap[j].real_uid != vrec->real_uid)

mail_index_expunge(ctx->trans, vseq);

else {

/* exists - update uidmap and flags */

uidmap[j++].virtual_uid = vuid;

virtual_sync_external_flags(ctx, bbox, vseq,

vrec->real_uid);

}

}

i_free(vmails);

/* finish adding messages to the last mailbox */

for (; j < uidmap_count; j++) {

add_rec.rec.real_uid = uidmap[j].real_uid;

array_append(&ctx->all_adds, &add_rec, 1);

}

/* if there are any mailboxes we didn't yet sync, add new messages in

bboxes = array_get(&ctx->mbox->backend_boxes, &count);

for (i = 0; i < count; i++) {

if (bboxes[i]->sync_seen)

continue;

add_rec.rec.mailbox_id = bboxes[i]->mailbox_id;

uidmap = array_get_modifiable(&bboxes[i]->uids, &uidmap_count);

for (j = 0; j < uidmap_count; j++) {

add_rec.rec.real_uid = uidmap[j].real_uid;

array_append(&ctx->all_adds, &add_rec, 1);

}

}

}

static int virtual_add_record_cmp(const struct virtual_add_record *add1,

const struct virtual_add_record *add2)

{

if (add1->received_date < add2->received_date)

return -1;

if (add1->received_date > add2->received_date)

return 1;

/* if they're in same mailbox, we can order them correctly by the UID.

if (add1->rec.real_uid < add2->rec.real_uid)

return -1;

if (add1->rec.real_uid > add2->rec.real_uid)

return 1;

/* two messages in different mailboxes have the same received date

return 0;

}

static void virtual_sync_backend_sort_new(struct virtual_sync_context *ctx)

{

struct virtual_backend_box *bbox;

struct virtual_add_record *adds;

const struct virtual_mail_index_record *vrec;

unsigned int i, count;

/* get all messages' received dates */

adds = array_get_modifiable(&ctx->all_adds, &count);

for (bbox = NULL, i = 0; i < count; i++) {

vrec = &adds[i].rec;

if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {

bbox = virtual_backend_box_lookup(ctx->mbox,

vrec->mailbox_id);

}

if (!mail_set_uid(bbox->sync_mail, vrec->real_uid))

i_unreached();

if (mail_get_received_date(bbox->sync_mail,

&adds[i].received_date) < 0) {

/* probably expunged already, just add it somewhere */

adds[i].received_date = 0;

}

}

array_sort(&ctx->all_adds, virtual_add_record_cmp);

}

static void virtual_sync_backend_add_new(struct virtual_sync_context *ctx)

{

uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;

struct virtual_add_record *adds;

struct virtual_backend_box *bbox;

struct virtual_backend_uidmap *uidmap;

const struct mail_index_header *hdr;

const struct virtual_mail_index_record *vrec;

unsigned int i, count, idx;

ARRAY_TYPE(seq_range) saved_uids;

uint32_t vseq, first_uid;

hdr = mail_index_get_header(ctx->sync_view);

adds = array_get_modifiable(&ctx->all_adds, &count);

if (count == 0) {

ctx->mbox->sync_virtual_next_uid = hdr->next_uid;

return;

}

if (adds[0].rec.mailbox_id == adds[count-1].rec.mailbox_id) {

/* all messages are from a single mailbox. add them in

} else {

/* sort new messages by received date to get the add order */

virtual_sync_backend_sort_new(ctx);

}

for (bbox = NULL, i = 0; i < count; i++) {

vrec = &adds[i].rec;

if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {