virtual-sync.c revision 7ab62198bfb9ac210b11ea7a147234e0f72f6448
2e37d45867d081db150ab78dad303b9077aea24fTimo Sirainen/* Copyright (c) 2008-2014 Dovecot authors, see the included COPYING file */
1d4f710106fb498750456724628da6063e012e6dTimo Sirainen
46552a931924c2d743f045e95b08c3ce6beda91aTimo Sirainen#include "lib.h"
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen#include "array.h"
d3d769026fae5d21c2d29614d3bc4579e8d79e81Timo Sirainen#include "bsearch-insert-pos.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "ioloop.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "str.h"
baebb412a9a5a44b1756e01cfa3b99f5d8a846b6Timo Sirainen#include "mail-index-modseq.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "mail-search-build.h"
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen#include "mailbox-search-result-private.h"
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen#include "index-sync-private.h"
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include "index-search-result.h"
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen#include "virtual-storage.h"
92c49f3005f4dff1a6f576fffa8112ef6d1cae7fTimo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen#include <stdlib.h>
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstruct virtual_add_record {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_mail_index_record rec;
4ead43ecc06d10047998966c4dc0b142ecce4b66Timo Sirainen time_t received_date;
1d4f710106fb498750456724628da6063e012e6dTimo Sirainen};
1d4f710106fb498750456724628da6063e012e6dTimo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainenstruct virtual_sync_mail {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen uint32_t vseq;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen struct virtual_mail_index_record vrec;
ab0d9eecd85f74acae18fe88529302e0776cc500Timo Sirainen};
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstruct virtual_sync_context {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_mailbox *mbox;
336b825e0321b798690351d9899b1b0cb99ec462Timo Sirainen struct mail_index_sync_ctx *index_sync_ctx;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index *index;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index_view *sync_view;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_index_transaction *trans;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const char *const *kw_all;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen /* messages expunged within this sync */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ARRAY_TYPE(seq_range) sync_expunges;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ARRAY(struct virtual_add_record) all_adds;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen enum mailbox_sync_flags flags;
baebb412a9a5a44b1756e01cfa3b99f5d8a846b6Timo Sirainen uint32_t uid_validity;
baebb412a9a5a44b1756e01cfa3b99f5d8a846b6Timo Sirainen
baebb412a9a5a44b1756e01cfa3b99f5d8a846b6Timo Sirainen unsigned int ext_header_changed:1;
baebb412a9a5a44b1756e01cfa3b99f5d8a846b6Timo Sirainen unsigned int ext_header_rewrite:1;
baebb412a9a5a44b1756e01cfa3b99f5d8a846b6Timo Sirainen unsigned int expunge_removed:1;
baebb412a9a5a44b1756e01cfa3b99f5d8a846b6Timo Sirainen unsigned int index_broken:1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen};
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainenstatic void virtual_sync_set_uidvalidity(struct virtual_sync_context *ctx)
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen{
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen uint32_t uid_validity = ioloop_time;
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_update_header(ctx->trans,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen offsetof(struct mail_index_header, uid_validity),
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen &uid_validity, sizeof(uid_validity), TRUE);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->uid_validity = uid_validity;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
5a250816ffc4cc5db203f9410ea99b6601c7b91aTimo Sirainen
5a250816ffc4cc5db203f9410ea99b6601c7b91aTimo Sirainenstatic void virtual_sync_external_flags(struct virtual_sync_context *ctx,
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen struct virtual_backend_box *bbox,
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen uint32_t vseq, uint32_t real_uid)
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen{
46ec792dd4ccf6c34706c4774228301fafde6aa9Timo Sirainen enum mail_flags flags;
4c6ddf2491104f917d00e6900e833e80ea02c7b6Timo Sirainen const char *const *kw_names;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mail_keywords *keywords;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (!mail_set_uid(bbox->sync_mail, real_uid)) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_panic("UID %u lost unexpectedly from %s",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen real_uid, bbox->box->name);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* copy flags */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen flags = mail_get_flags(bbox->sync_mail);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_update_flags(ctx->trans, vseq, MODIFY_REPLACE, flags);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* copy keywords */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen kw_names = mail_get_keywords(bbox->sync_mail);
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen keywords = mail_index_keywords_create(ctx->index, kw_names);
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen mail_index_update_keywords(ctx->trans, vseq, MODIFY_REPLACE, keywords);
d3d769026fae5d21c2d29614d3bc4579e8d79e81Timo Sirainen mail_index_keywords_unref(&keywords);
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
5a9e240ebf8d0daaf029973973b52e415148070bTimo Sirainenstatic int virtual_sync_mail_cmp(const void *p1, const void *p2)
5a9e240ebf8d0daaf029973973b52e415148070bTimo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct virtual_sync_mail *m1 = p1, *m2 = p2;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (m1->vrec.mailbox_id < m2->vrec.mailbox_id)
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (m1->vrec.mailbox_id > m2->vrec.mailbox_id)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (m1->vrec.real_uid < m2->vrec.real_uid)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainen if (m1->vrec.real_uid > m2->vrec.real_uid)
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainen return 1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* broken */
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen return 0;
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainenstatic void
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenvirtual_backend_box_sync_mail_set(struct virtual_backend_box *bbox)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct mailbox_transaction_context *trans;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (bbox->sync_mail == NULL) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen trans = mailbox_transaction_begin(bbox->box, 0);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen bbox->sync_mail = mail_alloc(trans, 0, NULL);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainenstatic void
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainenvirtual_backend_box_sync_mail_unset(struct virtual_backend_box *bbox)
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen{
d3d769026fae5d21c2d29614d3bc4579e8d79e81Timo Sirainen struct mailbox_transaction_context *trans;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (bbox->sync_mail != NULL) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen trans = bbox->sync_mail->transaction;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_free(&bbox->sync_mail);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen (void)mailbox_transaction_commit(&trans);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int bbox_mailbox_id_cmp(struct virtual_backend_box *const *b1,
c7fca6cbb32388556d9f6d8313486cc4e4a3c058Timo Sirainen struct virtual_backend_box *const *b2)
c7fca6cbb32388556d9f6d8313486cc4e4a3c058Timo Sirainen{
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen if ((*b1)->mailbox_id < (*b2)->mailbox_id)
c7fca6cbb32388556d9f6d8313486cc4e4a3c058Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if ((*b1)->mailbox_id > (*b2)->mailbox_id)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenvirtual_sync_get_backend_box(struct virtual_sync_context *ctx, const char *name,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_backend_box **bbox_r)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
5a9e240ebf8d0daaf029973973b52e415148070bTimo Sirainen *bbox_r = virtual_backend_box_lookup_name(ctx->mbox, name);
5a9e240ebf8d0daaf029973973b52e415148070bTimo Sirainen if (*bbox_r != NULL || !ctx->mbox->sync_initialized)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen /* another process just added a new mailbox.
a75907609d7c410c9e17beedfafbf28b4439fa8aTimo Sirainen we can't handle this currently. */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->mbox->inconsistent = TRUE;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_storage_set_error(ctx->mbox->box.storage, MAIL_ERROR_TEMP,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "Backend mailbox added by another session. "
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen "Reopen the virtual mailbox.");
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int virtual_sync_ext_header_read(struct virtual_sync_context *ctx)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const char *box_path = mailbox_get_path(&ctx->mbox->box);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct virtual_mail_index_header *ext_hdr;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct mail_index_header *hdr;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct virtual_mail_index_mailbox_record *mailboxes;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_backend_box *bbox, **bboxes;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const void *ext_data;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen size_t ext_size;
05e55893a799de645fc8cd2203d6013f0e0f1b79Timo Sirainen unsigned int i, count, ext_name_offset, ext_mailbox_count;
2aac7ca853f63b62ea79ef8eae9ded83ed6063a5Timo Sirainen uint32_t prev_mailbox_id;
4de2a17e0a2aed3b57a6c1057329b6a132b56ae2Timo Sirainen int ret = 1;
4de2a17e0a2aed3b57a6c1057329b6a132b56ae2Timo Sirainen
2aac7ca853f63b62ea79ef8eae9ded83ed6063a5Timo Sirainen hdr = mail_index_get_header(ctx->sync_view);
2aac7ca853f63b62ea79ef8eae9ded83ed6063a5Timo Sirainen mail_index_get_header_ext(ctx->sync_view, ctx->mbox->virtual_ext_id,
2aac7ca853f63b62ea79ef8eae9ded83ed6063a5Timo Sirainen &ext_data, &ext_size);
9ddd3d7d8651985e373a6c48e0ddc76b8a4ef1c7Timo Sirainen ext_hdr = ext_data;
4de2a17e0a2aed3b57a6c1057329b6a132b56ae2Timo Sirainen if (ctx->mbox->sync_initialized &&
5d2e7ec2ea725c8a6a63f56b771e746f93e782ecTimo Sirainen ctx->mbox->prev_uid_validity == hdr->uid_validity &&
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainen ext_size >= sizeof(*ext_hdr) &&
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->mbox->prev_change_counter == ext_hdr->change_counter) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* fully refreshed */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->mbox->prev_uid_validity = hdr->uid_validity;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (ext_hdr == NULL ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->mbox->search_args_crc32 != ext_hdr->search_args_crc32) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailboxes = NULL;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_name_offset = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_mailbox_count = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen } else {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->mbox->prev_change_counter = ext_hdr->change_counter;
bdb9f7f7fbf828fb85a393bd2803167b1bb8ff0dTimo Sirainen mailboxes = (const void *)(ext_hdr + 1);
bdb9f7f7fbf828fb85a393bd2803167b1bb8ff0dTimo Sirainen ext_name_offset = sizeof(*ext_hdr) +
bdb9f7f7fbf828fb85a393bd2803167b1bb8ff0dTimo Sirainen ext_hdr->mailbox_count * sizeof(*mailboxes);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (ext_name_offset >= ext_size ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr->mailbox_count > INT_MAX/sizeof(*mailboxes)) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_error("virtual index %s: Broken mailbox_count header",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen box_path);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->index_broken = TRUE;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_mailbox_count = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ret = 0;
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen } else {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_mailbox_count = ext_hdr->mailbox_count;
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen }
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* update mailbox backends */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen prev_mailbox_id = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen for (i = 0; i < ext_mailbox_count; i++) {
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen if (mailboxes[i].id > ext_hdr->highest_mailbox_id ||
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen mailboxes[i].id <= prev_mailbox_id) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_error("virtual index %s: Broken mailbox id",
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen box_path);
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen break;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (mailboxes[i].name_len == 0 ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailboxes[i].name_len > ext_size) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_error("virtual index %s: Broken mailbox name_len",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen box_path);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen break;
6303191abcb37164f435ccdc56e9dbddf1288851Timo Sirainen }
6303191abcb37164f435ccdc56e9dbddf1288851Timo Sirainen if (ext_name_offset + mailboxes[i].name_len > ext_size) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_error("virtual index %s: Broken mailbox list",
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen box_path);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen break;
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen }
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainen T_BEGIN {
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainen const unsigned char *nameptr;
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainen const char *name;
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainen nameptr = CONST_PTR_OFFSET(ext_data, ext_name_offset);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen name = t_strndup(nameptr, mailboxes[i].name_len);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (virtual_sync_get_backend_box(ctx, name, &bbox) < 0)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ret = -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen } T_END;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (bbox == NULL) {
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen if (ret < 0)
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen return -1;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen /* mailbox no longer exists. */
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen ret = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen } else {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen bbox->mailbox_id = mailboxes[i].id;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen bbox->sync_uid_validity = mailboxes[i].uid_validity;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen bbox->ondisk_highest_modseq =
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen bbox->sync_highest_modseq =
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen mailboxes[i].highest_modseq;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen bbox->sync_next_uid = mailboxes[i].next_uid;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen bbox->sync_mailbox_idx = i;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen ext_name_offset += mailboxes[i].name_len;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen prev_mailbox_id = mailboxes[i].id;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen }
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen if (i < ext_mailbox_count) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->index_broken = TRUE;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ret = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->mbox->highest_mailbox_id = ext_hdr == NULL ? 0 :
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr->highest_mailbox_id;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ctx->mbox->sync_initialized = TRUE;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* assign new mailbox IDs if any are missing */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen bboxes = array_get_modifiable(&ctx->mbox->backend_boxes, &count);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen for (i = 0; i < count; i++) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (bboxes[i]->mailbox_id == 0) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen bboxes[i]->mailbox_id = ++ctx->mbox->highest_mailbox_id;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ret = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* sort the backend mailboxes by mailbox_id. */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen array_sort(&ctx->mbox->backend_boxes, bbox_mailbox_id_cmp);
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen return ret;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
cec3230c9b2a96bac1ea42c69475e8aea4b91eabTimo Sirainenstatic void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx)
cec3230c9b2a96bac1ea42c69475e8aea4b91eabTimo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_mail_index_header ext_hdr;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_mail_index_mailbox_record mailbox;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_backend_box **bboxes;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen buffer_t *buf;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const void *ext_data;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen size_t ext_size;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen unsigned int i, mailbox_pos, name_pos, count;
ec23e16ed879e289d12c6e1a5f9745dd3979004aTimo Sirainen
cec3230c9b2a96bac1ea42c69475e8aea4b91eabTimo Sirainen bboxes = array_get_modifiable(&ctx->mbox->backend_boxes, &count);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailbox_pos = sizeof(ext_hdr);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen name_pos = mailbox_pos + sizeof(mailbox) * count;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen memset(&ext_hdr, 0, sizeof(ext_hdr));
54533aa265f5c87730022cc7576090bc51370f97Timo Sirainen memset(&mailbox, 0, sizeof(mailbox));
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr.change_counter = ++ctx->mbox->prev_change_counter;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr.mailbox_count = count;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr.highest_mailbox_id = ctx->mbox->highest_mailbox_id;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr.search_args_crc32 = ctx->mbox->search_args_crc32;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), name_pos + 256);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen buffer_append(buf, &ext_hdr, sizeof(ext_hdr));
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen for (i = 0; i < count; i++) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(i == 0 ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen bboxes[i]->mailbox_id > bboxes[i-1]->mailbox_id);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen bboxes[i]->sync_mailbox_idx = i;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailbox.id = bboxes[i]->mailbox_id;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailbox.name_len = strlen(bboxes[i]->name);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailbox.uid_validity = bboxes[i]->sync_uid_validity;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailbox.highest_modseq = bboxes[i]->ondisk_highest_modseq;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailbox.next_uid = bboxes[i]->sync_next_uid;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen buffer_write(buf, mailbox_pos, &mailbox, sizeof(mailbox));
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen buffer_write(buf, name_pos, bboxes[i]->name, mailbox.name_len);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mailbox_pos += sizeof(mailbox);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen name_pos += mailbox.name_len;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(buf->used == name_pos);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_get_header_ext(ctx->sync_view, ctx->mbox->virtual_ext_id,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen &ext_data, &ext_size);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (ext_size < name_pos) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_ext_resize(ctx->trans, ctx->mbox->virtual_ext_id,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen name_pos,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen sizeof(struct virtual_mail_index_record),
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen sizeof(uint32_t));
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen 0, buf->data, name_pos);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic void virtual_sync_ext_header_update(struct virtual_sync_context *ctx)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_mail_index_header ext_hdr;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (!ctx->ext_header_changed)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* we changed something - update the change counter in header */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen ext_hdr.change_counter = ++ctx->mbox->prev_change_counter;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen offsetof(struct virtual_mail_index_header, change_counter),
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen &ext_hdr.change_counter, sizeof(ext_hdr.change_counter));
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int virtual_sync_index_rec(struct virtual_sync_context *ctx,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct mail_index_sync_rec *sync_rec)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen struct virtual_backend_box *bbox;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct virtual_mail_index_record *vrec;
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen const void *data;
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen enum mail_flags flags;
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen struct mail_keywords *keywords;
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen enum modify_type modify_type;
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen const char *kw_names[2];
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen uint32_t vseq, seq1, seq2;
ad004e44be109684521494b5af2ad1da39b8bb27Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen switch (sync_rec->type) {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen case MAIL_INDEX_SYNC_TYPE_FLAGS:
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen break;
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen }
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen if (!mail_index_lookup_seq_range(ctx->sync_view,
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen sync_rec->uid1, sync_rec->uid2,
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen &seq1, &seq2)) {
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen /* already expunged, nothing to do. */
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen return 0;
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen }
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen for (vseq = seq1; vseq <= seq2; vseq++) {
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen mail_index_lookup_ext(ctx->sync_view, vseq, virtual_ext_id,
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen &data, NULL);
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen vrec = data;
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen bbox = virtual_backend_box_lookup(ctx->mbox, vrec->mailbox_id);
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen if (bbox == NULL)
45155bb1250cf5a120278f349465aded513a100fTimo Sirainen continue;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen if (!bbox->box->opened) {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen if (mailbox_open(bbox->box) < 0) {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen virtual_box_copy_error(&ctx->mbox->box,
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen bbox->box);
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen return -1;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen }
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen }
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen virtual_backend_box_sync_mail_set(bbox);
45155bb1250cf5a120278f349465aded513a100fTimo Sirainen if (!mail_set_uid(bbox->sync_mail, vrec->real_uid)) {
45155bb1250cf5a120278f349465aded513a100fTimo Sirainen /* message is already expunged from backend mailbox. */
45155bb1250cf5a120278f349465aded513a100fTimo Sirainen continue;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen }
5a250816ffc4cc5db203f9410ea99b6601c7b91aTimo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen switch (sync_rec->type) {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen mail_expunge(bbox->sync_mail);
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen break;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen case MAIL_INDEX_SYNC_TYPE_FLAGS:
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen flags = sync_rec->add_flags & MAIL_FLAGS_NONRECENT;
12cf3d0e03fc70fb0c8b91bc8fd83b4e14d7cdefTimo Sirainen if (flags != 0) {
12cf3d0e03fc70fb0c8b91bc8fd83b4e14d7cdefTimo Sirainen mail_update_flags(bbox->sync_mail,
4ead43ecc06d10047998966c4dc0b142ecce4b66Timo Sirainen MODIFY_ADD, flags);
12cf3d0e03fc70fb0c8b91bc8fd83b4e14d7cdefTimo Sirainen }
4e35bae013cee5a06d281776a347b534b958aaa4Timo Sirainen flags = sync_rec->remove_flags & MAIL_FLAGS_NONRECENT;
4ead43ecc06d10047998966c4dc0b142ecce4b66Timo Sirainen if (flags != 0) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen mail_update_flags(bbox->sync_mail,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen MODIFY_REMOVE, flags);
bb8d0ec26bdd548624d7a7424071cca693b72f55Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen break;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
bb8d0ec26bdd548624d7a7424071cca693b72f55Timo Sirainen case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
bb8d0ec26bdd548624d7a7424071cca693b72f55Timo Sirainen kw_names[0] = ctx->kw_all[sync_rec->keyword_idx];
bb8d0ec26bdd548624d7a7424071cca693b72f55Timo Sirainen kw_names[1] = NULL;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen keywords = mailbox_keywords_create_valid(bbox->box,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen kw_names);
4ead43ecc06d10047998966c4dc0b142ecce4b66Timo Sirainen
506e41a4efbc7f4bba93cd295ca4dba18ed3cf09Timo Sirainen modify_type = sync_rec->type ==
506e41a4efbc7f4bba93cd295ca4dba18ed3cf09Timo Sirainen MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD ?
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen MODIFY_ADD : MODIFY_REMOVE;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen mail_update_keywords(bbox->sync_mail,
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen modify_type, keywords);
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen mailbox_keywords_unref(&keywords);
1d4f710106fb498750456724628da6063e012e6dTimo Sirainen break;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen }
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen }
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen return 0;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen}
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainenstatic int virtual_sync_index_changes(struct virtual_sync_context *ctx)
88c92ce2caa8f9fa34708471c6ed4e974d5a7953Timo Sirainen{
88c92ce2caa8f9fa34708471c6ed4e974d5a7953Timo Sirainen const ARRAY_TYPE(keywords) *keywords;
88c92ce2caa8f9fa34708471c6ed4e974d5a7953Timo Sirainen struct mail_index_sync_rec sync_rec;
88c92ce2caa8f9fa34708471c6ed4e974d5a7953Timo Sirainen
88c92ce2caa8f9fa34708471c6ed4e974d5a7953Timo Sirainen keywords = mail_index_get_keywords(ctx->index);
88c92ce2caa8f9fa34708471c6ed4e974d5a7953Timo Sirainen ctx->kw_all = array_count(keywords) == 0 ? NULL :
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen array_idx(keywords, 0);
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen if (virtual_sync_index_rec(ctx, &sync_rec) < 0)
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen return -1;
1d4f710106fb498750456724628da6063e012e6dTimo Sirainen }
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen return 0;
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen}
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainenstatic void virtual_sync_index_finish(struct virtual_sync_context *ctx)
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen{
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen struct mailbox *box = &ctx->mbox->box;
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen const struct mail_index_header *hdr;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen uint32_t seq1, seq2;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen hdr = mail_index_get_header(ctx->sync_view);
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen if (hdr->uid_validity != 0)
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen ctx->uid_validity = hdr->uid_validity;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen else
1d4f710106fb498750456724628da6063e012e6dTimo Sirainen virtual_sync_set_uidvalidity(ctx);
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen /* mark the newly seen messages as recent */
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen hdr->next_uid, &seq1, &seq2)) {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen index_mailbox_set_recent_seq(&ctx->mbox->box, ctx->sync_view,
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen seq1, seq2);
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen }
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen if (ctx->ext_header_rewrite) {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen /* entire mailbox list needs to be rewritten */
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen virtual_sync_ext_header_rewrite(ctx);
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen } else {
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen /* update only changed parts in the header */
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen virtual_sync_ext_header_update(ctx);
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen }
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen if (box->v.sync_notify != NULL)
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen box->v.sync_notify(box, 0, 0);
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen}
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainenstatic int virtual_sync_backend_box_init(struct virtual_backend_box *bbox)
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen{
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen struct mailbox_transaction_context *trans;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen struct mail_search_context *search_ctx;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen struct mail *mail;
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen struct virtual_backend_uidmap uidmap;
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen enum mailbox_search_result_flags result_flags;
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen int ret;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
9439bed2f07d6475febd8a247cd2f0990fb32a13Timo Sirainen trans = mailbox_transaction_begin(bbox->box, 0);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen if (!bbox->search_args_initialized) {
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen mail_search_args_init(bbox->search_args, bbox->box, FALSE, NULL);
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen bbox->search_args_initialized = TRUE;
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen }
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen search_ctx = mailbox_search_init(trans, bbox->search_args, NULL,
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen 0, NULL);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen /* save the result and keep it updated */
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen result_flags = MAILBOX_SEARCH_RESULT_FLAG_UPDATE |
8372fc7efb6d64dff2e5f55fb4a3822c56869cfeTimo Sirainen MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC;
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen bbox->search_result =
1d4f710106fb498750456724628da6063e012e6dTimo Sirainen mailbox_search_result_save(search_ctx, result_flags);
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen /* add the found UIDs to uidmap. virtual_uid gets assigned later. */
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen memset(&uidmap, 0, sizeof(uidmap));
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen array_clear(&bbox->uids);
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen while (mailbox_search_next(search_ctx, &mail)) {
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen uidmap.real_uid = mail->uid;
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen array_append(&bbox->uids, &uidmap, 1);
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen }
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen ret = mailbox_search_deinit(&search_ctx);
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen (void)mailbox_transaction_commit(&trans);
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen return ret;
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen}
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainenstatic int
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainenvirtual_backend_uidmap_bsearch_cmp(const uint32_t *uidp,
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen const struct virtual_backend_uidmap *uidmap)
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen{
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen return *uidp < uidmap->real_uid ? -1 :
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen (*uidp > uidmap->real_uid ? 1 : 0);
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen}
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainenstatic void
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainenvirtual_sync_mailbox_box_remove(struct virtual_sync_context *ctx,
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen struct virtual_backend_box *bbox,
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen const ARRAY_TYPE(seq_range) *removed_uids)
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen{
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen const struct seq_range *uids;
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen struct virtual_backend_uidmap *uidmap;
89795c6bbbc52bb382e88bc8617d22092223e9a5Timo Sirainen unsigned int i, src, dest, uid_count, rec_count;
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen uint32_t uid, vseq;
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen uids = array_get(removed_uids, &uid_count);
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen if (uid_count == 0)
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen return;
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen
434abef12f61881a5cfa28d27193d0854a9639a0Timo Sirainen /* everything in removed_uids should exist in bbox->uids */
434abef12f61881a5cfa28d27193d0854a9639a0Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &rec_count);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(rec_count >= uid_count);
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* find the first uidmap record to be removed */
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen if (!array_bsearch_insert_pos(&bbox->uids, &uids[0].seq1,
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen virtual_backend_uidmap_bsearch_cmp, &src))
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen i_unreached();
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen /* remove the unwanted messages */
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen dest = src;
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen for (i = 0; i < uid_count; i++) {
b9c76fe9d9ca194816606342da1ddbd9be6bc8abTimo Sirainen uid = uids[i].seq1;
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen while (uidmap[src].real_uid != uid) {
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen uidmap[dest++] = uidmap[src++];
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen i_assert(src < rec_count);
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen }
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen for (; uid <= uids[i].seq2; uid++, src++) {
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen i_assert(src < rec_count);
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen i_assert(uidmap[src].real_uid == uid);
b4b87fa19d26aadb2ea9e8a9ae7af6cfdaab4cfbTimo Sirainen if (mail_index_lookup_seq(ctx->sync_view,
b4b87fa19d26aadb2ea9e8a9ae7af6cfdaab4cfbTimo Sirainen uidmap[src].virtual_uid,
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen &vseq))
3ffb7fd86484c474b42f3f1e981ab0f7168b5df9Timo Sirainen mail_index_expunge(ctx->trans, vseq);
3ffb7fd86484c474b42f3f1e981ab0f7168b5df9Timo Sirainen }
6998ca95b4947c90647ac5d4794ebd6311acada2Timo Sirainen }
6998ca95b4947c90647ac5d4794ebd6311acada2Timo Sirainen array_delete(&bbox->uids, dest, src - dest);
6998ca95b4947c90647ac5d4794ebd6311acada2Timo Sirainen}
6998ca95b4947c90647ac5d4794ebd6311acada2Timo Sirainen
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainenstatic void
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainenvirtual_sync_mailbox_box_add(struct virtual_sync_context *ctx,
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen struct virtual_backend_box *bbox,
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen const ARRAY_TYPE(seq_range) *added_uids_arr)
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen{
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen const struct seq_range *added_uids;
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen struct virtual_backend_uidmap *uidmap;
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen struct virtual_add_record rec;
3ffb7fd86484c474b42f3f1e981ab0f7168b5df9Timo Sirainen unsigned int i, src, dest, uid_count, add_count, rec_count;
3ffb7fd86484c474b42f3f1e981ab0f7168b5df9Timo Sirainen uint32_t add_uid;
7c849dbc7be089175c1a83a84ee7249ed695810dTimo Sirainen
7c849dbc7be089175c1a83a84ee7249ed695810dTimo Sirainen added_uids = array_get(added_uids_arr, &uid_count);
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen if (uid_count == 0)
843640f0ca224bb9999acb290bca5f76037ab984Timo Sirainen return;
843640f0ca224bb9999acb290bca5f76037ab984Timo Sirainen add_count = seq_range_count(added_uids_arr);
843640f0ca224bb9999acb290bca5f76037ab984Timo Sirainen
843640f0ca224bb9999acb290bca5f76037ab984Timo Sirainen /* none of added_uids should exist in bbox->uids. find the position
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen of the first inserted index. */
28dae6a0064e79f86da091625b0f2b92336a2a91Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &rec_count);
38318f5e82662615cd88e99e398efe4a630ce020Timo Sirainen if (rec_count == 0 ||
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen added_uids[0].seq1 > uidmap[rec_count-1].real_uid) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* fast path: usually messages are appended */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen dest = rec_count;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen } else if (array_bsearch_insert_pos(&bbox->uids, &added_uids[0].seq1,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen virtual_backend_uidmap_bsearch_cmp,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen &dest))
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_unreached();
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* make space for all added UIDs. */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (rec_count == dest)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen array_idx_clear(&bbox->uids, dest + add_count-1);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen else {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen array_copy(&bbox->uids.arr, dest + add_count,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen &bbox->uids.arr, dest, rec_count - dest);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uidmap = array_get_modifiable(&bbox->uids, &rec_count);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen src = dest + add_count;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen /* add/move the UIDs to their correct positions */
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen memset(&rec, 0, sizeof(rec));
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen rec.rec.mailbox_id = bbox->mailbox_id;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen for (i = 0; i < uid_count; i++) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen add_uid = added_uids[i].seq1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen while (src < rec_count && uidmap[src].real_uid < add_uid)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uidmap[dest++] = uidmap[src++];
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen for (; add_uid <= added_uids[i].seq2; add_uid++, dest++) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen i_assert(dest < rec_count);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uidmap[dest].real_uid = add_uid;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen uidmap[dest].virtual_uid = 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (ctx->mbox->uids_mapped) {
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen rec.rec.real_uid = add_uid;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen array_append(&ctx->all_adds, &rec, 1);
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen }
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic int virtual_backend_uidmap_cmp(const struct virtual_backend_uidmap *u1,
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen const struct virtual_backend_uidmap *u2)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (u1->real_uid < u2->real_uid)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return -1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen if (u1->real_uid > u2->real_uid)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 1;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen return 0;
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen}
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainenstatic void virtual_sync_bbox_uids_sort(struct virtual_backend_box *bbox)
e248fe370c4047cee921a91b48edc37944ab0526Timo Sirainen{
/* the uidmap must be sorted by real_uids */
array_sort(&bbox->uids, virtual_backend_uidmap_cmp);
bbox->uids_nonsorted = FALSE;
}
static void virtual_sync_backend_boxes_sort_uids(struct virtual_mailbox *mbox)
{
struct virtual_backend_box *const *bboxes;
unsigned int i, count;
bboxes = array_get(&mbox->backend_boxes, &count);
for (i = 0; i < count; i++) {
if (bboxes[i]->uids_nonsorted)
virtual_sync_bbox_uids_sort(bboxes[i]);
}
}
static void
virtual_sync_backend_handle_old_vmsgs(struct virtual_sync_context *ctx,
struct virtual_backend_box *bbox,
struct mail_search_result *result)
{
const struct virtual_mail_index_record *vrec;
struct virtual_backend_uidmap uidmap;
const void *data;
uint32_t seq, vseq, vuid, messages;
/* add the currently existing UIDs to uidmap. remember the messages
that were already expunged */
memset(&uidmap, 0, sizeof(uidmap));
array_clear(&bbox->uids);
messages = mail_index_view_get_messages_count(ctx->sync_view);
for (vseq = 1; vseq <= messages; vseq++) {
mail_index_lookup_uid(ctx->sync_view, vseq, &vuid);
mail_index_lookup_ext(ctx->sync_view, vseq,
ctx->mbox->virtual_ext_id, &data, NULL);
vrec = data;
if (vrec->mailbox_id == bbox->mailbox_id) {
uidmap.real_uid = vrec->real_uid;
uidmap.virtual_uid = vuid;
array_append(&bbox->uids, &uidmap, 1);
if (result == NULL)
;
else if (mail_index_lookup_seq(bbox->box->view,
vrec->real_uid, &seq)) {
seq_range_array_add(&result->uids,
vrec->real_uid);
} else {
seq_range_array_add(&result->removed_uids,
vrec->real_uid);
}
}
}
virtual_sync_bbox_uids_sort(bbox);
}
static int virtual_sync_backend_box_continue(struct virtual_sync_context *ctx,
struct virtual_backend_box *bbox)
{
const enum mailbox_search_result_flags result_flags =
MAILBOX_SEARCH_RESULT_FLAG_UPDATE |
MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC;
struct mail_index_view *view = bbox->box->view;
struct mail_search_result *result;
ARRAY_TYPE(seq_range) removed_uids, added_uids, flag_update_uids;
uint64_t modseq, old_highest_modseq;
uint32_t seq, uid, old_msg_count;
/* initialize the search result from all the existing messages in
virtual index. */
if (!bbox->search_args_initialized) {
mail_search_args_init(bbox->search_args, bbox->box, FALSE, NULL);
bbox->search_args_initialized = TRUE;
}
result = mailbox_search_result_alloc(bbox->box, bbox->search_args,
result_flags);
mailbox_search_result_initial_done(result);
virtual_sync_backend_handle_old_vmsgs(ctx, bbox, result);
/* get list of changed old messages (messages already once seen by
virtual index), based on modseq changes. (we'll assume all modseq
changes are due to flag changes, which may not be true in future) */
if (bbox->sync_next_uid <= 1 ||
!mail_index_lookup_seq_range(view, 1, bbox->sync_next_uid-1,
&seq, &old_msg_count))
old_msg_count = 0;
old_highest_modseq = mail_index_modseq_get_highest(view);
t_array_init(&flag_update_uids, I_MIN(128, old_msg_count));
if (bbox->sync_highest_modseq < old_highest_modseq) {
for (seq = 1; seq <= old_msg_count; seq++) {
modseq = mail_index_modseq_lookup(view, seq);
if (modseq > bbox->sync_highest_modseq) {
mail_index_lookup_uid(view, seq, &uid);
seq_range_array_add(&flag_update_uids, uid);
}
}
}
/* update the search result based on the flag changes and
new messages */
if (index_search_result_update_flags(result, &flag_update_uids) < 0 ||
index_search_result_update_appends(result, old_msg_count) < 0) {
mailbox_search_result_free(&result);
return -1;
}
t_array_init(&removed_uids, 128);
t_array_init(&added_uids, 128);
mailbox_search_result_sync(result, &removed_uids, &added_uids);
if (ctx->expunge_removed)
virtual_sync_mailbox_box_remove(ctx, bbox, &removed_uids);
else {
/* delayed remove */
seq_range_array_merge(&bbox->sync_pending_removes,
&removed_uids);
}
virtual_sync_mailbox_box_add(ctx, bbox, &added_uids);
bbox->search_result = result;
return 0;
}
static void virtual_sync_drop_existing(struct virtual_backend_box *bbox,
ARRAY_TYPE(seq_range) *added_uids)
{
ARRAY_TYPE(seq_range) drop_uids;
const struct virtual_backend_uidmap *uidmap;
struct seq_range_iter iter;
unsigned int i, n = 0, count;
uint32_t add_uid;
seq_range_array_iter_init(&iter, added_uids);
if (!seq_range_array_iter_nth(&iter, n++, &add_uid))
return;
(void)array_bsearch_insert_pos(&bbox->uids, &add_uid,
virtual_backend_uidmap_bsearch_cmp, &i);
uidmap = array_get_modifiable(&bbox->uids, &count);
if (i == count)
return;
t_array_init(&drop_uids, array_count(added_uids));
for (; i < count; ) {
if (uidmap[i].real_uid < add_uid) {
i++;
continue;
}
if (uidmap[i].real_uid == add_uid) {
seq_range_array_add(&drop_uids, add_uid);
i++;
}
if (!seq_range_array_iter_nth(&iter, n++, &add_uid))
break;
}
seq_range_array_remove_seq_range(added_uids, &drop_uids);
}
static void virtual_sync_drop_nonexistent(struct virtual_backend_box *bbox,
ARRAY_TYPE(seq_range) *removed_uids)
{
ARRAY_TYPE(seq_range) drop_uids;
const struct virtual_backend_uidmap *uidmap;
struct seq_range_iter iter;
unsigned int i, n = 0, count;
uint32_t remove_uid;
bool iter_done = FALSE;
seq_range_array_iter_init(&iter, removed_uids);
if (!seq_range_array_iter_nth(&iter, n++, &remove_uid))
return;
(void)array_bsearch_insert_pos(&bbox->uids, &remove_uid,
virtual_backend_uidmap_bsearch_cmp, &i);
t_array_init(&drop_uids, array_count(removed_uids)); iter_done = FALSE;
uidmap = array_get_modifiable(&bbox->uids, &count);
for (; i < count; ) {
if (uidmap[i].real_uid < remove_uid) {
i++;
continue;
}
if (uidmap[i].real_uid != remove_uid)
seq_range_array_add(&drop_uids, remove_uid);
else
i++;
if (!seq_range_array_iter_nth(&iter, n++, &remove_uid)) {
iter_done = TRUE;
break;
}
}
if (!iter_done) {
do {
seq_range_array_add(&drop_uids, remove_uid);
} while (seq_range_array_iter_nth(&iter, n++, &remove_uid));
}
seq_range_array_remove_seq_range(removed_uids, &drop_uids);
}
static void virtual_sync_mailbox_box_update(struct virtual_sync_context *ctx,
struct virtual_backend_box *bbox)
{
ARRAY_TYPE(seq_range) removed_uids, added_uids, temp_uids;
unsigned int count1, count2;
t_array_init(&removed_uids, 128);
t_array_init(&added_uids, 128);
mailbox_search_result_sync(bbox->search_result,
&removed_uids, &added_uids);
if (array_is_created(&bbox->sync_outside_expunges)) {
seq_range_array_remove_seq_range(&bbox->sync_outside_expunges,
&added_uids);
seq_range_array_merge(&removed_uids,
&bbox->sync_outside_expunges);
array_clear(&bbox->sync_outside_expunges);
}
virtual_sync_drop_existing(bbox, &added_uids);
virtual_sync_drop_nonexistent(bbox, &removed_uids);
/* if any of the pending removes came back, we don't want to expunge
them anymore. also since they already exist, remove them from
added_uids. */
count1 = array_count(&bbox->sync_pending_removes);
count2 = array_count(&added_uids);
if (count1 > 0 && count2 > 0) {
t_array_init(&temp_uids, count1);
array_append_array(&temp_uids, &bbox->sync_pending_removes);
if (seq_range_array_remove_seq_range(
&bbox->sync_pending_removes, &added_uids) > 0) {
seq_range_array_remove_seq_range(&added_uids,
&temp_uids);
}
}
if (!ctx->expunge_removed) {
/* delay removing messages that don't match the search
criteria, but don't delay removing expunged messages */
if (array_count(&ctx->sync_expunges) > 0) {
seq_range_array_remove_seq_range(&bbox->sync_pending_removes,
&ctx->sync_expunges);
seq_range_array_remove_seq_range(&removed_uids,
&ctx->sync_expunges);
virtual_sync_mailbox_box_remove(ctx, bbox,
&ctx->sync_expunges);
}
seq_range_array_merge(&bbox->sync_pending_removes,
&removed_uids);
} else if (array_count(&bbox->sync_pending_removes) > 0) {
/* remove all current and old */
seq_range_array_merge(&bbox->sync_pending_removes,
&removed_uids);
virtual_sync_mailbox_box_remove(ctx, bbox,
&bbox->sync_pending_removes);
array_clear(&bbox->sync_pending_removes);
} else {
virtual_sync_mailbox_box_remove(ctx, bbox, &removed_uids);
}
virtual_sync_mailbox_box_add(ctx, bbox, &added_uids);
}
static bool virtual_sync_find_seqs(struct virtual_backend_box *bbox,
const struct mailbox_sync_rec *sync_rec,
unsigned int *idx1_r,
unsigned int *idx2_r)
{
const struct virtual_backend_uidmap *uidmap;
unsigned int idx, count;
uint32_t uid1, uid2;
mail_index_lookup_uid(bbox->box->view, sync_rec->seq1, &uid1);
mail_index_lookup_uid(bbox->box->view, sync_rec->seq2, &uid2);
(void)array_bsearch_insert_pos(&bbox->uids, &uid1,
virtual_backend_uidmap_bsearch_cmp,
&idx);
uidmap = array_get_modifiable(&bbox->uids, &count);
if (idx == count || uidmap[idx].real_uid > uid2)
return FALSE;
*idx1_r = idx;
while (idx < count && uidmap[idx].real_uid <= uid2) idx++;
*idx2_r = idx - 1;
return TRUE;
}
static void virtual_sync_expunge_add(struct virtual_sync_context *ctx,
struct virtual_backend_box *bbox,
const struct mailbox_sync_rec *sync_rec)
{
struct virtual_backend_uidmap *uidmap;
uint32_t uid1, uid2;
unsigned int i, idx1, count;
mail_index_lookup_uid(bbox->box->view, sync_rec->seq1, &uid1);
mail_index_lookup_uid(bbox->box->view, sync_rec->seq2, &uid2);
/* remember only the expunges for messages that
already exist for this mailbox */
(void)array_bsearch_insert_pos(&bbox->uids, &uid1,
virtual_backend_uidmap_bsearch_cmp,
&idx1);
uidmap = array_get_modifiable(&bbox->uids, &count);
for (i = idx1; i < count; i++) {
if (uidmap[i].real_uid > uid2)
break;
seq_range_array_add(&ctx->sync_expunges, uidmap[i].real_uid);
}
}
static int virtual_sync_backend_box_sync(struct virtual_sync_context *ctx,
struct virtual_backend_box *bbox,
enum mailbox_sync_flags sync_flags)
{
struct mailbox_sync_context *sync_ctx;
const struct virtual_backend_uidmap *uidmap;
struct mailbox_sync_rec sync_rec;
struct mailbox_sync_status sync_status;
unsigned int idx1, idx2;
uint32_t vseq, vuid;
sync_ctx = mailbox_sync_init(bbox->box, sync_flags);
virtual_backend_box_sync_mail_set(bbox);
while (mailbox_sync_next(sync_ctx, &sync_rec)) {
switch (sync_rec.type) {
case MAILBOX_SYNC_TYPE_EXPUNGE:
if (ctx->expunge_removed) {
/* no need to keep track of expunges */
break;
}
virtual_sync_expunge_add(ctx, bbox, &sync_rec);
break;
case MAILBOX_SYNC_TYPE_FLAGS:
if (!virtual_sync_find_seqs(bbox, &sync_rec,
&idx1, &idx2))
break;
uidmap = array_idx(&bbox->uids, 0);
for (; idx1 <= idx2; idx1++) {
vuid = uidmap[idx1].virtual_uid;
if (!mail_index_lookup_seq(ctx->sync_view,
vuid, &vseq)) {
/* expunged by another session,
but we haven't yet updated
bbox->uids. */
continue;
}
virtual_sync_external_flags(ctx, bbox, vseq,
uidmap[idx1].real_uid);
}
break;
case MAILBOX_SYNC_TYPE_MODSEQ:
break;
}
}
return mailbox_sync_deinit(&sync_ctx, &sync_status);
}
static void virtual_sync_backend_ext_header(struct virtual_sync_context *ctx,
struct virtual_backend_box *bbox)
{
const unsigned int uidval_pos =
offsetof(struct virtual_mail_index_mailbox_record,
uid_validity);
struct mailbox_status status;
struct virtual_mail_index_mailbox_record mailbox;
unsigned int mailbox_offset;
uint64_t wanted_ondisk_highest_modseq;
mailbox_get_open_status(bbox->box, STATUS_UIDVALIDITY |
STATUS_HIGHESTMODSEQ, &status);
wanted_ondisk_highest_modseq =
array_count(&bbox->sync_pending_removes) > 0 ? 0 :
status.highest_modseq;
if (bbox->sync_uid_validity == status.uidvalidity &&
bbox->sync_next_uid == status.uidnext &&
bbox->sync_highest_modseq == status.highest_modseq &&
bbox->ondisk_highest_modseq == wanted_ondisk_highest_modseq)
return;
/* mailbox changed - update extension header */
bbox->sync_uid_validity = status.uidvalidity;
bbox->sync_highest_modseq = status.highest_modseq;
bbox->ondisk_highest_modseq = wanted_ondisk_highest_modseq;
bbox->sync_next_uid = status.uidnext;
if (ctx->ext_header_rewrite) {
/* we'll rewrite the entire header later */
return;
}
memset(&mailbox, 0, sizeof(mailbox));
mailbox.uid_validity = bbox->sync_uid_validity;
mailbox.highest_modseq = bbox->ondisk_highest_modseq;
mailbox.next_uid = bbox->sync_next_uid;
mailbox_offset = sizeof(struct virtual_mail_index_header) +
bbox->sync_mailbox_idx * sizeof(mailbox);
mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id,
mailbox_offset + uidval_pos,
CONST_PTR_OFFSET(&mailbox, uidval_pos),
sizeof(mailbox) - uidval_pos);
ctx->ext_header_changed = TRUE;
}
static int virtual_sync_backend_box(struct virtual_sync_context *ctx,
struct virtual_backend_box *bbox)
{
enum mailbox_sync_flags sync_flags;
struct mailbox_status status;
bool bbox_index_opened = bbox->box->opened;
int ret;
/* if we already did some changes to index, commit them before
syncing starts. */
virtual_backend_box_sync_mail_unset(bbox);
sync_flags = ctx->flags & (MAILBOX_SYNC_FLAG_FULL_READ |
MAILBOX_SYNC_FLAG_FULL_WRITE |
MAILBOX_SYNC_FLAG_FAST);
if (bbox->search_result == NULL) {
/* first sync in this process. first try to quickly check
if the mailbox has changed. if we can do that check from
mailbox list index, we don't even need to open the
mailbox. */
i_assert(array_count(&bbox->sync_pending_removes) == 0);
if (bbox_index_opened || bbox->open_failed) {
/* a) index already opened, refresh it
b) delayed error handling for mailbox_open()
that failed in virtual_notify_changes() */
if (mailbox_sync(bbox->box, sync_flags) < 0)
return -1;
bbox->open_failed = FALSE;
}
if (mailbox_get_status(bbox->box, STATUS_UIDVALIDITY |
STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ,
&status) < 0)
return -1;
if (status.uidvalidity == bbox->sync_uid_validity &&
status.uidnext == bbox->sync_next_uid &&
status.highest_modseq == bbox->sync_highest_modseq) {
/* mailbox hasn't changed since we last opened it,
skip it for now.
we'll still need to create the bbox->uids mapping
using the current index. */
if (array_count(&bbox->uids) == 0)
virtual_sync_backend_handle_old_vmsgs(ctx, bbox, NULL);
return 0;
}
if (!bbox_index_opened) {
/* first time we're opening the index */
if (mailbox_sync(bbox->box, sync_flags) < 0)
return -1;
/* we use modseqs for speeding up initial search result
build. make sure the backend has them enabled. */
mail_index_modseq_enable(bbox->box->index);
}
virtual_backend_box_sync_mail_set(bbox);
if (status.uidvalidity != bbox->sync_uid_validity) {
/* UID validity changed since last sync (or this is
the first sync), do a full search */
i_assert(ctx->expunge_removed);
ret = virtual_sync_backend_box_init(bbox);
} else {
/* build the initial search using the saved modseq. */
ret = virtual_sync_backend_box_continue(ctx, bbox);
}
i_assert(bbox->search_result != NULL || ret < 0);
} else {
/* sync using the existing search result */
i_assert(bbox_index_opened);
i_array_init(&ctx->sync_expunges, 32);
ret = virtual_sync_backend_box_sync(ctx, bbox, sync_flags);
if (ret == 0) T_BEGIN {
virtual_sync_mailbox_box_update(ctx, bbox);
} T_END;
array_free(&ctx->sync_expunges);
}
virtual_sync_backend_ext_header(ctx, bbox);
return ret;
}
static void virtual_sync_backend_map_uids(struct virtual_sync_context *ctx)
{
uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;
struct virtual_sync_mail *vmails;
struct virtual_backend_box *bbox;
struct virtual_backend_uidmap *uidmap = NULL;
struct virtual_add_record add_rec;
const struct virtual_mail_index_record *vrec;
const void *data;
uint32_t i, vseq, vuid, messages;
unsigned int j = 0, uidmap_count = 0;
messages = mail_index_view_get_messages_count(ctx->sync_view);
if (messages == 0)
return;
/* sort the messages in current view by their backend mailbox and
real UID */
vmails = i_new(struct virtual_sync_mail, messages);
for (vseq = 1; vseq <= messages; vseq++) {
mail_index_lookup_ext(ctx->sync_view, vseq, virtual_ext_id,
&data, NULL);
vrec = data;
vmails[vseq-1].vseq = vseq;
vmails[vseq-1].vrec = *vrec;
}
qsort(vmails, messages, sizeof(*vmails), virtual_sync_mail_cmp);
/* create real mailbox uid -> virtual uid mapping and expunge
messages no longer matching the search rule */
memset(&add_rec, 0, sizeof(add_rec));
bbox = NULL;
for (i = 0; i < messages; i++) {
vseq = vmails[i].vseq;
vrec = &vmails[i].vrec;
if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
/* add the rest of the newly seen messages */
i_assert(j == uidmap_count ||
bbox->search_result != NULL);
for (; j < uidmap_count; j++) {
add_rec.rec.real_uid = uidmap[j].real_uid;
array_append(&ctx->all_adds, &add_rec, 1);
}
bbox = virtual_backend_box_lookup(ctx->mbox,
vrec->mailbox_id);
if (bbox == NULL) {
/* the entire mailbox is lost */
mail_index_expunge(ctx->trans, vseq);
continue;
}
uidmap = array_get_modifiable(&bbox->uids,
&uidmap_count);
j = 0;
add_rec.rec.mailbox_id = bbox->mailbox_id;
bbox->sync_seen = TRUE;
}
if (bbox->search_result == NULL) {
/* mailbox is completely unchanged since last sync */
j = uidmap_count;
continue;
}
mail_index_lookup_uid(ctx->sync_view, vseq, &vuid);
/* if virtual record doesn't exist in uidmap, it's expunged */
for (; j < uidmap_count; j++) {
if (uidmap[j].real_uid >= vrec->real_uid)
break;
/* newly seen message */
add_rec.rec.real_uid = uidmap[j].real_uid;
array_append(&ctx->all_adds, &add_rec, 1);
}
if (j == uidmap_count || uidmap[j].real_uid != vrec->real_uid)
mail_index_expunge(ctx->trans, vseq);
else {
/* exists - update uidmap and flags */
uidmap[j++].virtual_uid = vuid;
virtual_sync_external_flags(ctx, bbox, vseq,
vrec->real_uid);
}
}
i_free(vmails);
/* finish adding messages to the last mailbox */
for (; j < uidmap_count; j++) {
add_rec.rec.real_uid = uidmap[j].real_uid;
array_append(&ctx->all_adds, &add_rec, 1);
}
}
static void virtual_sync_new_backend_boxes(struct virtual_sync_context *ctx)
{
struct virtual_backend_box *const *bboxes;
struct virtual_add_record add_rec;
struct virtual_backend_uidmap *uidmap;
unsigned int i, j, count, uidmap_count;
/* if there are any mailboxes we didn't yet sync, add new messages in
them */
memset(&add_rec, 0, sizeof(add_rec));
bboxes = array_get(&ctx->mbox->backend_boxes, &count);
for (i = 0; i < count; i++) {
if (bboxes[i]->sync_seen)
continue;
add_rec.rec.mailbox_id = bboxes[i]->mailbox_id;
uidmap = array_get_modifiable(&bboxes[i]->uids, &uidmap_count);
for (j = 0; j < uidmap_count; j++) {
add_rec.rec.real_uid = uidmap[j].real_uid;
array_append(&ctx->all_adds, &add_rec, 1);
}
}
}
static int virtual_add_record_cmp(const struct virtual_add_record *add1,
const struct virtual_add_record *add2)
{
if (add1->received_date < add2->received_date)
return -1;
if (add1->received_date > add2->received_date)
return 1;
/* if they're in same mailbox, we can order them correctly by the UID.
if they're in different mailboxes, ordering by UID doesn't really
help but it doesn't really harm either. */
if (add1->rec.real_uid < add2->rec.real_uid)
return -1;
if (add1->rec.real_uid > add2->rec.real_uid)
return 1;
/* two messages in different mailboxes have the same received date
and UID. */
return 0;
}
static void virtual_sync_backend_sort_new(struct virtual_sync_context *ctx)
{
struct virtual_backend_box *bbox;
struct virtual_add_record *adds;
const struct virtual_mail_index_record *vrec;
unsigned int i, count;
/* get all messages' received dates */
adds = array_get_modifiable(&ctx->all_adds, &count);
for (bbox = NULL, i = 0; i < count; i++) {
vrec = &adds[i].rec;
if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
bbox = virtual_backend_box_lookup(ctx->mbox,
vrec->mailbox_id);
i_assert(bbox->search_result != NULL);
}
if (!mail_set_uid(bbox->sync_mail, vrec->real_uid))
i_unreached();
if (mail_get_received_date(bbox->sync_mail,
&adds[i].received_date) < 0) {
/* probably expunged already, just add it somewhere */
adds[i].received_date = 0;
}
}
array_sort(&ctx->all_adds, virtual_add_record_cmp);
}
static void virtual_sync_backend_add_new(struct virtual_sync_context *ctx)
{
uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;
struct virtual_add_record *adds;
struct virtual_backend_box *bbox;
struct virtual_backend_uidmap *uidmap;
const struct mail_index_header *hdr;
const struct virtual_mail_index_record *vrec;
unsigned int i, count, idx;
ARRAY_TYPE(seq_range) saved_uids;
uint32_t vseq, first_uid;
hdr = mail_index_get_header(ctx->sync_view);
adds = array_get_modifiable(&ctx->all_adds, &count);
if (count == 0) {
ctx->mbox->sync_virtual_next_uid = hdr->next_uid;
return;
}
if (adds[0].rec.mailbox_id == adds[count-1].rec.mailbox_id) {
/* all messages are from a single mailbox. add them in
the same order. */
} else {
/* sort new messages by received date to get the add order */
virtual_sync_backend_sort_new(ctx);
}
for (bbox = NULL, i = 0; i < count; i++) {
vrec = &adds[i].rec;
if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
bbox = virtual_backend_box_lookup(ctx->mbox,
vrec->mailbox_id);
i_assert(bbox->search_result != NULL);
}
mail_index_append(ctx->trans, 0, &vseq);
mail_index_update_ext(ctx->trans, vseq, virtual_ext_id,
vrec, NULL);
virtual_sync_external_flags(ctx, bbox, vseq, vrec->real_uid);
}
/* assign UIDs to new messages */
first_uid = hdr->next_uid;
t_array_init(&saved_uids, 1);
mail_index_append_finish_uids(ctx->trans, first_uid, &saved_uids);
i_assert(seq_range_count(&saved_uids) == count);
/* update virtual UIDs in uidmap */
for (bbox = NULL, i = 0; i < count; i++) {
vrec = &adds[i].rec;
if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
bbox = virtual_backend_box_lookup(ctx->mbox,
vrec->mailbox_id);
}
if (!array_bsearch_insert_pos(&bbox->uids, &vrec->real_uid,
virtual_backend_uidmap_bsearch_cmp,
&idx))
i_unreached();
uidmap = array_idx_modifiable(&bbox->uids, idx);
i_assert(uidmap->virtual_uid == 0);
uidmap->virtual_uid = first_uid + i;
}
ctx->mbox->sync_virtual_next_uid = first_uid + i;
}
static int
virtual_sync_apply_existing_appends(struct virtual_sync_context *ctx)
{
uint32_t virtual_ext_id = ctx->mbox->virtual_ext_id;
struct virtual_backend_box *bbox = NULL;
const struct mail_index_header *hdr;
const struct virtual_mail_index_record *vrec;
struct virtual_backend_uidmap uidmap;
const void *data;
uint32_t seq, seq2;
if (!ctx->mbox->uids_mapped)
return 0;
hdr = mail_index_get_header(ctx->sync_view);
if (ctx->mbox->sync_virtual_next_uid >= hdr->next_uid)
return 0;
/* another process added messages to virtual index. get backend boxes'
uid lists up-to-date by adding the new messages there. */
if (!mail_index_lookup_seq_range(ctx->sync_view,
ctx->mbox->sync_virtual_next_uid,
(uint32_t)-1, &seq, &seq2))
return 0;
memset(&uidmap, 0, sizeof(uidmap));
for (; seq <= seq2; seq++) {
mail_index_lookup_ext(ctx->sync_view, seq, virtual_ext_id,
&data, NULL);
vrec = data;
uidmap.real_uid = vrec->real_uid;
mail_index_lookup_uid(ctx->sync_view, seq, &uidmap.virtual_uid);
if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
bbox = virtual_backend_box_lookup(ctx->mbox,
vrec->mailbox_id);
if (bbox == NULL) {
mail_storage_set_critical(
ctx->mbox->box.storage,
"Mailbox ID %u unexpectedly lost",
vrec->mailbox_id);
return -1;
}
}
array_append(&bbox->uids, &uidmap, 1);
bbox->uids_nonsorted = TRUE;
}
virtual_sync_backend_boxes_sort_uids(ctx->mbox);
return 0;
}
static void
virtual_sync_apply_existing_expunges(struct virtual_mailbox *mbox,
struct mailbox_sync_context *sync_ctx)
{
struct index_mailbox_sync_context *isync_ctx =
(struct index_mailbox_sync_context *)sync_ctx;
struct virtual_backend_box *bbox = NULL;
struct seq_range_iter iter;
const struct virtual_mail_index_record *vrec;
const void *data;
unsigned int n = 0;
uint32_t seq;
if (isync_ctx->expunges == NULL)
return;
seq_range_array_iter_init(&iter, isync_ctx->expunges);
while (seq_range_array_iter_nth(&iter, n++, &seq)) {
mail_index_lookup_ext(mbox->box.view, seq,
mbox->virtual_ext_id, &data, NULL);
vrec = data;
if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) {
bbox = virtual_backend_box_lookup(mbox,
vrec->mailbox_id);
if (!array_is_created(&bbox->sync_outside_expunges))
i_array_init(&bbox->sync_outside_expunges, 32);
}
seq_range_array_add(&bbox->sync_outside_expunges,
vrec->real_uid);
}
}
static int virtual_sync_backend_boxes(struct virtual_sync_context *ctx)
{
struct virtual_backend_box *const *bboxes;
unsigned int i, count;
if (virtual_sync_apply_existing_appends(ctx) < 0)
return -1;
i_array_init(&ctx->all_adds, 128);
bboxes = array_get(&ctx->mbox->backend_boxes, &count);
for (i = 0; i < count; i++) {
if (virtual_sync_backend_box(ctx, bboxes[i]) < 0) {
/* backend failed, copy the error */
virtual_box_copy_error(&ctx->mbox->box,
bboxes[i]->box);
return -1;
}
}
if (!ctx->mbox->uids_mapped) {
/* initial sync: assign virtual UIDs to existing messages and
sync all flags */
ctx->mbox->uids_mapped = TRUE;
virtual_sync_backend_map_uids(ctx);
virtual_sync_new_backend_boxes(ctx);
}
virtual_sync_backend_add_new(ctx);
array_free(&ctx->all_adds);
return 0;
}
static void virtual_sync_backend_boxes_finish(struct virtual_sync_context *ctx)
{
struct virtual_backend_box *const *bboxes;
unsigned int i, count;
bboxes = array_get(&ctx->mbox->backend_boxes, &count);
for (i = 0; i < count; i++)
virtual_backend_box_sync_mail_unset(bboxes[i]);
}
static int virtual_sync_finish(struct virtual_sync_context *ctx, bool success)
{
int ret = success ? 0 : -1;
virtual_sync_backend_boxes_finish(ctx);
if (success) {
if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
mailbox_set_index_error(&ctx->mbox->box);
ret = -1;
}
} else {
if (ctx->index_broken) {
/* make sure we don't complain about the same errors
over and over again. */
if (mail_index_unlink(ctx->index) < 0) {
i_error("virtual index %s: Failed to unlink() "
"broken indexes: %m",
mailbox_get_path(&ctx->mbox->box));
}
}
mail_index_sync_rollback(&ctx->index_sync_ctx);
}
i_free(ctx);
return ret;
}
static int virtual_sync(struct virtual_mailbox *mbox,
enum mailbox_sync_flags flags)
{
struct virtual_sync_context *ctx;
enum mail_index_sync_flags index_sync_flags;
int ret;
ctx = i_new(struct virtual_sync_context, 1);
ctx->mbox = mbox;
ctx->flags = flags;
ctx->index = mbox->box.index;
/* Removed messages are expunged when
a) EXPUNGE is used
b) Mailbox is being opened (FIX_INCONSISTENT is set) */
ctx->expunge_removed =
(ctx->flags & (MAILBOX_SYNC_FLAG_EXPUNGE |
MAILBOX_SYNC_FLAG_FIX_INCONSISTENT)) != 0;
index_sync_flags = MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY |
MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;
if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0)
index_sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
ret = mail_index_sync_begin(ctx->index, &ctx->index_sync_ctx,
&ctx->sync_view, &ctx->trans,
index_sync_flags);
if (ret <= 0) {
if (ret < 0)
mailbox_set_index_error(&mbox->box);
i_free(ctx);
return ret;
}
ret = virtual_sync_ext_header_read(ctx);
if (ret < 0)
return virtual_sync_finish(ctx, FALSE);
if (ret == 0)
ctx->ext_header_rewrite = TRUE;
/* apply changes from virtual index to backend mailboxes */
if (virtual_sync_index_changes(ctx) < 0)
return virtual_sync_finish(ctx, FALSE);
/* update list of UIDs in backend mailboxes */
if (virtual_sync_backend_boxes(ctx) < 0)
return virtual_sync_finish(ctx, FALSE);
virtual_sync_index_finish(ctx);
return virtual_sync_finish(ctx, TRUE);
}
struct mailbox_sync_context *
virtual_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
struct mailbox_sync_context *sync_ctx;
int ret = 0;
if (!box->opened) {
if (mailbox_open(box) < 0)
ret = -1;
}
if (index_mailbox_want_full_sync(&mbox->box, flags) && ret == 0)
ret = virtual_sync(mbox, flags);
sync_ctx = index_mailbox_sync_init(box, flags, ret < 0);
virtual_sync_apply_existing_expunges(mbox, sync_ctx);
return sync_ctx;
}