mail-index-sync.c revision 4f22fc1d063d48ed368b4cb124b44183bc27773c
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher/* Copyright (C) 2003-2004 Timo Sirainen */
ad805face83ba7d67b1cf2067a1982c7e63d1060Jakub Hrozekstatic void mail_index_sync_sort_flags(struct mail_index_sync_ctx *ctx)
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek const struct mail_transaction_flag_update *src, *src_end;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher const struct mail_transaction_flag_update *dest;
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher struct mail_transaction_flag_update new_update;
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher struct mail_transaction_expunge_traverse_ctx *exp_ctx;
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher src_end = PTR_OFFSET(src, ctx->hdr->size);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher dest = buffer_get_data(ctx->updates_buf, &dest_count);
b47fd11a259c50e63cd674c7cba0da3f2549cae0Jakub Hrozek exp_ctx = mail_transaction_expunge_traverse_init(ctx->expunges_buf);
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek /* find seq1 */
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher /* find seq2 */
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher mail_transaction_expunge_traverse_to(exp_ctx,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher /* insert it into buffer, split it in multiple parts if needed
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher to make sure the ordering stays the same */
f10ebaa51ecdcbbd10f171d19fe8e680e5bc74aaJakub Hrozek for (; i < dest_count; i++) {
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher buffer_insert(ctx->updates_buf, i * sizeof(new_update),
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher dest = buffer_get_data(ctx->updates_buf, NULL);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher buffer_insert(ctx->updates_buf, i * sizeof(new_update),
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher dest = buffer_get_data(ctx->updates_buf, NULL);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher mail_transaction_expunge_traverse_deinit(exp_ctx);
b47fd11a259c50e63cd674c7cba0da3f2549cae0Jakub Hrozekstatic void mail_index_sync_sort_transaction(struct mail_index_sync_ctx *ctx)
b47fd11a259c50e63cd674c7cba0da3f2549cae0Jakub Hrozek switch (ctx->hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
b47fd11a259c50e63cd674c7cba0da3f2549cae0Jakub Hrozek if (buffer_get_used_size(ctx->expunges_buf) == 0) {
9a839b29816c8906d4a6b074cf76df790cac9209Jakub Hrozek mail_transaction_log_sort_expunges(ctx->expunges_buf,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher if (buffer_get_used_size(ctx->expunges_buf) == 0 &&
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher buffer_get_used_size(ctx->updates_buf) == 0) {
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher buffer_append(ctx->updates_buf, ctx->data,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher buffer_append(ctx->appends_buf, ctx->data, ctx->hdr->size);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagherstatic int mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx,
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher flag = external ? MAIL_TRANSACTION_EXTERNAL : 0;
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher while ((ret = mail_transaction_log_view_next(ctx->view->log_view,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) == flag)
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher ctx->expunges = buffer_get_data(ctx->expunges_buf, &size);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher ctx->expunges_count = size / sizeof(*ctx->expunges);
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher ctx->updates = buffer_get_data(ctx->updates_buf, &size);
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher ctx->updates_count = size / sizeof(*ctx->updates);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagherstatic int mail_index_need_lock(struct mail_index *index,
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher uint32_t log_file_seq, uoff_t log_file_offset)
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher if (index->hdr->log_file_seq > log_file_seq ||
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher (index->hdr->log_file_seq == log_file_seq &&
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher index->hdr->log_file_offset >= log_file_offset)) {
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher /* already synced */
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagherint mail_index_sync_begin(struct mail_index *index,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher uint32_t log_file_seq, uoff_t log_file_offset)
b47fd11a259c50e63cd674c7cba0da3f2549cae0Jakub Hrozek if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
9a839b29816c8906d4a6b074cf76df790cac9209Jakub Hrozek if (mail_index_lock_shared(index, TRUE, &lock_id) < 0) {
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher mail_transaction_log_sync_unlock(index->log);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher if (!mail_index_need_lock(index, log_file_seq, log_file_offset)) {
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher mail_transaction_log_sync_unlock(index->log);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher ctx = i_new(struct mail_index_sync_ctx, 1);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher if (mail_transaction_log_view_set(ctx->view->log_view,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher /* we need to have all the transactions sorted to optimize
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher caller's mailbox access patterns */
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher ctx->expunges_buf = buffer_create_dynamic(default_pool,
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher ctx->updates_buf = buffer_create_dynamic(default_pool,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher ctx->appends_buf = buffer_create_dynamic(default_pool,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher if (mail_index_sync_read_and_sort(ctx, FALSE) < 0) {
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallaghermail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher const struct mail_transaction_expunge *exp)
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallaghermail_index_sync_get_update(struct mail_index_sync_rec *rec,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher const struct mail_transaction_flag_update *update)
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher memcpy(rec->add_keywords, update->add_keywords,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher memcpy(rec->remove_keywords, update->remove_keywords,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagherstatic int mail_index_sync_rec_check(struct mail_index_view *view,
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek mail_transaction_log_view_set_corrupted(view->log_view,
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek "Broken sequence: %u..%u (type 0x%x)",
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek message_count = mail_index_view_get_message_count(view);
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek mail_transaction_log_view_set_corrupted(view->log_view,
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek "Sequence out of range: %u > %u (type 0x%x)",
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozekint mail_index_sync_next(struct mail_index_sync_ctx *ctx,
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek const struct mail_transaction_expunge *next_exp;
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek const struct mail_transaction_flag_update *next_update;
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek next_exp = ctx->expunge_idx == ctx->expunges_count ? NULL :
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek next_update = ctx->update_idx == ctx->updates_count ? NULL :
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek // FIXME: return dirty flagged records as flag updates
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek /* the ugliness here is to avoid returning overlapping expunge
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek and update areas. For example:
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek updates[] = A { 1, 7 }, B { 1, 3 }
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek expunges[] = { 5, 6 }
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek will make us return
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek update A: 1, 4
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek update B: 1, 3
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek expunge : 5, 6
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek update A: 7, 7
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek (next_exp == NULL || next_update->seq1 < next_exp->seq1)) {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek mail_index_sync_get_update(sync_rec, next_update);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek /* it's overlapping.. */
f45a20d6ba9e8d695ec3ab707f0cc082999aa4a3Jakub Hrozek return mail_index_sync_rec_check(ctx->view, sync_rec);
f45a20d6ba9e8d695ec3ab707f0cc082999aa4a3Jakub Hrozek /* a few sanity checks here, we really don't ever want to
f45a20d6ba9e8d695ec3ab707f0cc082999aa4a3Jakub Hrozek accidentally expunge a message. If sequence and UID matches,
f45a20d6ba9e8d695ec3ab707f0cc082999aa4a3Jakub Hrozek it's quite unlikely this expunge was caused by some bug. */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek mail_index_sync_get_expunge(sync_rec, next_exp);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (mail_index_sync_rec_check(ctx->view, sync_rec) < 0)
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek if (mail_index_lookup_uid(ctx->view, next_exp->seq1, &uid1) < 0)
481ec0e1eb0058195732cb320845b41f6f4d43ebJakub Hrozek if (mail_index_lookup_uid(ctx->view, next_exp->seq2, &uid2) < 0)
481ec0e1eb0058195732cb320845b41f6f4d43ebJakub Hrozek if (next_exp->uid1 != uid1 || next_exp->uid2 != uid2) {
9a839b29816c8906d4a6b074cf76df790cac9209Jakub Hrozek "UIDs %u..%u doesn't match real UIDs %u..%u",
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek /* scan updates again from the beginning */
481ec0e1eb0058195732cb320845b41f6f4d43ebJakub Hrozek sync_rec->seq1 = ctx->index->map->records_count+1;
9a839b29816c8906d4a6b074cf76df790cac9209Jakub Hrozek sync_rec->appends = buffer_get_data(ctx->appends_buf, NULL);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozekint mail_index_sync_have_more(struct mail_index_sync_ctx *ctx)
9a839b29816c8906d4a6b074cf76df790cac9209Jakub Hrozek return (ctx->update_idx != ctx->updates_count) ||
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozekint mail_index_sync_set_dirty(struct mail_index_sync_ctx *ctx, uint32_t seq)
d25fa6f2608d5fe0617ada47f9d426f45deb96ffJakub Hrozek i_assert(seq <= ctx->view->map->records_count);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek ctx->view->map->records[seq-1].flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozekint mail_index_sync_end(struct mail_index_sync_ctx *ctx,
7465d6a1ef6e83825dba3a4dc4dda7271671aba0Jakub Hrozek if (mail_transaction_log_view_is_corrupted(ctx->view->log_view))
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek mail_transaction_log_get_head(ctx->index->log, &seq, &offset);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher mail_transaction_log_view_unset(ctx->view->log_view);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher if (mail_transaction_log_view_set(ctx->view->log_view,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher if (mail_index_sync_update_index(ctx, sync_stamp,
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher mail_index_unlock(ctx->index, ctx->dirty_lock_id);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher mail_transaction_log_sync_unlock(ctx->index->log);
b47fd11a259c50e63cd674c7cba0da3f2549cae0Jakub Hrozekvoid mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec,
36b56482ca1e53d832accef0354124fd79711172Jakub Hrozek i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
6b0f9cd2ee601121cb7fe1d9ad8ebce782aa8f39Stephen Gallagher *flags = (*flags & ~sync_rec->remove_flags) | sync_rec->add_flags;
1008001f34abb42df75f840db17f14a83f0c21d4Stephen Gallagher for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
b47fd11a259c50e63cd674c7cba0da3f2549cae0Jakub Hrozek keywords[i] = (keywords[i] & ~sync_rec->remove_keywords[i]) |