mail-index-view-sync.c revision 882fd2a050eff86d740f5e5b9fc2fc4f21ed3734
0N/A/* Copyright (c) 2003-2011 Dovecot authors, see the included COPYING file */
0N/A
0N/A#include "lib.h"
0N/A#include "array.h"
0N/A#include "buffer.h"
0N/A#include "mail-index-view-private.h"
115N/A#include "mail-index-sync-private.h"
0N/A#include "mail-index-modseq.h"
0N/A#include "mail-transaction-log.h"
0N/A
0N/A#include <stdlib.h>
0N/A
975N/Astruct mail_index_view_sync_ctx {
1073N/A struct mail_index_view *view;
0N/A enum mail_index_view_sync_flags flags;
116N/A struct mail_index_sync_map_ctx sync_map_ctx;
0N/A
0N/A /* After syncing view, map is replaced with sync_new_map. */
0N/A struct mail_index_map *sync_new_map;
115N/A
115N/A ARRAY_TYPE(seq_range) expunges;
1086N/A unsigned int finish_min_msg_count;
13N/A
0N/A const struct mail_transaction_header *hdr;
0N/A const void *data;
0N/A
0N/A /* temporary variables while handling lost transaction logs: */
0N/A ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw;
0N/A buffer_t *lost_kw_buf;
0N/A uint32_t lost_new_ext_idx;
0N/A /* result of lost transaction logs: */
975N/A ARRAY_TYPE(seq_range) lost_flags;
975N/A unsigned int lost_flag_idx;
975N/A
975N/A size_t data_offset;
975N/A unsigned int failed:1;
975N/A unsigned int sync_map_update:1;
975N/A unsigned int skipped_expunges:1;
975N/A unsigned int last_read:1;
975N/A unsigned int log_was_lost:1;
975N/A unsigned int hidden:1;
975N/A};
0N/A
0N/Astatic int
0N/Aview_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges,
0N/A bool *reset_r)
115N/A{
115N/A const struct mail_index_header *hdr = &view->index->map->hdr;
975N/A uint32_t start_seq, end_seq;
975N/A uoff_t start_offset, end_offset;
975N/A int ret;
0N/A
0N/A start_seq = view->log_file_expunge_seq;
0N/A start_offset = view->log_file_expunge_offset;
975N/A end_seq = hdr->log_file_seq;
975N/A end_offset = hdr->log_file_head_offset;
975N/A
975N/A if (end_seq < view->log_file_head_seq ||
975N/A (end_seq == view->log_file_head_seq &&
975N/A end_offset < view->log_file_head_offset)) {
975N/A mail_index_set_error(view->index,
975N/A "%s log position went backwards "
975N/A "(%u,%"PRIuUOFF_T" < %u,%"PRIuUOFF_T")",
979N/A view->index->filepath, end_seq, end_offset,
979N/A view->log_file_head_seq, view->log_file_head_offset);
975N/A return -1;
979N/A }
975N/A
975N/A for (;;) {
975N/A /* the view begins from the first non-synced transaction */
979N/A ret = mail_transaction_log_view_set(view->log_view,
975N/A start_seq, start_offset,
975N/A end_seq, end_offset,
975N/A reset_r);
975N/A if (ret <= 0)
975N/A return ret;
975N/A
975N/A if (!*reset_r || sync_expunges)
975N/A break;
975N/A
975N/A /* log was reset, but we don't want to sync expunges.
975N/A we can't do this, so sync only up to the reset. */
975N/A mail_transaction_log_view_get_prev_pos(view->log_view,
975N/A &end_seq, &end_offset);
975N/A end_seq--; end_offset = (uoff_t)-1;
975N/A if (end_seq < start_seq) {
975N/A /* we have only this reset log */
975N/A mail_transaction_log_view_clear(view->log_view,
975N/A view->log_file_expunge_seq);
975N/A break;
975N/A }
975N/A }
975N/A return 1;
975N/A}
975N/A
975N/Astatic unsigned int
979N/Aview_sync_expunges2seqs(struct mail_index_view_sync_ctx *ctx)
975N/A{
975N/A struct mail_index_view *view = ctx->view;
975N/A struct seq_range *src, *src_end, *dest;
979N/A unsigned int count, expunge_count = 0;
975N/A uint32_t prev_seq = 0;
975N/A
975N/A /* convert UIDs to sequences */
979N/A src = dest = array_get_modifiable(&ctx->expunges, &count);
975N/A src_end = src + count;
975N/A for (; src != src_end; src++) {
975N/A if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
975N/A &dest->seq1, &dest->seq2))
975N/A count--;
975N/A else {
975N/A i_assert(dest->seq1 > prev_seq);
975N/A prev_seq = dest->seq2;
975N/A
975N/A expunge_count += dest->seq2 - dest->seq1 + 1;
975N/A dest++;
975N/A }
975N/A }
975N/A array_delete(&ctx->expunges, count,
975N/A array_count(&ctx->expunges) - count);
975N/A return expunge_count;
975N/A}
975N/A
975N/Astatic void
975N/Aview_sync_add_expunge_range(ARRAY_TYPE(seq_range) *dest,
975N/A const struct seq_range *src, size_t src_size)
975N/A{
979N/A unsigned int i, src_count;
979N/A
975N/A i_assert(src_size % sizeof(*src) == 0);
975N/A
975N/A src_count = src_size / sizeof(*src);
975N/A for (i = 0; i < src_count; i++)
975N/A seq_range_array_add_range(dest, src[i].seq1, src[i].seq2);
979N/A}
979N/A
975N/Astatic void
975N/Aview_sync_add_expunge_guids(ARRAY_TYPE(seq_range) *dest,
975N/A const struct mail_transaction_expunge_guid *src,
975N/A size_t src_size)
975N/A{
975N/A unsigned int i, src_count;
975N/A
975N/A i_assert(src_size % sizeof(*src) == 0);
975N/A
984N/A src_count = src_size / sizeof(*src);
975N/A for (i = 0; i < src_count; i++)
975N/A seq_range_array_add(dest, 0, src[i].uid);
975N/A}
975N/A
975N/Astatic int
979N/Aview_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
979N/A unsigned int *expunge_count_r)
979N/A{
979N/A struct mail_index_view *view = ctx->view;
975N/A const struct mail_transaction_header *hdr;
975N/A const void *data;
975N/A int ret;
975N/A
984N/A /* get a list of expunge transactions. there may be some that we have
984N/A already synced, but it doesn't matter because they'll get dropped
975N/A out when converting to sequences. the uid ranges' validity has
984N/A already been verified, so we can use them directly. */
984N/A mail_transaction_log_view_mark(view->log_view);
984N/A while ((ret = mail_transaction_log_view_next(view->log_view,
984N/A &hdr, &data)) > 0) {
984N/A if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
984N/A /* skip expunge requests */
975N/A continue;
975N/A }
984N/A if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) {
984N/A view_sync_add_expunge_guids(&ctx->expunges,
984N/A data, hdr->size);
975N/A } else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
975N/A view_sync_add_expunge_range(&ctx->expunges,
975N/A data, hdr->size);
0N/A }
0N/A }
0N/A mail_transaction_log_view_rewind(view->log_view);
0N/A
299N/A *expunge_count_r = view_sync_expunges2seqs(ctx);
911N/A return ret;
299N/A}
911N/A
0N/Astatic bool have_existing_expunges(struct mail_index_view *view,
0N/A const struct seq_range *range, size_t size)
0N/A{
0N/A const struct seq_range *range_end;
0N/A uint32_t seq1, seq2;
205N/A
0N/A range_end = CONST_PTR_OFFSET(range, size);
0N/A for (; range != range_end; range++) {
0N/A if (mail_index_lookup_seq_range(view, range->seq1, range->seq2,
0N/A &seq1, &seq2))
0N/A return TRUE;
0N/A }
0N/A return FALSE;
0N/A}
0N/A
0N/Astatic bool
0N/Ahave_existing_guid_expunge(struct mail_index_view *view,
0N/A const struct mail_transaction_expunge_guid *expunges,
0N/A size_t size)
0N/A{
0N/A const struct mail_transaction_expunge_guid *expunges_end;
0N/A uint32_t seq;
0N/A
0N/A expunges_end = CONST_PTR_OFFSET(expunges, size);
0N/A for (; expunges != expunges_end; expunges++) {
0N/A if (mail_index_lookup_seq(view, expunges->uid, &seq))
0N/A return TRUE;
0N/A }
115N/A return FALSE;
0N/A}
116N/A
0N/Astatic bool view_sync_have_expunges(struct mail_index_view *view)
0N/A{
979N/A const struct mail_transaction_header *hdr;
979N/A const void *data;
0N/A bool have_expunges = FALSE;
0N/A int ret;
0N/A
979N/A mail_transaction_log_view_mark(view->log_view);
979N/A
979N/A while ((ret = mail_transaction_log_view_next(view->log_view,
0N/A &hdr, &data)) > 0) {
979N/A if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
0N/A /* skip expunge requests */
979N/A continue;
0N/A }
116N/A if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) {
116N/A /* we have an expunge. see if it still exists. */
979N/A if (have_existing_guid_expunge(view, data, hdr->size)) {
116N/A have_expunges = TRUE;
979N/A break;
979N/A }
116N/A } else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
979N/A /* we have an expunge. see if it still exists. */
979N/A if (have_existing_expunges(view, data, hdr->size)) {
979N/A have_expunges = TRUE;
116N/A break;
979N/A }
979N/A }
979N/A }
979N/A
979N/A mail_transaction_log_view_rewind(view->log_view);
116N/A
979N/A /* handle failures as having expunges (which is safer).
116N/A we'll probably fail later. */
979N/A return ret < 0 || have_expunges;
116N/A}
979N/A
979N/Astatic int uint_cmp(const void *p1, const void *p2)
979N/A{
979N/A const unsigned int *u1 = p1, *u2 = p2;
979N/A
116N/A if (*u1 < *u2)
979N/A return -1;
975N/A if (*u1 > *u2)
0N/A return 1;
863N/A return 0;
863N/A}
863N/A
863N/Astatic bool view_sync_lost_keywords_equal(struct mail_index_view_sync_ctx *ctx)
901N/A{
901N/A unsigned int *old_idx, *new_idx;
863N/A unsigned int old_count, new_count;
863N/A
1086N/A old_idx = array_get_modifiable(&ctx->lost_old_kw, &old_count);
863N/A new_idx = array_get_modifiable(&ctx->lost_new_kw, &new_count);
863N/A if (old_count != new_count)
863N/A return FALSE;
863N/A
863N/A qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp);
863N/A qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp);
863N/A return memcmp(old_idx, new_idx, old_count * sizeof(old_idx)) == 0;
863N/A}
863N/A
863N/Astatic int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx,
863N/A uint32_t uid)
863N/A{
863N/A struct mail_transaction_header thdr;
863N/A struct mail_transaction_keyword_update kw_up;
863N/A const unsigned int *kw_idx;
863N/A const char *const *kw_names;
863N/A unsigned int i, count;
863N/A
863N/A kw_idx = array_get(&ctx->lost_new_kw, &count);
863N/A if (count == 0)
863N/A return 0;
863N/A kw_names = array_idx(&ctx->view->index->keywords, 0);
863N/A
863N/A memset(&thdr, 0, sizeof(thdr));
863N/A thdr.type = MAIL_TRANSACTION_KEYWORD_UPDATE | MAIL_TRANSACTION_EXTERNAL;
863N/A memset(&kw_up, 0, sizeof(kw_up));
863N/A kw_up.modify_type = MODIFY_ADD;
863N/A /* add new flags one by one */
863N/A for (i = 0; i < count; i++) {
863N/A kw_up.name_size = strlen(kw_names[kw_idx[i]]);
863N/A buffer_set_used_size(ctx->lost_kw_buf, 0);
863N/A buffer_append(ctx->lost_kw_buf, &kw_up, sizeof(kw_up));
975N/A buffer_append(ctx->lost_kw_buf, kw_names[kw_idx[i]],
975N/A kw_up.name_size);
975N/A if (ctx->lost_kw_buf->used % 4 != 0) {
975N/A buffer_append_zero(ctx->lost_kw_buf,
975N/A 4 - ctx->lost_kw_buf->used % 4);
975N/A }
975N/A buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
979N/A buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
975N/A
975N/A thdr.size = ctx->lost_kw_buf->used;
975N/A if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
975N/A ctx->lost_kw_buf->data) < 0)
975N/A return -1;
975N/A }
975N/A return 0;
975N/A}
975N/A
975N/Astatic int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
975N/A uint32_t old_seq, uint32_t new_seq)
975N/A{
975N/A struct mail_index_map *old_map = ctx->view->map;
975N/A struct mail_index_map *new_map = ctx->view->index->map;
975N/A const struct mail_index_record *old_rec, *new_rec;
1086N/A struct mail_transaction_header thdr;
975N/A const struct mail_index_ext *ext;
975N/A const uint64_t *modseqp;
975N/A uint64_t new_modseq;
975N/A bool changed = FALSE;
975N/A
975N/A old_rec = MAIL_INDEX_MAP_IDX(old_map, old_seq - 1);
975N/A new_rec = MAIL_INDEX_MAP_IDX(new_map, new_seq - 1);
975N/A
975N/A memset(&thdr, 0, sizeof(thdr));
1086N/A if (old_rec->flags != new_rec->flags) {
975N/A struct mail_transaction_flag_update flag_update;
975N/A
975N/A /* check this before syncing the record, since it updates
975N/A old_rec. */
975N/A if ((old_rec->flags & MAIL_INDEX_FLAGS_MASK) !=
975N/A (new_rec->flags & MAIL_INDEX_FLAGS_MASK))
975N/A changed = TRUE;
975N/A
975N/A thdr.type = MAIL_TRANSACTION_FLAG_UPDATE |
975N/A MAIL_TRANSACTION_EXTERNAL;
975N/A thdr.size = sizeof(flag_update);
975N/A
975N/A memset(&flag_update, 0, sizeof(flag_update));
975N/A flag_update.uid1 = flag_update.uid2 = new_rec->uid;
975N/A flag_update.add_flags = new_rec->flags;
975N/A flag_update.remove_flags = ~new_rec->flags & 0xff;
975N/A if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
975N/A &flag_update) < 0)
975N/A return -1;
1086N/A }
975N/A
975N/A mail_index_map_lookup_keywords(old_map, old_seq, &ctx->lost_old_kw);
975N/A mail_index_map_lookup_keywords(new_map, new_seq, &ctx->lost_new_kw);
975N/A if (!view_sync_lost_keywords_equal(ctx)) {
975N/A struct mail_transaction_keyword_reset kw_reset;
975N/A
975N/A thdr.type = MAIL_TRANSACTION_KEYWORD_RESET |
975N/A MAIL_TRANSACTION_EXTERNAL;
975N/A thdr.size = sizeof(kw_reset);
975N/A
979N/A /* remove all old flags by resetting them */
979N/A memset(&kw_reset, 0, sizeof(kw_reset));
979N/A kw_reset.uid1 = kw_reset.uid2 = new_rec->uid;
979N/A if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
979N/A &kw_reset) < 0)
979N/A return -1;
979N/A
979N/A view_sync_update_keywords(ctx, new_rec->uid);
979N/A changed = TRUE;
979N/A }
979N/A
979N/A if (changed) {
979N/A /* flags or keywords changed */
979N/A } else if (ctx->view->highest_modseq != 0 &&
979N/A ctx->lost_new_ext_idx != (uint32_t)-1) {
979N/A /* if modseq has changed include this message in changed flags
979N/A list, even if we didn't see any changes above. */
979N/A ext = array_idx(&new_map->extensions, ctx->lost_new_ext_idx);
979N/A modseqp = CONST_PTR_OFFSET(new_rec, ext->record_offset);
979N/A new_modseq = *modseqp;
979N/A
295N/A if (new_modseq > ctx->view->highest_modseq)
295N/A changed = TRUE;
295N/A }
295N/A
295N/A /* without modseqs lost_flags isn't updated perfectly correctly, because
295N/A by the time we're comparing old flags it may have changed from what
295N/A we last sent to the client (because the map is shared). This could
341N/A be avoided by always keeping a private copy of the map in the view,
295N/A but that's a waste of memory for as rare of a problem as this. */
295N/A if (changed)
295N/A seq_range_array_add(&ctx->lost_flags, 0, new_rec->uid);
295N/A return 0;
295N/A}
295N/A
295N/Astatic int
295N/Aview_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx,
295N/A unsigned int *expunge_count_r)
295N/A{
295N/A struct mail_index_view *view = ctx->view;
295N/A struct mail_index_map *old_map = view->map;
295N/A struct mail_index_map *new_map = view->index->map;
295N/A const unsigned int old_count = old_map->hdr.messages_count;
295N/A const unsigned int new_count = new_map->hdr.messages_count;
295N/A const struct mail_index_record *old_rec, *new_rec;
341N/A struct mail_transaction_header thdr;
341N/A unsigned int i, j;
341N/A
341N/A /* we don't update the map in the same order as it's typically done.
341N/A map->rec_map may already have some messages appended that we don't
341N/A want. get an atomic map to make sure these get removed. */
341N/A (void)mail_index_sync_get_atomic_map(&ctx->sync_map_ctx);
341N/A
341N/A if (!mail_index_map_get_ext_idx(new_map, view->index->modseq_ext_id,
301N/A &ctx->lost_new_ext_idx))
301N/A ctx->lost_new_ext_idx = (uint32_t)-1;
301N/A
301N/A i_array_init(&ctx->lost_flags, 64);
301N/A t_array_init(&ctx->lost_old_kw, 32);
336N/A t_array_init(&ctx->lost_new_kw, 32);
301N/A ctx->lost_kw_buf = buffer_create_dynamic(pool_datastack_create(), 128);
301N/A
301N/A /* handle expunges and sync flags */
301N/A i = j = 0;
301N/A while (i < old_count && j < new_count) {
301N/A old_rec = MAIL_INDEX_MAP_IDX(old_map, i);
301N/A new_rec = MAIL_INDEX_MAP_IDX(new_map, j);
301N/A if (old_rec->uid == new_rec->uid) {
301N/A /* message found - check if flags have changed */
301N/A if (view_sync_apply_lost_changes(ctx, i + 1, j + 1) < 0)
301N/A return -1;
301N/A i++; j++;
301N/A } else if (old_rec->uid < new_rec->uid) {
1086N/A /* message expunged */
301N/A seq_range_array_add(&ctx->expunges, 0, old_rec->uid);
301N/A i++;
301N/A } else {
301N/A /* new message appeared out of nowhere */
301N/A mail_index_set_error(view->index,
301N/A "%s view is inconsistent: "
301N/A "uid=%u inserted in the middle of mailbox",
301N/A view->index->filepath, new_rec->uid);
301N/A return -1;
301N/A }
301N/A }
301N/A /* if there are old messages left, they're all expunged */
1086N/A for (; i < old_count; i++) {
301N/A old_rec = MAIL_INDEX_MAP_IDX(old_map, i);
301N/A seq_range_array_add(&ctx->expunges, 0, old_rec->uid);
301N/A }
301N/A /* if there are new messages left, they're all new messages */
301N/A thdr.type = MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL;
301N/A thdr.size = sizeof(*new_rec);
301N/A for (; j < new_count; j++) {
301N/A new_rec = MAIL_INDEX_MAP_IDX(new_map, j);
301N/A if (mail_index_sync_record(&ctx->sync_map_ctx,
301N/A &thdr, new_rec) < 0)
301N/A return -1;
301N/A mail_index_map_lookup_keywords(new_map, j + 1,
301N/A &ctx->lost_new_kw);
301N/A view_sync_update_keywords(ctx, new_rec->uid);
301N/A }
301N/A *expunge_count_r = view_sync_expunges2seqs(ctx);
576N/A
576N/A /* we have no idea how far we've synced - make sure these aren't used */
576N/A old_map->hdr.log_file_seq = 0;
576N/A old_map->hdr.log_file_head_offset = 0;
576N/A old_map->hdr.log_file_tail_offset = 0;
576N/A
576N/A if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
576N/A array_clear(&ctx->expunges);
379N/A ctx->skipped_expunges = *expunge_count_r > 0;
379N/A } else {
379N/A view->log_file_head_seq = new_map->hdr.log_file_seq;
379N/A view->log_file_head_offset = new_map->hdr.log_file_head_offset;
379N/A }
379N/A return 0;
379N/A}
379N/A
1113N/Astatic int mail_index_view_sync_init_fix(struct mail_index_view_sync_ctx *ctx)
398N/A{
379N/A struct mail_index_view *view = ctx->view;
379N/A uint32_t seq;
379N/A uoff_t offset;
1113N/A bool reset;
1113N/A
379N/A /* replace the view's map */
379N/A view->index->map->refcount++;
379N/A mail_index_unmap(&view->map);
379N/A view->map = view->index->map;
1113N/A
1113N/A /* update log positions */
379N/A view->log_file_head_seq = seq = view->map->hdr.log_file_seq;
379N/A view->log_file_head_offset = offset =
379N/A view->map->hdr.log_file_head_offset;
379N/A
379N/A if (mail_transaction_log_view_set(view->log_view, seq, offset,
411N/A seq, offset, &reset) < 0)
379N/A return -1;
398N/A view->inconsistent = FALSE;
398N/A return 0;
398N/A}
398N/A
379N/Astruct mail_index_view_sync_ctx *
379N/Amail_index_view_sync_begin(struct mail_index_view *view,
379N/A enum mail_index_view_sync_flags flags)
411N/A{
997N/A struct mail_index_view_sync_ctx *ctx;
411N/A struct mail_index_map *tmp_map;
411N/A unsigned int expunge_count = 0;
411N/A bool reset, sync_expunges, have_expunges;
411N/A int ret;
411N/A
411N/A i_assert(!view->syncing);
411N/A i_assert(view->transactions == 0);
411N/A
975N/A view->syncing = TRUE;
975N/A
411N/A /* Syncing the view invalidates all previous looked up records.
411N/A Unreference the mappings this view keeps because of them. */
411N/A mail_index_view_unref_maps(view);
411N/A
975N/A ctx = i_new(struct mail_index_view_sync_ctx, 1);
411N/A ctx->view = view;
411N/A ctx->flags = flags;
411N/A
411N/A sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
411N/A if (sync_expunges)
411N/A i_array_init(&ctx->expunges, 64);
411N/A if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) != 0) {
411N/A /* just get this view synced - don't return anything */
411N/A i_assert(sync_expunges);
411N/A if (mail_index_view_sync_init_fix(ctx) < 0)
411N/A ctx->failed = TRUE;
411N/A return ctx;
411N/A }
411N/A if (mail_index_view_is_inconsistent(view)) {
411N/A mail_index_set_error(view->index, "%s view is inconsistent",
411N/A view->index->filepath);
975N/A ctx->failed = TRUE;
411N/A return ctx;
997N/A }
997N/A
997N/A ret = view_sync_set_log_view_range(view, sync_expunges, &reset);
997N/A if (ret < 0) {
997N/A ctx->failed = TRUE;
997N/A return ctx;
997N/A }
997N/A
997N/A if (ret == 0) {
624N/A ctx->log_was_lost = TRUE;
624N/A if (!sync_expunges)
624N/A i_array_init(&ctx->expunges, 64);
624N/A mail_index_sync_map_init(&ctx->sync_map_ctx, view,
624N/A MAIL_INDEX_SYNC_HANDLER_VIEW);
624N/A ret = view_sync_get_log_lost_changes(ctx, &expunge_count);
624N/A mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
624N/A mail_index_sync_map_deinit(&ctx->sync_map_ctx);
624N/A if (ret < 0) {
624N/A mail_index_set_error(view->index,
624N/A "%s view syncing failed to apply changes",
624N/A view->index->filepath);
624N/A view->inconsistent = TRUE;
624N/A ctx->failed = TRUE;
624N/A return ctx;
624N/A }
624N/A have_expunges = expunge_count > 0;
624N/A } else if (sync_expunges) {
624N/A /* get list of all expunges first */
624N/A if (view_sync_get_expunges(ctx, &expunge_count) < 0) {
624N/A ctx->failed = TRUE;
624N/A return ctx;
624N/A }
0N/A have_expunges = expunge_count > 0;
0N/A } else {
0N/A have_expunges = view_sync_have_expunges(view);
0N/A }
0N/A
13N/A ctx->finish_min_msg_count = reset ? 0 :
136N/A view->map->hdr.messages_count - expunge_count;
975N/A if (reset && view->map->hdr.messages_count > 0) {
view->inconsistent = TRUE;
mail_index_set_error(view->index,
"%s reset, view is now inconsistent",
view->index->filepath);
}
if (!have_expunges) {
/* no expunges, we can just replace the map */
if (view->index->map->hdr.messages_count <
ctx->finish_min_msg_count) {
mail_index_set_error(view->index,
"Index %s lost messages without expunging "
"(%u -> %u)", view->index->filepath,
view->map->hdr.messages_count,
view->index->map->hdr.messages_count);
ctx->finish_min_msg_count = 0;
view->inconsistent = TRUE;
}
view->index->map->refcount++;
mail_index_unmap(&view->map);
view->map = view->index->map;
} else {
/* expunges seen. create a private map which we update.
if we're syncing expunges the map will finally be replaced
with the head map to remove the expunged messages. */
ctx->sync_map_update = TRUE;
if (view->map->refcount > 1) {
tmp_map = mail_index_map_clone(view->map);
mail_index_unmap(&view->map);
view->map = tmp_map;
}
if (sync_expunges) {
ctx->sync_new_map = view->index->map;
ctx->sync_new_map->refcount++;
}
}
mail_index_sync_map_init(&ctx->sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_VIEW);
#ifdef DEBUG
mail_index_map_check(view->map);
#endif
return ctx;
}
static bool
view_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
{
const struct mail_index_view_log_sync_area *sync;
if (!array_is_created(&view->syncs_hidden))
return FALSE;
array_foreach(&view->syncs_hidden, sync) {
if (sync->log_file_offset <= offset &&
offset - sync->log_file_offset < sync->length &&
sync->log_file_seq == seq)
return TRUE;
}
return FALSE;
}
static bool
mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
const struct mail_transaction_header *hdr)
{
struct mail_index_view *view = ctx->view;
uint32_t seq;
uoff_t offset, next_offset;
mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
next_offset = offset + sizeof(*hdr) + hdr->size;
if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE |
MAIL_TRANSACTION_EXPUNGE_GUID)) != 0 &&
(hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
i_assert(!LOG_IS_BEFORE(seq, offset,
view->log_file_expunge_seq,
view->log_file_expunge_offset));
if (!ctx->skipped_expunges) {
view->log_file_expunge_seq = seq;
view->log_file_expunge_offset = offset;
ctx->skipped_expunges = TRUE;
}
return FALSE;
}
if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
view->log_file_expunge_offset)) {
/* already synced */
return FALSE;
}
}
if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
view->log_file_head_offset)) {
/* already synced */
return FALSE;
}
view->log_file_head_seq = seq;
view->log_file_head_offset = next_offset;
return TRUE;
}
static int
mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
{
struct mail_transaction_log_view *log_view = ctx->view->log_view;
struct mail_index_view *view = ctx->view;
const struct mail_transaction_header *hdr;
uint32_t seq;
uoff_t offset;
int ret;
bool synced_to_map;
do {
/* Get the next transaction from log. */
ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
&ctx->data);
if (ret <= 0) {
if (ret < 0)
return -1;
ctx->hdr = NULL;
ctx->last_read = TRUE;
return 0;
}
hdr = ctx->hdr;
/* skip records we've already synced */
} while (!mail_index_view_sync_want(ctx, hdr));
mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
/* If we started from a map that we didn't create ourself,
some of the transactions may already be synced. at the end
of this view sync we'll update file_seq=0 so that this check
always becomes FALSE for subsequent syncs. */
synced_to_map = view->map->hdr.log_file_seq != 0 &&
LOG_IS_BEFORE(seq, offset, view->map->hdr.log_file_seq,
view->map->hdr.log_file_head_offset);
/* Apply transaction to view's mapping if needed (meaning we
didn't just re-map the view to head mapping). */
if (ctx->sync_map_update && !synced_to_map) {
if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE |
MAIL_TRANSACTION_EXPUNGE_GUID)) == 0) {
ret = mail_index_sync_record(&ctx->sync_map_ctx,
hdr, ctx->data);
}
if (ret < 0)
return -1;
}
ctx->hidden = view_sync_is_hidden(view, seq, offset);
return 1;
}
static bool
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *rec)
{
const struct mail_transaction_header *hdr = ctx->hdr;
const void *data = ctx->data;
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *update =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_flag_update[] */
for (;;) {
ctx->data_offset += sizeof(*update);
if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(update))
break;
/* skip internal flag changes */
if (ctx->data_offset == ctx->hdr->size)
return 0;
update = CONST_PTR_OFFSET(data, ctx->data_offset);
}
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = update->uid1;
rec->uid2 = update->uid2;
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *update = data;
const uint32_t *uids;
/* data contains mail_transaction_keyword_update header,
the keyword name and an array of { uint32_t uid1, uid2; } */
if (ctx->data_offset == 0) {
/* skip over the header and name */
ctx->data_offset = sizeof(*update) + update->name_size;
if ((ctx->data_offset % 4) != 0)
ctx->data_offset += 4 - (ctx->data_offset % 4);
}
uids = CONST_PTR_OFFSET(data, ctx->data_offset);
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = uids[0];
rec->uid2 = uids[1];
ctx->data_offset += sizeof(uint32_t) * 2;
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *reset =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_keyword_reset[] */
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = reset->uid1;
rec->uid2 = reset->uid2;
ctx->data_offset += sizeof(*reset);
break;
}
default:
ctx->hdr = NULL;
return FALSE;
}
rec->hidden = ctx->hidden;
return TRUE;
}
static bool
mail_index_view_sync_next_lost(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
const struct seq_range *range;
unsigned int count;
range = array_get(&ctx->lost_flags, &count);
if (ctx->lost_flag_idx == count) {
ctx->last_read = TRUE;
return FALSE;
}
sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
sync_rec->uid1 = range[ctx->lost_flag_idx].seq1;
sync_rec->uid2 = range[ctx->lost_flag_idx].seq2;
ctx->lost_flag_idx++;
return TRUE;
}
bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
int ret;
if (ctx->log_was_lost)
return mail_index_view_sync_next_lost(ctx, sync_rec);
do {
if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
ret = mail_index_view_sync_get_next_transaction(ctx);
if (ret <= 0) {
if (ret < 0)
ctx->failed = TRUE;
return FALSE;
}
ctx->data_offset = 0;
}
} while (!mail_index_view_sync_get_rec(ctx, sync_rec));
return TRUE;
}
void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
const ARRAY_TYPE(seq_range) **expunges_r)
{
*expunges_r = &ctx->expunges;
}
static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
{
const struct mail_index_view_log_sync_area *syncs;
unsigned int i, count;
if (!array_is_created(&view->syncs_hidden))
return;
/* Clean up to view's tail */
syncs = array_get(&view->syncs_hidden, &count);
for (i = 0; i < count; i++) {
if ((syncs[i].log_file_offset +
syncs[i].length > view->log_file_expunge_offset &&
syncs[i].log_file_seq == view->log_file_expunge_seq) ||
syncs[i].log_file_seq > view->log_file_expunge_seq)
break;
}
if (i > 0)
array_delete(&view->syncs_hidden, 0, i);
}
int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx,
bool *delayed_expunges_r)
{
struct mail_index_view_sync_ctx *ctx = *_ctx;
struct mail_index_view *view = ctx->view;
int ret = ctx->failed ? -1 : 0;
i_assert(view->syncing);
*_ctx = NULL;
*delayed_expunges_r = ctx->skipped_expunges;
if ((!ctx->last_read || view->inconsistent) &&
(ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
/* we didn't sync everything */
view->inconsistent = TRUE;
ret = -1;
}
if (ctx->sync_map_ctx.modseq_ctx != NULL)
mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
if (ctx->sync_new_map != NULL) {
mail_index_unmap(&view->map);
view->map = ctx->sync_new_map;
} else if (ctx->sync_map_update) {
/* log offsets have no meaning in views. make sure they're not
tried to be used wrong by setting them to zero. */
view->map->hdr.log_file_seq = 0;
view->map->hdr.log_file_head_offset = 0;
view->map->hdr.log_file_tail_offset = 0;
}
i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count);
if (!ctx->skipped_expunges) {
view->log_file_expunge_seq = view->log_file_head_seq;
view->log_file_expunge_offset = view->log_file_head_offset;
}
if (ctx->sync_map_ctx.view != NULL)
mail_index_sync_map_deinit(&ctx->sync_map_ctx);
mail_index_view_sync_clean_log_syncs(ctx->view);
#ifdef DEBUG
mail_index_map_check(view->map);
#endif
/* set log view to empty range so unneeded memory gets freed */
mail_transaction_log_view_clear(view->log_view,
view->log_file_expunge_seq);
if (array_is_created(&ctx->expunges))
array_free(&ctx->expunges);
if (array_is_created(&ctx->lost_flags))
array_free(&ctx->lost_flags);
view->highest_modseq = mail_index_map_modseq_get_highest(view->map);
view->syncing = FALSE;
i_free(ctx);
return ret;
}
void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset,
unsigned int length)
{
struct mail_index_view_log_sync_area *area;
if (!array_is_created(&view->syncs_hidden))
i_array_init(&view->syncs_hidden, 32);
area = array_append_space(&view->syncs_hidden);
area->log_file_seq = log_file_seq;
area->log_file_offset = log_file_offset;
area->length = length;
}