src/lib-index/mail-index-view-sync.c

	mail-index-view-sync.c revision ba94d6d7e2f7b07d51f5c27e7532f6502ac9a298
6ae232055d4d8a97267517c5e50074c2c819941and/* Copyright (C) 2003-2004 Timo Sirainen */
6ae232055d4d8a97267517c5e50074c2c819941and
fd9abdda70912b99b24e3bf1a38f26fde908a74cnd#include "lib.h"
fd9abdda70912b99b24e3bf1a38f26fde908a74cnd#include "array.h"
fd9abdda70912b99b24e3bf1a38f26fde908a74cnd#include "buffer.h"
6ae232055d4d8a97267517c5e50074c2c819941and#include "mail-index-view-private.h"
6ae232055d4d8a97267517c5e50074c2c819941and#include "mail-index-sync-private.h"
6ae232055d4d8a97267517c5e50074c2c819941and#include "mail-transaction-log.h"
6ae232055d4d8a97267517c5e50074c2c819941and
96ad5d81ee4a2cc66a4ae19893efc8aa6d06fae7jailletcstruct mail_index_view_sync_ctx {
6ae232055d4d8a97267517c5e50074c2c819941and    struct mail_index_view *view;
6ae232055d4d8a97267517c5e50074c2c819941and    enum mail_index_view_sync_flags flags;
d29d9ab4614ff992b0e8de6e2b88d52b6f1f153erbowen    struct mail_index_sync_map_ctx sync_map_ctx;
2e545ce2450a9953665f701bb05350f0d3f26275nd    ARRAY_TYPE(seq_range) expunges;
d29d9ab4614ff992b0e8de6e2b88d52b6f1f153erbowen
d29d9ab4614ff992b0e8de6e2b88d52b6f1f153erbowen    const struct mail_transaction_header *hdr;
6ae232055d4d8a97267517c5e50074c2c819941and    const void *data;
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and    size_t data_offset;
af33a4994ae2ff15bc67d19ff1a7feb906745bf8rbowen    unsigned int sync_map_update:1;
3f08db06526d6901aa08c110b5bc7dde6bc39905nd    unsigned int skipped_expunges:1;
6ae232055d4d8a97267517c5e50074c2c819941and    unsigned int last_read:1;
6ae232055d4d8a97267517c5e50074c2c819941and};
6ae232055d4d8a97267517c5e50074c2c819941and
b43f840409794ed298e8634f6284741f193b6c4ftakashistatic int
6ae232055d4d8a97267517c5e50074c2c819941andmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
6ae232055d4d8a97267517c5e50074c2c819941and                   const struct seq_range *src, size_t src_size)
6ae232055d4d8a97267517c5e50074c2c819941and{
b43f840409794ed298e8634f6284741f193b6c4ftakashi    /* Note that all the sequences are actually still UIDs at this point */
11495c9f0bd33e51a25b4d532beadfbcf9b944a3nilgun    const struct seq_range *src_end;
6ae232055d4d8a97267517c5e50074c2c819941and    struct seq_range *dest, new_exp;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    unsigned int first, i, dest_count;
f086b4b402fa9a2fefc7dda85de2a3cc1cd0a654rjung
6ae232055d4d8a97267517c5e50074c2c819941and    i_assert(src_size % sizeof(*src) == 0);
b43f840409794ed298e8634f6284741f193b6c4ftakashi
b43f840409794ed298e8634f6284741f193b6c4ftakashi    /* @UNSAFE */
b43f840409794ed298e8634f6284741f193b6c4ftakashi    dest = array_get_modifiable(expunges, &dest_count);
b43f840409794ed298e8634f6284741f193b6c4ftakashi    if (dest_count == 0) {
6ae232055d4d8a97267517c5e50074c2c819941and        array_append(expunges, src, src_size / sizeof(*src));
6ae232055d4d8a97267517c5e50074c2c819941and        return 0;
6ae232055d4d8a97267517c5e50074c2c819941and    }
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and    src_end = CONST_PTR_OFFSET(src, src_size);
6ae232055d4d8a97267517c5e50074c2c819941and    for (i = 0; src != src_end; src++) {
6ae232055d4d8a97267517c5e50074c2c819941and        /* src[] must be sorted. */
6ae232055d4d8a97267517c5e50074c2c819941and        if (src->seq1 > src->seq2 ||
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            (src+1 != src_end && src->seq2 >= src[1].seq1))
6ae232055d4d8a97267517c5e50074c2c819941and            return -1;
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and        for (; i < dest_count; i++) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            if (src->seq1 < dest[i].seq1)
6ae232055d4d8a97267517c5e50074c2c819941and                break;
6ae232055d4d8a97267517c5e50074c2c819941and        }
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and        new_exp = *src;
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and        first = i;
6ae232055d4d8a97267517c5e50074c2c819941and        while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
6ae232055d4d8a97267517c5e50074c2c819941and            /* we can/must merge with next record */
6ae232055d4d8a97267517c5e50074c2c819941and            if (new_exp.seq2 < dest[i].seq2)
6ae232055d4d8a97267517c5e50074c2c819941and                new_exp.seq2 = dest[i].seq2;
6ae232055d4d8a97267517c5e50074c2c819941and            i++;
6ae232055d4d8a97267517c5e50074c2c819941and        }
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and        if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
6ae232055d4d8a97267517c5e50074c2c819941and            /* continue previous record */
6ae232055d4d8a97267517c5e50074c2c819941and            if (dest[first-1].seq2 < new_exp.seq2)
6ae232055d4d8a97267517c5e50074c2c819941and                dest[first-1].seq2 = new_exp.seq2;
6ae232055d4d8a97267517c5e50074c2c819941and        } else if (i == first) {
6ae232055d4d8a97267517c5e50074c2c819941and            array_insert(expunges, i, &new_exp, 1);
6ae232055d4d8a97267517c5e50074c2c819941and            i++; first++;
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and            dest = array_get_modifiable(expunges, &dest_count);
6ae232055d4d8a97267517c5e50074c2c819941and        } else {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            /* use next record */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            dest[first] = new_exp;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            first++;
6ae232055d4d8a97267517c5e50074c2c819941and        }
1f1b6bf13313fdd14a45e52e553d3ff28689b717coar
6ae232055d4d8a97267517c5e50074c2c819941and        if (i > first) {
6ae232055d4d8a97267517c5e50074c2c819941and            array_delete(expunges, first, i - first);
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and            dest = array_get_modifiable(expunges, &dest_count);
1f1b6bf13313fdd14a45e52e553d3ff28689b717coar            i = first;
1f1b6bf13313fdd14a45e52e553d3ff28689b717coar        }
1f1b6bf13313fdd14a45e52e553d3ff28689b717coar    }
6ae232055d4d8a97267517c5e50074c2c819941and    return 0;
6ae232055d4d8a97267517c5e50074c2c819941and}
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941andstatic int
6ae232055d4d8a97267517c5e50074c2c819941andview_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
6ae232055d4d8a97267517c5e50074c2c819941and    const struct mail_index_header *hdr = &view->index->map->hdr;
6ae232055d4d8a97267517c5e50074c2c819941and    uint32_t start_seq, end_seq;
6ae232055d4d8a97267517c5e50074c2c819941and    uoff_t start_offset, end_offset;
6ae232055d4d8a97267517c5e50074c2c819941and    bool reset;
6ae232055d4d8a97267517c5e50074c2c819941and    int ret;
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and    start_seq = view->log_file_expunge_seq;
6ae232055d4d8a97267517c5e50074c2c819941and    start_offset = view->log_file_expunge_offset;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    end_seq = hdr->log_file_seq;
6ae232055d4d8a97267517c5e50074c2c819941and    end_offset = hdr->log_file_head_offset;
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and    for (;;) {
6ae232055d4d8a97267517c5e50074c2c819941and        /* the view begins from the first non-synced transaction */
6ae232055d4d8a97267517c5e50074c2c819941and        ret = mail_transaction_log_view_set(view->log_view,
6ae232055d4d8a97267517c5e50074c2c819941and                            start_seq, start_offset,
6ae232055d4d8a97267517c5e50074c2c819941and                            end_seq, end_offset,
6ae232055d4d8a97267517c5e50074c2c819941and                            &reset);
6ae232055d4d8a97267517c5e50074c2c819941and        if (ret <= 0) {
6ae232055d4d8a97267517c5e50074c2c819941and            if (ret < 0)
6ae232055d4d8a97267517c5e50074c2c819941and                return -1;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
6ae232055d4d8a97267517c5e50074c2c819941and            /* FIXME: use the new index to get needed
6ae232055d4d8a97267517c5e50074c2c819941and               changes */
6ae232055d4d8a97267517c5e50074c2c819941and            mail_index_set_error(view->index,
6ae232055d4d8a97267517c5e50074c2c819941and                "Transaction log got desynced for index %s",
6ae232055d4d8a97267517c5e50074c2c819941and                view->index->filepath);
6ae232055d4d8a97267517c5e50074c2c819941and            view->inconsistent = TRUE;
6ae232055d4d8a97267517c5e50074c2c819941and            return -1;
6ae232055d4d8a97267517c5e50074c2c819941and        }
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and        if (!reset || sync_expunges)
6ae232055d4d8a97267517c5e50074c2c819941and            break;
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and        /* we can't do this. sync only up to reset. */
6ae232055d4d8a97267517c5e50074c2c819941and        mail_transaction_log_view_get_prev_pos(view->log_view,
6ae232055d4d8a97267517c5e50074c2c819941and                               &end_seq, &end_offset);
6ae232055d4d8a97267517c5e50074c2c819941and        end_seq--; end_offset = (uoff_t)-1;
6ae232055d4d8a97267517c5e50074c2c819941and        if (end_seq < start_seq) {
6ae232055d4d8a97267517c5e50074c2c819941and            /* we have only this reset log */
6ae232055d4d8a97267517c5e50074c2c819941and            mail_transaction_log_view_clear(view->log_view);
6ae232055d4d8a97267517c5e50074c2c819941and            break;
6ae232055d4d8a97267517c5e50074c2c819941and        }
6ae232055d4d8a97267517c5e50074c2c819941and    }
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941and    return 0;
6ae232055d4d8a97267517c5e50074c2c819941and}
6ae232055d4d8a97267517c5e50074c2c819941and
6ae232055d4d8a97267517c5e50074c2c819941andstatic int
b43f840409794ed298e8634f6284741f193b6c4ftakashiview_sync_get_expunges(struct mail_index_view *view,
11495c9f0bd33e51a25b4d532beadfbcf9b944a3nilgun               ARRAY_TYPE(seq_range) *expunges_r)
6ae232055d4d8a97267517c5e50074c2c819941and{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    const struct mail_transaction_header *hdr;
f086b4b402fa9a2fefc7dda85de2a3cc1cd0a654rjung    struct seq_range *src, *src_end, *dest;
727872d18412fc021f03969b8641810d8896820bhumbedooh    const void *data;
0d0ba3a410038e179b695446bb149cce6264e0abnd    unsigned int count;
727872d18412fc021f03969b8641810d8896820bhumbedooh    int ret;
cc7e1025de9ac63bd4db6fe7f71c158b2cf09fe4humbedooh
0d0ba3a410038e179b695446bb149cce6264e0abnd    if (view_sync_set_log_view_range(view, TRUE) < 0)
cc7e1025de9ac63bd4db6fe7f71c158b2cf09fe4humbedooh        return -1;
727872d18412fc021f03969b8641810d8896820bhumbedooh
0d0ba3a410038e179b695446bb149cce6264e0abnd    /* get a list of expunge transactions. there may be some that we have
0d0ba3a410038e179b695446bb149cce6264e0abnd       already synced, but it doesn't matter because they'll get dropped
0d0ba3a410038e179b695446bb149cce6264e0abnd       out when converting to sequences */
ac082aefa89416cbdc9a1836eaf3bed9698201c8humbedooh    i_array_init(expunges_r, 64);
0d0ba3a410038e179b695446bb149cce6264e0abnd    while ((ret = mail_transaction_log_view_next(view->log_view,
0d0ba3a410038e179b695446bb149cce6264e0abnd                             &hdr, &data)) > 0) {
0d0ba3a410038e179b695446bb149cce6264e0abnd        if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
727872d18412fc021f03969b8641810d8896820bhumbedooh            continue;
0d0ba3a410038e179b695446bb149cce6264e0abnd        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
0d0ba3a410038e179b695446bb149cce6264e0abnd            /* this is simply a request for expunge */
30471a4650391f57975f60bbb6e4a90be7b284bfhumbedooh            continue;
205f749042ed530040a4f0080dbcb47ceae8a374rjung        }
af33a4994ae2ff15bc67d19ff1a7feb906745bf8rbowen
0d0ba3a410038e179b695446bb149cce6264e0abnd        if (mail_transaction_log_sort_expunges(expunges_r, data,
7fec19672a491661b2fe4b29f685bc7f4efa64d4nd                               hdr->size) < 0) {
7fec19672a491661b2fe4b29f685bc7f4efa64d4nd            mail_transaction_log_view_set_corrupted(view->log_view,
7fec19672a491661b2fe4b29f685bc7f4efa64d4nd                "Corrupted expunge record");
6ae232055d4d8a97267517c5e50074c2c819941and            ret = -1;
            break;
        }
    }

    if (ret < 0) {
        array_free(expunges_r);
        return -1;
    }

    /* convert UIDs to sequences */
    src = dest = array_get_modifiable(expunges_r, &count);
    src_end = src + count;
    for (; src != src_end; src++) {
        ret = mail_index_lookup_uid_range(view, src->seq1,
                          src->seq2,
                          &dest->seq1,
                          &dest->seq2);
        i_assert(ret == 0);

        if (dest->seq1 == 0)
            count--;
        else
            dest++;
    }
    array_delete(expunges_r, count, array_count(expunges_r) - count);
    return 0;
}

static int have_existing_expunges(struct mail_index_view *view,
                  const struct seq_range *range, size_t size)
{
    const struct seq_range *range_end;
    uint32_t seq1, seq2;

    range_end = CONST_PTR_OFFSET(range, size);
    for (; range < range_end && seq1 != 0; range++) {
        if (mail_index_lookup_uid_range(view, range->seq1, range->seq2,
                        &seq1, &seq2) < 0)
            return -1;
        if (seq1 != 0)
            return 1;
    }
    return 0;
}

static bool view_sync_have_expunges(struct mail_index_view *view)
{
    const struct mail_transaction_header *hdr;
    const void *data;
    uint32_t seq;
    uoff_t offset;
    int ret = 0;

    mail_transaction_log_view_get_prev_pos(view->log_view,
                           &seq, &offset);

    while ((ret = mail_transaction_log_view_next(view->log_view,
                             &hdr, &data)) > 0) {
        if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
            continue;
        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
            /* this is simply a request for expunge */
            continue;
        }

        /* we have an expunge. see if it still exists. */
        ret = have_existing_expunges(view, data, hdr->size);
        if (ret != 0)
            break;
    }

    mail_transaction_log_view_seek(view->log_view, seq, offset);

    /* handle failures as having expunges (which is safer).
       we'll probably fail later. */
    return ret != 0;
}

int mail_index_view_sync_begin(struct mail_index_view *view,
                               enum mail_index_view_sync_flags flags,
                   struct mail_index_view_sync_ctx **ctx_r)
{
    struct mail_index_view_sync_ctx *ctx;
    struct mail_index_map *map;
    ARRAY_TYPE(seq_range) expunges = ARRAY_INIT;
    bool sync_expunges;

    i_assert(!view->syncing);
    i_assert(view->transactions == 0);

    if (mail_index_map_lock(view->index->map) < 0)
        return -1;

    sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
    if (sync_expunges) {
        /* get list of all expunges first */
        if (view_sync_get_expunges(view, &expunges) < 0)
            return -1;
    }

    if (view_sync_set_log_view_range(view, sync_expunges) < 0) {
        if (array_is_created(&expunges))
            array_free(&expunges);
        return -1;
    }

    ctx = i_new(struct mail_index_view_sync_ctx, 1);
    ctx->view = view;
    ctx->flags = flags;
    ctx->expunges = expunges;
    mail_index_sync_map_init(&ctx->sync_map_ctx, view,
                 MAIL_INDEX_SYNC_HANDLER_VIEW);

    if (sync_expunges || !view_sync_have_expunges(view)) {
        view->sync_new_map = view->index->map;
        view->sync_new_map->refcount++;

        /* keep the old mapping without expunges until we're
           fully synced */
    } else {
        /* We need a private copy of the map if we don't want to
           sync expunges.

           If view's map is the head map, it means that it contains
           already all the latest changes and there's no need for us
           to apply any changes to it. This can only happen if there
           hadn't been any expunges. */
        if (view->map != view->index->map) {
            /* Using non-head mapping. We have to apply
               transactions to it to get latest changes into it. */
            ctx->sync_map_update = TRUE;
        }

        /* Unless map was synced at the exact same position as
           view, the message flags can't be reliably used to
           update flag counters. note that map->hdr may contain
           old information if another process updated the
           index file since. */
        if (view->map->mmap_base != NULL) {
            // FIXME: locking should do this..?..
            const struct mail_index_header *hdr;

            hdr = view->map->mmap_base;
            view->map->hdr = *hdr;
        }

        if (view->map->refcount > 1) {
            map = mail_index_map_clone(view->map);
            mail_index_unmap(&view->map);
            view->map = map;
        } else {
            map = view->map;
        }
        i_assert(map->records_count == map->hdr.messages_count);
    }

#ifdef DEBUG
    mail_index_map_check(view->map);
#endif

    /* Syncing the view invalidates all previous looked up records.
       Unreference the mappings this view keeps because of them. */
    mail_index_view_unref_maps(view);
    view->syncing = TRUE;

    *ctx_r = ctx;
    return 0;
}

static bool
view_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
{
    const struct mail_index_view_log_sync_area *syncs;
    unsigned int i, count;

    if (!array_is_created(&view->syncs_hidden))
        return FALSE;

    syncs = array_get(&view->syncs_hidden, &count);
    for (i = 0; i < count; i++) {
        if (syncs[i].log_file_offset <= offset &&
            offset - syncs[i].log_file_offset < syncs[i].length &&
            syncs[i].log_file_seq == seq)
            return TRUE;
    }

    return FALSE;
}

static bool
mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
              const struct mail_transaction_header *hdr)
{
        struct mail_index_view *view = ctx->view;
    uint32_t seq;
    uoff_t offset, next_offset;

    mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
    next_offset = offset + sizeof(*hdr) + hdr->size;

    if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
        (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
        if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
            i_assert(!LOG_IS_BEFORE(seq, offset,
                        view->log_file_expunge_seq,
                        view->log_file_expunge_offset));
            if (!ctx->skipped_expunges) {
                view->log_file_expunge_seq = seq;
                view->log_file_expunge_offset = offset;
                ctx->skipped_expunges = TRUE;
            }
            return FALSE;
        }
        if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
                  view->log_file_expunge_offset)) {
            /* already synced */
            return FALSE;
        }
    }

    if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
              view->log_file_head_offset)) {
        /* already synced */
        return FALSE;
    }

    view->log_file_head_seq = seq;
    view->log_file_head_offset = next_offset;
    return TRUE;
}

static int
mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
{
        struct mail_transaction_log_view *log_view = ctx->view->log_view;
    struct mail_index_view *view = ctx->view;
    const struct mail_transaction_header *hdr;
    uint32_t seq;
    uoff_t offset;
    int ret;
    bool synced_to_map;

    for (;;) {
        /* Get the next transaction from log. */
        ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
                             &ctx->data);
        if (ret <= 0) {
            if (ret < 0)
                return -1;

            ctx->hdr = NULL;
            ctx->last_read = TRUE;
            return 0;
        }

        hdr = ctx->hdr;
        if (!mail_index_view_sync_want(ctx, hdr)) {
            /* This is a visible record that we don't want to
               sync. */
            continue;
        }

        mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);

        /* If we started from a map that we didn't create ourself,
           some of the transactions may already be synced. at the end
           of this view sync we'll update file_seq=0 so that this check
           always becomes FALSE for subsequent syncs. */
        synced_to_map = view->map->hdr.log_file_seq != 0 &&
            !LOG_IS_BEFORE(seq, offset,
                       view->map->hdr.log_file_seq,
                       view->map->hdr.log_file_head_offset);

        /* Apply transaction to view's mapping if needed (meaning we
           didn't just re-map the view to head mapping). */
        if (ctx->sync_map_update && !synced_to_map) {
            i_assert((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0 ||
                 (hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0);

            if (mail_index_sync_record(&ctx->sync_map_ctx,
                           hdr, ctx->data) < 0)
                return -1;
        }

        /* skip changes committed by hidden transactions (eg. in IMAP
           store +flags.silent command) */
        if (view_sync_is_hidden(view, seq, offset))
            continue;
        break;
    }
    return 1;
}

#define FLAG_UPDATE_IS_INTERNAL(u) \
    ((((u)->add_flags | (u)->remove_flags) & \
      ~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0)

static bool
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
                 struct mail_index_view_sync_rec *rec)
{
    const struct mail_transaction_header *hdr = ctx->hdr;
    const void *data = ctx->data;

    switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
    case MAIL_TRANSACTION_APPEND: {
        /* data contains the appended records, but we don't care */
        rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
        rec->uid1 = rec->uid2 = 0;
        ctx->data_offset += hdr->size;
        break;
    }
    case MAIL_TRANSACTION_EXPUNGE: {
        const struct mail_transaction_expunge *exp =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
            /* this is simply a request for expunge */
            ctx->data_offset = ctx->hdr->size;
            return 0;
        }

        /* data contains mail_transaction_expunge[] */
        rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
        rec->uid1 = exp->uid1;
        rec->uid2 = exp->uid2;

        ctx->data_offset += sizeof(*exp);
        break;
    }
    case MAIL_TRANSACTION_FLAG_UPDATE: {
        const struct mail_transaction_flag_update *update =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_flag_update[] */
        for (;;) {
            ctx->data_offset += sizeof(*update);
            if (!FLAG_UPDATE_IS_INTERNAL(update))
                break;

            /* skip internal flag changes */
            if (ctx->data_offset == ctx->hdr->size)
                return 0;

            update = CONST_PTR_OFFSET(data, ctx->data_offset);
        }

        rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
        rec->uid1 = update->uid1;
        rec->uid2 = update->uid2;
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_UPDATE: {
        const struct mail_transaction_keyword_update *update = data;
        const uint32_t *uids;

        /* data contains mail_transaction_keyword_update header,
           the keyword name and an array of { uint32_t uid1, uid2; } */

        if (ctx->data_offset == 0) {
            /* skip over the header and name */
            ctx->data_offset = sizeof(*update) + update->name_size;
            if ((ctx->data_offset % 4) != 0)
                ctx->data_offset += 4 - (ctx->data_offset % 4);
        }

        uids = CONST_PTR_OFFSET(data, ctx->data_offset);
        rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
        rec->uid1 = uids[0];
        rec->uid2 = uids[1];

        ctx->data_offset += sizeof(uint32_t) * 2;
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_RESET: {
        const struct mail_transaction_keyword_reset *reset =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_keyword_reset[] */
        rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
        rec->uid1 = reset->uid1;
        rec->uid2 = reset->uid2;
        ctx->data_offset += sizeof(*reset);
        break;
    }
    default:
        ctx->hdr = NULL;
        return FALSE;
    }
    return TRUE;
}

int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
                  struct mail_index_view_sync_rec *sync_rec)
{
    int ret;

    do {
        if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
            ret = mail_index_view_sync_get_next_transaction(ctx);
            if (ret <= 0)
                return ret;

            ctx->data_offset = 0;
        }
    } while (!mail_index_view_sync_get_rec(ctx, sync_rec));

    return 1;
}

void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
                       const ARRAY_TYPE(seq_range) **expunges_r)
{
    *expunges_r = &ctx->expunges;
}

static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
{
    const struct mail_index_view_log_sync_area *syncs;
    unsigned int i, count;

    if (!array_is_created(&view->syncs_hidden))
        return;

    /* Clean up to view's tail */
    syncs = array_get(&view->syncs_hidden, &count);
    for (i = 0; i < count; i++) {
        if ((syncs[i].log_file_offset +
             syncs[i].length > view->log_file_expunge_offset &&
                     syncs[i].log_file_seq == view->log_file_expunge_seq) ||
            syncs[i].log_file_seq > view->log_file_expunge_seq)
            break;
    }
    if (i > 0)
        array_delete(&view->syncs_hidden, 0, i);
}

void mail_index_view_sync_end(struct mail_index_view_sync_ctx **_ctx)
{
        struct mail_index_view_sync_ctx *ctx = *_ctx;
        struct mail_index_view *view = ctx->view;

    i_assert(view->syncing);

    *_ctx = NULL;

    if (!ctx->last_read) {
        /* we didn't sync everything */
        view->inconsistent = TRUE;
    }

    if (view->sync_new_map != NULL) {
        mail_index_unmap(&view->map);
        view->map = view->sync_new_map;
        view->sync_new_map = NULL;
    }

    if (!ctx->skipped_expunges) {
        view->log_file_expunge_seq = view->log_file_head_seq;
        view->log_file_expunge_offset = view->log_file_head_offset;
    }

    if (ctx->sync_map_update) {
        /* log offsets have no meaning in views. make sure they're not
           tried to be used wrong by setting them to zero. */
        view->map->hdr.log_file_seq = 0;
        view->map->hdr.log_file_head_offset = 0;
        view->map->hdr.log_file_tail_offset = 0;
    }

    mail_index_sync_map_deinit(&ctx->sync_map_ctx);
    mail_index_view_sync_clean_log_syncs(ctx->view);

#ifdef DEBUG
    mail_index_map_check(view->map);
#endif

    /* set log view to empty range so unneeded memory gets freed */
    mail_transaction_log_view_clear(view->log_view);

    if (array_is_created(&ctx->expunges))
        array_free(&ctx->expunges);

    view->syncing = FALSE;
    i_free(ctx);
}

void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
                        uint32_t log_file_seq,
                        uoff_t log_file_offset,
                        unsigned int length)
{
    struct mail_index_view_log_sync_area *area;

    if (!array_is_created(&view->syncs_hidden))
        i_array_init(&view->syncs_hidden, 32);

    area = array_append_space(&view->syncs_hidden);
    area->log_file_seq = log_file_seq;
    area->log_file_offset = log_file_offset;
    area->length = length;
}