src/lib-index/mail-index-sync-update.c

	mail-index-sync-update.c revision 659fe5d24825b160cae512538088020d97a60239
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen/* Copyright (C) 2004 Timo Sirainen */
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen#include "lib.h"
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen#include "buffer.h"
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen#include "file-set-size.h"
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen#include "mmap-util.h"
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen#include "mail-index-view-private.h"
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen#include "mail-index-sync-private.h"
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen#include "mail-transaction-log.h"
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainenstruct mail_index_update_ctx {
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    struct mail_index *index;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    struct mail_index_view *view;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    struct mail_index_header hdr;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    struct mail_transaction_log_view *log_view;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen};
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainenvoid mail_index_header_update_counts(struct mail_index_header *hdr,
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                     uint8_t old_flags, uint8_t new_flags)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen{
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        /* different seen-flag */
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        if ((old_flags & MAIL_SEEN) == 0)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen            hdr->seen_messages_count++;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        else
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen            hdr->seen_messages_count--;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    }
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        /* different deleted-flag */
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        if ((old_flags & MAIL_DELETED) == 0)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen            hdr->deleted_messages_count++;
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen        else
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen            hdr->deleted_messages_count--;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    }
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen}
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainenvoid mail_index_header_update_lowwaters(struct mail_index_header *hdr,
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                    const struct mail_index_record *rec)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen{
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    if ((rec->flags & MAIL_RECENT) != 0 &&
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        rec->uid < hdr->first_recent_uid_lowwater)
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        hdr->first_recent_uid_lowwater = rec->uid;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    if ((rec->flags & MAIL_SEEN) == 0 &&
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        rec->uid < hdr->first_unseen_uid_lowwater)
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        hdr->first_unseen_uid_lowwater = rec->uid;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    if ((rec->flags & MAIL_DELETED) != 0 &&
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        rec->uid < hdr->first_deleted_uid_lowwater)
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        hdr->first_deleted_uid_lowwater = rec->uid;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen}
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainenstatic void mail_index_sync_update_expunges(struct mail_index_update_ctx *ctx,
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen                        uint32_t seq1, uint32_t seq2)
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen{
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    struct mail_index_record *rec;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    rec = &ctx->index->map->records[seq1-1];
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    for (; seq1 <= seq2; seq1++, rec++)
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        mail_index_header_update_counts(&ctx->hdr, rec->flags, 0);
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen}
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainenstatic void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx,
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen                     struct mail_index_sync_rec *syncrec)
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen{
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    struct mail_index_record *rec, *end;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    uint8_t flag_mask, old_flags;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    keywords_mask_t keyword_mask;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    uint32_t seq1, seq2;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    int i, update_keywords, ret;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    ret = mail_index_lookup_uid_range(ctx->view, syncrec->uid1,
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen                      syncrec->uid2, &seq1, &seq2);
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    i_assert(ret == 0);
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (seq1 == 0)
4b41116563110d00330896a568eff1078c382827Timo Sirainen        return;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    update_keywords = FALSE;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
4b41116563110d00330896a568eff1078c382827Timo Sirainen        if (syncrec->add_keywords[i] != 0)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen            update_keywords = TRUE;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        if (syncrec->remove_keywords[i] != 0)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen            update_keywords = TRUE;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        keyword_mask[i] = ~syncrec->remove_keywords[i];
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    }
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    flag_mask = ~syncrec->remove_flags;
4b41116563110d00330896a568eff1078c382827Timo Sirainen    rec = &ctx->index->map->records[seq1-1];
4b41116563110d00330896a568eff1078c382827Timo Sirainen    end = rec + (seq2 - seq1) + 1;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    for (; rec != end; rec++) {
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        old_flags = rec->flags;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        rec->flags = (rec->flags & flag_mask) | syncrec->add_flags;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        if (update_keywords) {
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen            for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                rec->keywords[i] =
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                    (rec->keywords[i] & keyword_mask[i]) |
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                    syncrec->add_keywords[i];
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen            }
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        }
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        mail_index_header_update_counts(&ctx->hdr,
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen                        old_flags, rec->flags);
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen                mail_index_header_update_lowwaters(&ctx->hdr, rec);
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    }
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen}
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainenstatic int mail_index_grow(struct mail_index *index, unsigned int count)
4b41116563110d00330896a568eff1078c382827Timo Sirainen{
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    struct mail_index_map *map = index->map;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    unsigned int records_count;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    size_t size;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen    if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen        (void)buffer_append_space_unsafe(map->buffer,
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen            count * sizeof(struct mail_index_record));
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        map->records = buffer_get_modifyable_data(map->buffer, NULL);
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        return 0;
4b41116563110d00330896a568eff1078c382827Timo Sirainen    }
4b41116563110d00330896a568eff1078c382827Timo Sirainen
4b41116563110d00330896a568eff1078c382827Timo Sirainen    size = map->hdr->header_size +
4b41116563110d00330896a568eff1078c382827Timo Sirainen        (map->records_count + count) * sizeof(struct mail_index_record);
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (size <= map->mmap_size)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        return 0;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    /* when we grow fast, do it exponentially */
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (count < index->last_grow_count)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        count = index->last_grow_count;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (count < MAIL_INDEX_MAX_POWER_GROW)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        count = nearest_power(count);
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    index->last_grow_count = count;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    size = map->hdr->header_size +
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        (map->records_count + count) * sizeof(struct mail_index_record);
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (file_set_size(index->fd, (off_t)size) < 0)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        return mail_index_set_syscall_error(index, "file_set_size()");
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    records_count = map->records_count;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (mail_index_map(index, TRUE) <= 0)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        return -1;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    i_assert(map->mmap_size >= size);
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    map->records_count = records_count;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    return 0;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen}
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainenstatic int mail_index_sync_appends(struct mail_index_update_ctx *ctx,
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                   const struct mail_index_record *appends,
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                   unsigned int count)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen{
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    struct mail_index_map *map = ctx->index->map;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    unsigned int i;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    uint32_t next_uid;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    if (mail_index_grow(ctx->index, count) < 0)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        return -1;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    next_uid = ctx->hdr.next_uid;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    for (i = 0; i < count; i++) {
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        mail_index_header_update_counts(&ctx->hdr, 0, appends[i].flags);
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                mail_index_header_update_lowwaters(&ctx->hdr, &appends[i]);
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        if (appends[i].uid < next_uid) {
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen            mail_transaction_log_view_set_corrupted(ctx->log_view,
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                "Append with UID %u, but next_uid = %u",
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen                appends[i].uid, next_uid);
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen            return -1;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        }
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        next_uid = appends[i].uid+1;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    }
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    ctx->hdr.next_uid = next_uid;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    memcpy(map->records + map->records_count, appends,
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen           count * sizeof(*appends));
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    map->records_count += count;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    return 0;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen}
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainenint mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen                 uint32_t sync_stamp, uint64_t sync_size)
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen{
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    struct mail_index *index = sync_ctx->index;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    struct mail_index_map *map;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        struct mail_index_update_ctx ctx;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    struct mail_index_sync_rec rec;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    const struct mail_index_record *appends;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    unsigned int append_count;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    uint32_t count, file_seq, src_idx, dest_idx, dirty_flag;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    uint32_t seq1, seq2;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    uoff_t file_offset;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    unsigned int lock_id;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    int ret, changed;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    /* rewind */
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    sync_ctx->update_idx = sync_ctx->expunge_idx = 0;
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen    sync_ctx->sync_appends =
c6a57378d3c54988f525f81e19c0c5d132a0770dTimo Sirainen        buffer_get_used_size(sync_ctx->appends_buf) != 0;

    changed = mail_index_sync_have_more(sync_ctx);

    memset(&ctx, 0, sizeof(ctx));
    ctx.index = index;
    ctx.view = sync_ctx->view;
    ctx.hdr = *index->hdr;
    ctx.log_view = sync_ctx->view->log_view;

    dirty_flag = sync_ctx->have_dirty ? MAIL_INDEX_HDR_FLAG_HAVE_DIRTY : 0;
    if ((ctx.hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != dirty_flag) {
        ctx.hdr.flags ^= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
        changed = TRUE;
    }

    /* see if we need to update sync headers */
    if (ctx.hdr.sync_stamp != sync_stamp && sync_stamp != 0) {
        ctx.hdr.sync_stamp = sync_stamp;
        changed = TRUE;
    }
    if (ctx.hdr.sync_size != sync_size && sync_size != 0) {
        ctx.hdr.sync_size = sync_size;
        changed = TRUE;
    }

    if (!changed) {
        /* nothing to sync */
        return 0;
    }

    if (mail_index_lock_exclusive(index, &lock_id) < 0)
        return -1;

    map = index->map;
    if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
        map->write_to_disk = TRUE;

    src_idx = dest_idx = 0;
    append_count = 0; appends = NULL;
    while (mail_index_sync_next(sync_ctx, &rec) > 0) {
        switch (rec.type) {
        case MAIL_INDEX_SYNC_TYPE_APPEND:
            i_assert(appends == NULL);
            appends = rec.appends;
            append_count = rec.appends_count;
            break;
        case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
            ret = mail_index_lookup_uid_range(sync_ctx->view,
                              rec.uid1, rec.uid2,
                              &seq1, &seq2);
            i_assert(ret == 0);

            if (seq1 == 0)
                break;

            if (src_idx == 0) {
                /* expunges have to be atomic. so we'll have
                   to copy the mapping, do the changes there
                   and then finally replace the whole index
                   file. to avoid extra disk I/O we copy the
                   index into memory rather than to temporary
                   file */
                map = mail_index_map_to_memory(map);
                mail_index_unmap(index, index->map);
                index->map = map;
                index->hdr = map->hdr;
                map->write_to_disk = TRUE;

                dest_idx = seq1-1;
            } else {
                count = (seq1-1) - src_idx;
                memmove(map->records + dest_idx,
                    map->records + src_idx,
                    count * sizeof(*map->records));
                dest_idx += count;
            }

            mail_index_sync_update_expunges(&ctx, seq1, seq2);
            src_idx = seq2;
            break;
        case MAIL_INDEX_SYNC_TYPE_FLAGS:
            mail_index_sync_update_flags(&ctx, &rec);
            break;
        }
    }

    if (src_idx != 0) {
        count = map->records_count - src_idx;
        memmove(map->records + dest_idx,
            map->records + src_idx,
            count * sizeof(*map->records));
        dest_idx += count;

        map->records_count = dest_idx;
    }

    ret = 0;
    if (append_count > 0)
        ret = mail_index_sync_appends(&ctx, appends, append_count);

    mail_transaction_log_get_head(index->log, &file_seq, &file_offset);

    ctx.hdr.messages_count = map->records_count;
    ctx.hdr.log_file_seq = file_seq;
    ctx.hdr.log_file_offset = file_offset;

    if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
        map->mmap_used_size = index->hdr->header_size +
            map->records_count * sizeof(struct mail_index_record);

        memcpy(map->mmap_base, &ctx.hdr, sizeof(ctx.hdr));
        if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
            mail_index_set_syscall_error(index, "msync()");
            ret = -1;
        }
    } else {
        map->hdr_copy = ctx.hdr;
        map->hdr = &map->hdr_copy;
    }

    mail_index_unlock(index, lock_id);
    return ret;
}