src/lib-index/mail-index-sync-update.c

	mail-index-sync-update.c revision 1032e5427bf10566098f3b3bb9110e2bc1227e85
/* Copyright (c) 2004-2008 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "ioloop.h"
#include "array.h"
#include "mmap-util.h"
#include "mail-index-modseq.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
#include "mail-transaction-log-private.h"

/* If we have less than this many bytes to sync from log file, don't bother
   reading the main index */
#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048

static void
mail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
                  struct mail_index_map *map, bool eol)
{
    uint32_t prev_seq;
    uoff_t prev_offset;

    mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
                           &prev_seq, &prev_offset);

    if (!eol) {
        if (prev_offset == ctx->ext_intro_end_offset &&
            prev_seq == ctx->ext_intro_seq) {
            /* previous transaction was an extension introduction.
               we probably came here from
               mail_index_sync_ext_reset(). if there are any more
               views which want to continue syncing it needs the
               intro. so back up a bit more.

               don't do this in case the last transaction in the
               log is the extension intro, so we don't keep trying
               to sync it over and over again. */
            prev_offset = ctx->ext_intro_offset;
        }
    } else {
        i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
        if (map->hdr.log_file_seq != prev_seq) {
            map->hdr.log_file_seq = prev_seq;
            map->hdr.log_file_tail_offset = 0;
        }
    }
    map->hdr.log_file_head_offset = prev_offset;
}

static void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
                    struct mail_index_map *map)
{
        struct mail_index_view *view = ctx->view;

    i_assert(view->map != map);

    mail_index_sync_update_log_offset(ctx, view->map, FALSE);
    mail_index_unmap(&view->map);
    view->map = map;

    if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
        view->index->map = map;
}

static void
mail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
{
    struct mail_index_map *map = ctx->view->map;

    i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) ||
         map->rec_map->lock_id != 0);

    if (map->refcount > 1) {
        map = mail_index_map_clone(map);
        mail_index_sync_replace_map(ctx, map);
    }

    if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
        mail_index_map_move_to_memory(ctx->view->map);
}

struct mail_index_map *
mail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
{
    mail_index_sync_move_to_private_memory(ctx);
    mail_index_record_map_move_to_private(ctx->view->map);
    ctx->view->map->write_atomic = TRUE;
    return ctx->view->map;
}

static int
mail_index_header_update_counts(struct mail_index_header *hdr,
                uint8_t old_flags, uint8_t new_flags,
                const char **error_r)
{
    if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
        /* different seen-flag */
        if ((old_flags & MAIL_SEEN) != 0) {
            if (hdr->seen_messages_count == 0) {
                *error_r = "Seen counter wrong";
                return -1;
            }
            hdr->seen_messages_count--;
        } else {
            if (hdr->seen_messages_count >= hdr->messages_count) {
                *error_r = "Seen counter wrong";
                return -1;
            }

            if (++hdr->seen_messages_count == hdr->messages_count)
                hdr->first_unseen_uid_lowwater = hdr->next_uid;
        }
    }

    if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
        /* different deleted-flag */
        if ((old_flags & MAIL_DELETED) == 0) {
            hdr->deleted_messages_count++;
            if (hdr->deleted_messages_count > hdr->messages_count) {
                *error_r = "Deleted counter wrong";
                return -1;
            }
        } else {
            if (hdr->deleted_messages_count == 0 ||
                hdr->deleted_messages_count > hdr->messages_count) {
                *error_r = "Deleted counter wrong";
                return -1;
            }

            if (--hdr->deleted_messages_count == 0)
                hdr->first_deleted_uid_lowwater = hdr->next_uid;
        }
    }
    return 0;
}

static void
mail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
                     uint32_t uid,
                     uint8_t old_flags, uint8_t new_flags)
{
    struct mail_index_map *const *maps;
    const char *error;
    unsigned int i, count;

    maps = array_get(&ctx->view->map->rec_map->maps, &count);
    for (i = 0; i < count; i++) {
        if (uid >= maps[i]->hdr.next_uid)
            continue;

        if (mail_index_header_update_counts(&maps[i]->hdr,
                            old_flags, new_flags,
                            &error) < 0)
            mail_index_sync_set_corrupted(ctx, "%s", error);
    }
}

static void
mail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
                     uint32_t uid, uint8_t old_flags,
                     uint8_t new_flags, bool all)
{
    const char *error;

    if (all) {
        mail_index_sync_header_update_counts_all(ctx, uid, old_flags,
                             new_flags);
    } else {
        i_assert(uid < ctx->view->map->hdr.next_uid);
        if (mail_index_header_update_counts(&ctx->view->map->hdr,
                            old_flags, new_flags,
                            &error) < 0)
            mail_index_sync_set_corrupted(ctx, "%s", error);
    }
}

static void
mail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
                   uint32_t uid, enum mail_flags flags)
{
    struct mail_index_map *const *maps;
    unsigned int i, count;

    maps = array_get(&ctx->view->map->rec_map->maps, &count);
    for (i = 0; i < count; i++) {
        if ((flags & MAIL_SEEN) == 0 &&
            uid < maps[i]->hdr.first_unseen_uid_lowwater)
            maps[i]->hdr.first_unseen_uid_lowwater = uid;
        if ((flags & MAIL_DELETED) != 0 &&
            uid < maps[i]->hdr.first_deleted_uid_lowwater)
            maps[i]->hdr.first_deleted_uid_lowwater = uid;
    }
}

static void
sync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
               uint32_t seq1, uint32_t seq2)
{
    const struct mail_index_expunge_handler *eh;
    struct mail_index_record *rec;
    unsigned int i, count;
    uint32_t seq;

    /* call expunge handlers only when syncing index file */
    if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE)
        return;

    if (!ctx->expunge_handlers_set)
        mail_index_sync_init_expunge_handlers(ctx);

    if (!array_is_created(&ctx->expunge_handlers))
        return;

    eh = array_get(&ctx->expunge_handlers, &count);
    for (i = 0; i < count; i++, eh++) {
        for (seq = seq1; seq <= seq2; seq++) {
            rec = MAIL_INDEX_MAP_IDX(ctx->view->map, seq-1);
            /* FIXME: does expunge handler's return value matter?
               we probably shouldn't disallow expunges if the
               handler returns failure.. should it be just changed
               to return void? */
            (void)eh->handler(ctx, seq,
                      PTR_OFFSET(rec, eh->record_offset),
                      eh->sync_context, eh->context);
        }
    }
}

static int
sync_expunge(const struct mail_transaction_expunge *e, unsigned int count,
         struct mail_index_sync_map_ctx *ctx)
{
    struct mail_index_map *map = ctx->view->map;
    struct mail_index_record *rec;
    uint32_t seq_count, seq, seq1, seq2;
    unsigned int i;

    for (i = 0; i < count; i++, e++) {
        if (!mail_index_lookup_seq_range(ctx->view, e->uid1, e->uid2,
                         &seq1, &seq2)) {
            /* everything expunged already */
            continue;
        }

        sync_expunge_call_handlers(ctx, seq1, seq2);

        map = mail_index_sync_get_atomic_map(ctx);
        for (seq = seq1; seq <= seq2; seq++) {
            rec = MAIL_INDEX_MAP_IDX(map, seq-1);
            mail_index_sync_header_update_counts(ctx, rec->uid,
                                 rec->flags, 0,
                                 FALSE);
        }

        /* @UNSAFE */
        memmove(MAIL_INDEX_MAP_IDX(map, seq1-1),
            MAIL_INDEX_MAP_IDX(map, seq2),
            (map->rec_map->records_count - seq2) *
            map->hdr.record_size);

        seq_count = seq2 - seq1 + 1;
        map->rec_map->records_count -= seq_count;
        map->hdr.messages_count -= seq_count;
        mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2);
    }
    return 1;
}

void mail_index_sync_write_seq_update(struct mail_index_sync_map_ctx *ctx,
                      uint32_t seq1, uint32_t seq2)
{
    struct mail_index_map *map = ctx->view->map;

    if (map->rec_map->write_seq_first == 0 ||
        map->rec_map->write_seq_first > seq1)
        map->rec_map->write_seq_first = seq1;
    if (map->rec_map->write_seq_last < seq2)
        map->rec_map->write_seq_last = seq2;
}

static int sync_append(const struct mail_index_record *rec,
               struct mail_index_sync_map_ctx *ctx)
{
    struct mail_index_view *view = ctx->view;
    struct mail_index_map *map = view->map;
    const struct mail_index_record *old_rec;
    enum mail_flags new_flags;
    void *dest;
    size_t append_pos;

    if (rec->uid < map->hdr.next_uid) {
        mail_index_sync_set_corrupted(ctx,
            "Append with UID %u, but next_uid = %u",
            rec->uid, map->hdr.next_uid);
        return -1;
    }

    /* move to memory. the mapping is written when unlocking so we don't
       waste time re-mmap()ing multiple times or waste space growing index
       file too large */
    mail_index_sync_move_to_private_memory(ctx);
    map = view->map;

    if (rec->uid <= map->rec_map->last_appended_uid) {
        i_assert(map->hdr.messages_count < map->rec_map->records_count);
        /* the flags may have changed since it was added to map.
           use the updated flags already, so flag counters won't get
           broken. */
        old_rec = MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count);
        i_assert(old_rec->uid == rec->uid);
        new_flags = old_rec->flags;
    } else {
        /* don't rely on buffer->used being at the correct position.
           at least expunges can move it */
        append_pos = map->rec_map->records_count * map->hdr.record_size;
        dest = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
                           map->hdr.record_size);
        map->rec_map->records =
            buffer_get_modifiable_data(map->rec_map->buffer, NULL);

        memcpy(dest, rec, sizeof(*rec));
        memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
               map->hdr.record_size - sizeof(*rec));
        map->rec_map->records_count++;
        map->rec_map->last_appended_uid = rec->uid;
        new_flags = rec->flags;

        mail_index_sync_write_seq_update(ctx,
                         map->rec_map->records_count,
                         map->rec_map->records_count);
        mail_index_modseq_append(ctx->modseq_ctx,
                     map->rec_map->records_count);
    }

    map->hdr.messages_count++;
    map->hdr.next_uid = rec->uid+1;

    if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
        map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;

    mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
    mail_index_sync_header_update_counts(ctx, rec->uid,
                         0, new_flags, FALSE);
    return 1;
}

static int sync_flag_update(const struct mail_transaction_flag_update *u,
                struct mail_index_sync_map_ctx *ctx)
{
    struct mail_index_view *view = ctx->view;
    struct mail_index_record *rec;
    uint8_t flag_mask, old_flags;
    uint32_t idx, seq1, seq2;

    if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2))
        return 1;

    mail_index_sync_write_seq_update(ctx, seq1, seq2);
    if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u)) {
        mail_index_modseq_update_flags(ctx->modseq_ctx,
                           u->add_flags | u->remove_flags,
                           seq1, seq2);
    }

    if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
        view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;

        flag_mask = ~u->remove_flags;

    if (((u->add_flags | u->remove_flags) &
         (MAIL_SEEN | MAIL_DELETED)) == 0) {
        /* we're not modifying any counted/lowwatered flags */
        for (idx = seq1-1; idx < seq2; idx++) {
            rec = MAIL_INDEX_MAP_IDX(view->map, idx);
            rec->flags = (rec->flags & flag_mask) | u->add_flags;
        }
    } else {
        for (idx = seq1-1; idx < seq2; idx++) {
            rec = MAIL_INDEX_MAP_IDX(view->map, idx);

            old_flags = rec->flags;
            rec->flags = (rec->flags & flag_mask) | u->add_flags;

            mail_index_header_update_lowwaters(ctx, rec->uid,
                               rec->flags);
            mail_index_sync_header_update_counts(ctx, rec->uid,
                                 old_flags,
                                 rec->flags, TRUE);
        }
    }
    return 1;
}

static int sync_header_update(const struct mail_transaction_header_update *u,
                  struct mail_index_sync_map_ctx *ctx)
{
#define MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, field) \
    ((u)->offset <= offsetof(struct mail_index_header, field) && \
     (u)->offset + (u)->size > offsetof(struct mail_index_header, field))
    struct mail_index_map *map = ctx->view->map;
    uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
    uint32_t orig_next_uid = map->hdr.next_uid;
    uint32_t orig_uid_validity = map->hdr.uid_validity;

    if (u->offset >= map->hdr.base_header_size ||
        u->offset + u->size > map->hdr.base_header_size) {
        mail_index_sync_set_corrupted(ctx,
            "Header update outside range: %u + %u > %u",
            u->offset, u->size, map->hdr.base_header_size);
        return -1;
    }

    buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
    map->hdr_base = map->hdr_copy_buf->data;
    map->write_base_header = TRUE;

    /* @UNSAFE */
    if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
        memcpy(PTR_OFFSET(&map->hdr, u->offset),
               u + 1, u->size);
    } else if (u->offset < sizeof(map->hdr)) {
        memcpy(PTR_OFFSET(&map->hdr, u->offset),
               u + 1, sizeof(map->hdr) - u->offset);
    }

    /* UIDVALIDITY can be changed only by resetting the index */
    if (orig_uid_validity != 0 &&
        MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, uid_validity)) {
        mail_index_sync_set_corrupted(ctx,
            "uid_validity updated unexpectedly: %u -> %u",
            orig_uid_validity, map->hdr.uid_validity);
        /* let it through anyway, although this could give wrong
           "next_uid shrank" errors if the value actually changed.. */
    }
    if (map->hdr.next_uid < orig_next_uid) {
        mail_index_sync_set_corrupted(ctx,
            "next_uid shrank ignored: %u -> %u",
            orig_next_uid, map->hdr.next_uid);
        map->hdr.next_uid = orig_next_uid;
        return -1;
    }

    /* the tail offset updates are intended for internal transaction
       log handling. we'll update the offset in the header only when
       the sync is finished. */
    map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
    return 1;
}

int mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
               const struct mail_transaction_header *hdr,
               const void *data)
{
    int ret = 0;

    switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
    case MAIL_TRANSACTION_APPEND: {
        const struct mail_index_record *rec, *end;

        end = CONST_PTR_OFFSET(data, hdr->size);
        for (rec = data; rec < end; rec++) {
            ret = sync_append(rec, ctx);
            if (ret <= 0)
                break;
        }
        break;
    }
    case MAIL_TRANSACTION_EXPUNGE:
    case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
        const struct mail_transaction_expunge *rec = data, *end;

        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
            /* this is simply a request for expunge */
            break;
        }
        end = CONST_PTR_OFFSET(data, hdr->size);
        ret = sync_expunge(rec, end - rec, ctx);
        break;
    }
    case MAIL_TRANSACTION_FLAG_UPDATE: {
        const struct mail_transaction_flag_update *rec, *end;

        end = CONST_PTR_OFFSET(data, hdr->size);
        for (rec = data; rec < end; rec++) {
            ret = sync_flag_update(rec, ctx);
            if (ret <= 0)
                break;
        }
        break;
    }
    case MAIL_TRANSACTION_HEADER_UPDATE: {
        const struct mail_transaction_header_update *rec;
        unsigned int i;

        for (i = 0; i < hdr->size; ) {
            rec = CONST_PTR_OFFSET(data, i);
            ret = sync_header_update(rec, ctx);
            if (ret <= 0)
                break;

            i += sizeof(*rec) + rec->size;
            if ((i % 4) != 0)
                i += 4 - (i % 4);
        }
        break;
    }
    case MAIL_TRANSACTION_EXT_INTRO: {
        const struct mail_transaction_ext_intro *rec = data;
        unsigned int i;
        uint32_t prev_seq;
        uoff_t prev_offset;

        mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
                               &prev_seq, &prev_offset);
        ctx->ext_intro_seq = prev_seq;
        ctx->ext_intro_offset = prev_offset;
        ctx->ext_intro_end_offset =
            prev_offset + hdr->size + sizeof(*hdr);

        for (i = 0; i < hdr->size; ) {
            if (i + sizeof(*rec) > hdr->size) {
                /* should be just extra padding */
                break;
            }

            rec = CONST_PTR_OFFSET(data, i);
            if (i + sizeof(*rec) + rec->name_size > hdr->size) {
                mail_index_sync_set_corrupted(ctx,
                    "ext intro: name_size too large");
                ret = -1;
                break;
            }

            ret = mail_index_sync_ext_intro(ctx, rec);
            if (ret <= 0)
                break;

            i += sizeof(*rec) + rec->name_size;
            if ((i % 4) != 0)
                i += 4 - (i % 4);
        }
        break;
    }
    case MAIL_TRANSACTION_EXT_RESET: {
        struct mail_transaction_ext_reset rec;

        /* old versions have only new_reset_id */
        if (hdr->size < sizeof(uint32_t)) {
            mail_index_sync_set_corrupted(ctx,
                "ext reset: invalid record size");
            ret = -1;
            break;
        }
        memcpy(&rec, data, I_MIN(hdr->size, sizeof(rec)));
        ret = mail_index_sync_ext_reset(ctx, &rec);
        break;
    }
    case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
        const struct mail_transaction_ext_hdr_update *rec = data;
        unsigned int i;

        for (i = 0; i < hdr->size; ) {
            rec = CONST_PTR_OFFSET(data, i);

            if (i + sizeof(*rec) > hdr->size ||
                i + sizeof(*rec) + rec->size > hdr->size) {
                mail_index_sync_set_corrupted(ctx,
                    "ext hdr update: invalid record size");
                ret = -1;
                break;
            }

            ret = mail_index_sync_ext_hdr_update(ctx, rec);
            if (ret <= 0)
                break;

            i += sizeof(*rec) + rec->size;
            if ((i % 4) != 0)
                i += 4 - (i % 4);
        }
        break;
    }
    case MAIL_TRANSACTION_EXT_REC_UPDATE: {
        const struct mail_transaction_ext_rec_update *rec;
        const struct mail_index_ext *ext;
        unsigned int i, record_size;

        if (ctx->cur_ext_map_idx == (uint32_t)-1) {
            mail_index_sync_set_corrupted(ctx,
                "Extension record updated "
                "without intro prefix");
            ret = -1;
            break;
        }

        if (ctx->cur_ext_ignore) {
            ret = 1;
            break;
        }

        ext = array_idx(&ctx->view->map->extensions,
                ctx->cur_ext_map_idx);
        /* the record is padded to 32bits in the transaction log */
        record_size = (sizeof(*rec) + ext->record_size + 3) & ~3;

        for (i = 0; i < hdr->size; i += record_size) {
            rec = CONST_PTR_OFFSET(data, i);

            if (i + record_size > hdr->size) {
                mail_index_sync_set_corrupted(ctx,
                    "ext rec update: invalid record size");
                ret = -1;
                break;
            }

            ret = mail_index_sync_ext_rec_update(ctx, rec);
            if (ret <= 0)
                break;
        }
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_UPDATE: {
        const struct mail_transaction_keyword_update *rec = data;

        ret = mail_index_sync_keywords(ctx, hdr, rec);
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_RESET: {
        const struct mail_transaction_keyword_reset *rec = data;

        ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
        break;
    }
    default:
        mail_index_sync_set_corrupted(ctx,
            "Unknown transaction record type 0x%x",
            (hdr->type & MAIL_TRANSACTION_TYPE_MASK));
        ret = -1;
        break;
    }
    return ret;
}

void mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
                  struct mail_index_view *view,
                  enum mail_index_sync_handler_type type)
{
    memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
    sync_map_ctx->view = view;
    sync_map_ctx->cur_ext_map_idx = (uint32_t)-1;
    sync_map_ctx->type = type;
    sync_map_ctx->modseq_ctx = mail_index_modseq_sync_begin(sync_map_ctx);

    mail_index_sync_init_handlers(sync_map_ctx);
}

void mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
{
    i_assert(sync_map_ctx->modseq_ctx == NULL);

    if (sync_map_ctx->unknown_extensions != NULL)
        buffer_free(&sync_map_ctx->unknown_extensions);
    if (sync_map_ctx->expunge_handlers_used)
        mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
    mail_index_sync_deinit_handlers(sync_map_ctx);
}

static void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
{
    const struct mail_index_record *rec;
    unsigned int i;

    if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0)
        return;

    /* do we have dirty flags anymore? */
    for (i = 0; i < map->rec_map->records_count; i++) {
        rec = MAIL_INDEX_MAP_IDX(map, i);
        if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
            map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
            break;
        }
    }
}

#ifdef DEBUG
void mail_index_map_check(struct mail_index_map *map)
{
    const struct mail_index_header *hdr = &map->hdr;
    unsigned int i, del = 0, seen = 0;
    uint32_t prev_uid = 0;

    i_assert(hdr->messages_count <= map->rec_map->records_count);
    for (i = 0; i < hdr->messages_count; i++) {
        const struct mail_index_record *rec;

        rec = MAIL_INDEX_MAP_IDX(map, i);
        i_assert(rec->uid > prev_uid);
        prev_uid = rec->uid;

        if (rec->flags & MAIL_DELETED) {
            i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
            del++;
        }
        if (rec->flags & MAIL_SEEN)
            seen++;
        else
            i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
    }
    i_assert(del == hdr->deleted_messages_count);
    i_assert(seen == hdr->seen_messages_count);
}
#endif

int mail_index_sync_map(struct mail_index_map **_map,
            enum mail_index_sync_handler_type type, bool force)
{
    struct mail_index_map *map = *_map;
    struct mail_index *index = map->index;
    struct mail_index_view *view;
    struct mail_index_sync_map_ctx sync_map_ctx;
    const struct mail_transaction_header *thdr;
    const void *tdata;
    uint32_t prev_seq;
    uoff_t start_offset, prev_offset;
    int ret;
    bool had_dirty, reset;

    i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);

    if (index->log->head == NULL) {
        i_assert(!force);
        return 0;
    }

    start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
        map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
    if (!force && !index->mmap_disable) {
        /* see if we'd prefer to reopen the index file instead of
           syncing the current map from the transaction log.
           don't check this if mmap is disabled, because reopening
           index causes sync to get lost. */
        uoff_t log_size, index_size;

        if (index->fd == -1 &&
            index->log->head->hdr.prev_file_seq != 0) {
            /* we don't know the index's size, so use the
               smallest index size we're willing to read */
            index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
        } else {
            index_size = map->hdr.header_size +
                map->rec_map->records_count *
                map->hdr.record_size;
        }

        /* this isn't necessary correct currently, but it should be
           close enough */
        log_size = index->log->head->last_size;
        if (log_size > start_offset &&
            log_size - start_offset > index_size)
            return 0;
    }

    view = mail_index_view_open_with_map(index, map);
    ret = mail_transaction_log_view_set(view->log_view,
                        map->hdr.log_file_seq, start_offset,
                        (uint32_t)-1, (uoff_t)-1, &reset);
    if (ret <= 0) {
        if (force && ret == 0) {
            /* the seq/offset is probably broken */
            (void)mail_index_fsck(index);
        }
        /* can't use it. sync by re-reading index. */
        mail_index_view_close(&view);
        return 0;
    }

    /* view referenced the map. avoid unnecessary map cloning by
       unreferencing the map while view exists. */
    map->refcount--;

    had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
    if (had_dirty) {
        map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
        map->write_base_header = TRUE;
    }

    if (map->hdr_base != map->hdr_copy_buf->data) {
        /* if syncing updates the header, it updates hdr_copy_buf
           and updates hdr_base to hdr_copy_buf. so the buffer must
           initially contain a valid header or we'll break it when
           writing it. */
        buffer_reset(map->hdr_copy_buf);
        buffer_append(map->hdr_copy_buf, map->hdr_base,
                  map->hdr.header_size);
        map->hdr_base = map->hdr_copy_buf->data;
    }

    mail_transaction_log_view_get_prev_pos(view->log_view,
                           &prev_seq, &prev_offset);

    mail_index_sync_map_init(&sync_map_ctx, view, type);
    if (reset) {
        /* Reset the entire index. Leave only indexid and
           log_file_seq. */
        mail_transaction_log_view_get_prev_pos(view->log_view,
                               &prev_seq, &prev_offset);
        map = mail_index_map_alloc(index);
        map->hdr.log_file_seq = prev_seq;
        map->hdr.log_file_tail_offset = 0;
        mail_index_sync_replace_map(&sync_map_ctx, map);
    }
    map = NULL;

    /* FIXME: when transaction sync lock is removed, we'll need to handle
       the case when a transaction is committed while mailbox is being
       synced ([synced transactions][new transaction][ext transaction]).
       this means int_offset contains [synced] and ext_offset contains
       all */
    while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
                             &tdata)) > 0) {
        mail_transaction_log_view_get_prev_pos(view->log_view,
                               &prev_seq, &prev_offset);

        if (LOG_IS_BEFORE(prev_seq, prev_offset,
                  view->map->hdr.log_file_seq,
                  view->map->hdr.log_file_head_offset)) {
            /* this has been synced already. we're here only to call
               expunge handlers and extension update handlers. */
            i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);

            if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
                continue;
            if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
                continue;
        }

        /* we'll just skip over broken entries */
        T_BEGIN {
            (void)mail_index_sync_record(&sync_map_ctx,
                             thdr, tdata);
        } T_END;
    }
    map = view->map;

    if (had_dirty)
        mail_index_sync_update_hdr_dirty_flag(map);
    mail_index_modseq_sync_end(&sync_map_ctx.modseq_ctx);

    mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);

#ifdef DEBUG
    mail_index_map_check(map);
#endif
    i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0);

    /* transaction log tracks internally the current tail offset.
       besides using header updates, it also updates the offset to skip
       over following external transactions to avoid extra unneeded log
       reading. */
    i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq);
    if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) {
        map->hdr.log_file_tail_offset =
            index->log->head->max_tail_offset;
    }

    buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
    if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
        memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
               map->hdr_copy_buf->used);
    }

    /* restore refcount before closing the view. this is necessary also
       if map got cloned, because view closing would otherwise destroy it */
    map->refcount++;
    mail_index_sync_map_deinit(&sync_map_ctx);
    mail_index_view_close(&view);

    i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);

    if (mail_index_map_check_header(map) <= 0) {
        mail_index_set_error(index,
            "Synchronization corrupted index header: %s",
            index->filepath);
        (void)mail_index_fsck(index);
        map = index->map;
    } else if (sync_map_ctx.errors) {
        /* make sure the index looks valid now */
        (void)mail_index_fsck(index);
        map = index->map;
    }

    *_map = map;
    return ret < 0 ? -1 : 1;
}