src/lib-index/mail-index-sync-update.c

	mail-index-sync-update.c revision 287ba82a8da3eaa473b5735d4eeac2fb4c5d8117
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco/* Copyright (C) 2004 Timo Sirainen */
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "lib.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "ioloop.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "buffer.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "file-set-size.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "mmap-util.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "mail-index-view-private.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "mail-index-sync-private.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "mail-transaction-log.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco#include "mail-transaction-util.h"
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Koscovoid mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco                 struct mail_index_map *map)
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco{
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco        struct mail_index_view *view = ctx->view;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    /* if map still exists after this, it's only in views. */
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    view->map->write_to_disk = FALSE;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen    mail_index_unmap(view->index, view->map);
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    view->map = map;
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen    view->map->refcount++;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    mail_index_unmap(view->index, view->index->map);
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    view->index->map = map;
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen    view->index->hdr = &map->hdr;
bfc104f8240fac5b59d4347db4c27be705f6dfc2Knut Anders Hatlen
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen    if (ctx->type == MAIL_INDEX_SYNC_HANDLER_INDEX)
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes        map->write_to_disk = TRUE;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco}
6d7c6f82e644c205bc679ee5b1fa2929ec949963Lubos Kosco
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Koscostatic void
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Koscomail_index_header_update_counts(struct mail_index_header *hdr,
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco                uint8_t old_flags, uint8_t new_flags)
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco{
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    if (((old_flags ^ new_flags) & MAIL_RECENT) != 0) {
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco        /* different recent-flag */
6d7c6f82e644c205bc679ee5b1fa2929ec949963Lubos Kosco        if ((old_flags & MAIL_RECENT) == 0)
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco            hdr->recent_messages_count++;
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco        else if (--hdr->recent_messages_count == 0)
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco            hdr->first_recent_uid_lowwater = hdr->next_uid;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    }
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco        /* different seen-flag */
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco        if ((old_flags & MAIL_SEEN) != 0)
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco            hdr->seen_messages_count--;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco        else if (++hdr->seen_messages_count == hdr->messages_count)
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes            hdr->first_unseen_uid_lowwater = hdr->next_uid;
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes    }
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco
bfc104f8240fac5b59d4347db4c27be705f6dfc2Knut Anders Hatlen    if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen        /* different deleted-flag */
bfc104f8240fac5b59d4347db4c27be705f6dfc2Knut Anders Hatlen        if ((old_flags & MAIL_DELETED) == 0)
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes            hdr->deleted_messages_count++;
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes        else if (--hdr->deleted_messages_count == 0)
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes            hdr->first_deleted_uid_lowwater = hdr->next_uid;
8dc938f5c6296cffbadd8e06925dc0762dd79c5aLubos Kosco    }
8dc938f5c6296cffbadd8e06925dc0762dd79c5aLubos Kosco}
8dc938f5c6296cffbadd8e06925dc0762dd79c5aLubos Kosco
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlenstatic void
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlenmail_index_header_update_lowwaters(struct mail_index_header *hdr,
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen                   const struct mail_index_record *rec)
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen{
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen    if ((rec->flags & MAIL_RECENT) != 0 &&
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen        rec->uid < hdr->first_recent_uid_lowwater)
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen        hdr->first_recent_uid_lowwater = rec->uid;
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen    if ((rec->flags & MAIL_SEEN) == 0 &&
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen        rec->uid < hdr->first_unseen_uid_lowwater)
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen        hdr->first_unseen_uid_lowwater = rec->uid;
4750e1be4c1b2ba11705d5b73b86dd1b9dd4e1acKnut Anders Hatlen    if ((rec->flags & MAIL_DELETED) != 0 &&
bfc104f8240fac5b59d4347db4c27be705f6dfc2Knut Anders Hatlen        rec->uid < hdr->first_deleted_uid_lowwater)
bfc104f8240fac5b59d4347db4c27be705f6dfc2Knut Anders Hatlen        hdr->first_deleted_uid_lowwater = rec->uid;
bfc104f8240fac5b59d4347db4c27be705f6dfc2Knut Anders Hatlen}
bfc104f8240fac5b59d4347db4c27be705f6dfc2Knut Anders Hatlen
bfc104f8240fac5b59d4347db4c27be705f6dfc2Knut Anders Hatlenstatic int sync_expunge(const struct mail_transaction_expunge *e,
6d7c6f82e644c205bc679ee5b1fa2929ec949963Lubos Kosco            struct mail_index_sync_map_ctx *ctx)
ff5eba819da0cf7964d884630fb13262ef12c505Trond Norbye{
50203181d8c071cfd69cb197b5da0eda5c4d6372Jens Elkner    struct mail_index_view *view = ctx->view;
c6e0f8b39af7343c04ec7558a085c965159f4ea0Lubos Kosco    struct mail_index_map *map = view->map;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    struct mail_index_record *rec;
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco    uint32_t count, seq, seq1, seq2;
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco        const struct mail_index_expunge_handler *expunge_handlers, *eh;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    unsigned int i, expunge_handlers_count;
ff5eba819da0cf7964d884630fb13262ef12c505Trond Norbye
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    if (e->uid1 > e->uid2 || e->uid1 == 0) {
b310c0aeaec52a0246021104b8f52cbb31b68480Lubos Kosco        mail_transaction_log_view_set_corrupted(ctx->view->log_view,
ff5eba819da0cf7964d884630fb13262ef12c505Trond Norbye                "Invalid UID range in expunge (%u .. %u)",
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco                e->uid1, e->uid2);
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco        return -1;
817883e9f0d419428e8236a09b77cdeeaa034df7Knut Anders Hatlen    }
be5cdf850da5383468637c6937c016f26bd339cfLubos Kosco
6d7c6f82e644c205bc679ee5b1fa2929ec949963Lubos Kosco    if (!view->map->write_to_disk || view->map->refcount != 1) {
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco        /* expunges have to be atomic. so we'll have to copy
6d7c6f82e644c205bc679ee5b1fa2929ec949963Lubos Kosco           the mapping, do the changes there and then finally
6d7c6f82e644c205bc679ee5b1fa2929ec949963Lubos Kosco           replace the whole index file. to avoid extra disk
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco           I/O we copy the index into memory rather than to
b645988bdc1cf4f2f82b8c00ed041ddddd822c24Lubos Kosco           temporary file */
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes        map = mail_index_map_clone(map, map->hdr.record_size);
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes        mail_index_sync_replace_map(ctx, map);
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes    }
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes    i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map));
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes    if (mail_index_lookup_uid_range(view, e->uid1, e->uid2,
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes                    &seq1, &seq2) < 0)
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes        return -1;
b6ada49236374a1f4ae462c384d69bab7e6f0a1dGustavo Lopes
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco    if (seq1 == 0)
        return 1;

    /* don't call expunge handlers if we're syncing view */
    if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW &&
        !ctx->expunge_handlers_set)
        mail_index_sync_init_expunge_handlers(ctx);

    if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW &&
        array_is_created(&ctx->expunge_handlers)) {
        expunge_handlers = array_get(&ctx->expunge_handlers,
                         &expunge_handlers_count);
    } else {
        expunge_handlers = NULL;
        expunge_handlers_count = 0;
    }

    for (seq = seq1; seq <= seq2; seq++) {
                rec = MAIL_INDEX_MAP_IDX(map, seq-1);
        mail_index_header_update_counts(&map->hdr, rec->flags, 0);
    }

    for (i = 0; i < expunge_handlers_count; i++) {
        eh = &expunge_handlers[i];

        for (seq = seq1; seq <= seq2; seq++) {
            rec = MAIL_INDEX_MAP_IDX(map, seq-1);
            eh->handler(ctx, seq,
                    PTR_OFFSET(rec, eh->record_offset),
                    eh->context);
        }
    }

    /* @UNSAFE */
    count = seq2 - seq1 + 1;
    memmove(MAIL_INDEX_MAP_IDX(map, seq1-1), MAIL_INDEX_MAP_IDX(map, seq2),
        (map->records_count - seq2) * map->hdr.record_size);

    map->records_count -= count;
    map->hdr.messages_count -= count;
    view->hdr.messages_count -= count;

    if (map->buffer != NULL) {
        buffer_set_used_size(map->buffer, map->records_count *
                     map->hdr.record_size);
        map->records = buffer_get_modifyable_data(map->buffer, NULL);
    }
    return 1;
}

static int sync_append(const struct mail_index_record *rec,
               struct mail_index_sync_map_ctx *ctx)
{
    struct mail_index_view *view = ctx->view;
    struct mail_index_map *map = view->map;
    void *dest;

    if (rec->uid < map->hdr.next_uid) {
        mail_transaction_log_view_set_corrupted(view->log_view,
            "Append with UID %u, but next_uid = %u",
            rec->uid, map->hdr.next_uid);
        return -1;
    }

    if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
        i_assert(map->records_count * map->hdr.record_size ==
             buffer_get_used_size(map->buffer));
        dest = buffer_append_space_unsafe(map->buffer,
                          map->hdr.record_size);
        map->records = buffer_get_modifyable_data(map->buffer, NULL);
    } else {
        i_assert((map->records_count+1) * map->hdr.record_size <=
             map->mmap_size);
        dest = MAIL_INDEX_MAP_IDX(map, map->records_count);
    }
    memcpy(dest, rec, sizeof(*rec));
    memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
           map->hdr.record_size - sizeof(*rec));

    map->hdr.messages_count++;
    map->hdr.next_uid = rec->uid+1;
    map->records_count++;
    view->hdr.messages_count++;

    if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
        map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;

    mail_index_header_update_counts(&map->hdr, 0, rec->flags);
    mail_index_header_update_lowwaters(&map->hdr, rec);
    return 1;
}

static int sync_flag_update(const struct mail_transaction_flag_update *u,
                struct mail_index_sync_map_ctx *ctx)
{
    struct mail_index_view *view = ctx->view;
    struct mail_index_header *hdr;
    struct mail_index_record *rec;
    uint8_t flag_mask, old_flags;
    uint32_t idx, seq1, seq2;

    if (u->uid1 > u->uid2 || u->uid1 == 0) {
        mail_transaction_log_view_set_corrupted(ctx->view->log_view,
                "Invalid UID range in flag update (%u .. %u)",
                u->uid1, u->uid2);
        return -1;
    }

    if (mail_index_lookup_uid_range(view, u->uid1, u->uid2,
                    &seq1, &seq2) < 0)
        return -1;

    if (seq1 == 0)
        return 1;

    hdr = &view->map->hdr;
    if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
        hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;

        flag_mask = ~u->remove_flags;

    for (idx = seq1-1; idx < seq2; idx++) {
                rec = MAIL_INDEX_MAP_IDX(view->map, idx);

        old_flags = rec->flags;
        rec->flags = (rec->flags & flag_mask) | u->add_flags;

        mail_index_header_update_counts(hdr, old_flags, rec->flags);
                mail_index_header_update_lowwaters(hdr, rec);
    }
    return 1;
}

static int sync_header_update(const struct mail_transaction_header_update *u,
                  struct mail_index_sync_map_ctx *ctx)
{
    struct mail_index_map *map = ctx->view->map;

    if (u->offset >= map->hdr.base_header_size ||
        u->offset + u->size > map->hdr.base_header_size) {
        mail_transaction_log_view_set_corrupted(ctx->view->log_view,
            "Header update outside range: %u + %u > %u",
            u->offset, u->size, map->hdr.base_header_size);
        return -1;
    }

    buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
    map->hdr_base = map->hdr_copy_buf->data;

    /* @UNSAFE */
    if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
        memcpy(PTR_OFFSET(&map->hdr, u->offset),
               u + 1, u->size);
    } else if (u->offset < sizeof(map->hdr)) {
        memcpy(PTR_OFFSET(&map->hdr, u->offset),
               u + 1, sizeof(map->hdr) - u->offset);
    }
    return 1;
}

static int mail_index_grow(struct mail_index *index, struct mail_index_map *map,
               unsigned int count)
{
    void *hdr_copy;
    size_t size, hdr_copy_size;

    if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
        return 0;

    i_assert(map == index->map);

    size = map->hdr.header_size +
        (map->records_count + count) * map->hdr.record_size;
    if (size <= map->mmap_size)
        return 0;

    /* when we grow fast, do it exponentially */
    if (count < index->last_grow_count)
        count = index->last_grow_count;
    if (count < MAIL_INDEX_MAX_POWER_GROW)
        count = nearest_power(count);
    index->last_grow_count = count;

    size = map->hdr.header_size +
        (map->records_count + count) * map->hdr.record_size;
    if (file_set_size(index->fd, (off_t)size) < 0)
        return mail_index_set_syscall_error(index, "file_set_size()");

    /* we only wish to grow the file, but mail_index_map() updates the
       headers as well and may break our modified hdr_copy. so, take
       a backup of it and put it back afterwards */
    t_push();
    i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
        hdr_copy_size = map->hdr_copy_buf->used;
    hdr_copy = t_malloc(hdr_copy_size);
    memcpy(hdr_copy, map->hdr_copy_buf->data, hdr_copy_size);
    memcpy(hdr_copy, &map->hdr, sizeof(map->hdr));

    if (mail_index_map(index, TRUE) <= 0) {
        t_pop();
        return -1;
    }

    map = index->map;
    buffer_reset(map->hdr_copy_buf);
    buffer_append(map->hdr_copy_buf, hdr_copy, hdr_copy_size);

    map->hdr_base = map->hdr_copy_buf->data;
    memcpy(&map->hdr, hdr_copy, sizeof(map->hdr));
    map->records_count = map->hdr.messages_count;

    i_assert(map->mmap_size >= size);
    t_pop();
    return 0;
}

static void
mail_index_update_day_headers(struct mail_index_header *hdr, uint32_t uid)
{
    const int max_days =
        sizeof(hdr->day_first_uid) / sizeof(hdr->day_first_uid[0]);
    struct tm tm;
    time_t stamp;
    int i, days;

    /* get beginning of today */
    tm = *localtime(&ioloop_time);
    tm.tm_hour = 0;
    tm.tm_min = 0;
    tm.tm_sec = 0;
    stamp = mktime(&tm);
    if (stamp == (time_t)-1)
        i_panic("mktime(today) failed");

    if ((time_t)hdr->day_stamp >= stamp)
        return;

    /* get number of days since last message */
    days = (stamp - hdr->day_stamp) / (3600*24);
    if (days > max_days)
        days = max_days;

    /* @UNSAFE: move days forward and fill the missing days with old
       day_first_uid[0]. */
    memcpy(hdr->day_first_uid + days,
           hdr->day_first_uid, max_days - days);
    for (i = 1; i < days; i++)
        hdr->day_first_uid[i] = hdr->day_first_uid[0];

    hdr->day_stamp = stamp;
    hdr->day_first_uid[0] = uid;
}

int mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
               const struct mail_transaction_header *hdr,
               const void *data)
{
    int ret = 0;

    t_push();
    switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
    case MAIL_TRANSACTION_APPEND: {
        const struct mail_index_record *rec, *end;

        end = CONST_PTR_OFFSET(data, hdr->size);
        for (rec = data; rec < end; rec++) {
            ret = sync_append(rec, ctx);
            if (ret <= 0)
                break;
        }
        break;
    }
    case MAIL_TRANSACTION_EXPUNGE:
    case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
        const struct mail_transaction_expunge *rec, *end;

        end = CONST_PTR_OFFSET(data, hdr->size);
        for (rec = data; rec < end; rec++) {
            ret = sync_expunge(rec, ctx);
            if (ret <= 0)
                break;
        }
        break;
    }
    case MAIL_TRANSACTION_FLAG_UPDATE: {
        const struct mail_transaction_flag_update *rec, *end;

        end = CONST_PTR_OFFSET(data, hdr->size);
        for (rec = data; rec < end; rec++) {
            ret = sync_flag_update(rec, ctx);
            if (ret <= 0)
                break;
        }
        break;
    }
    case MAIL_TRANSACTION_HEADER_UPDATE: {
        const struct mail_transaction_header_update *rec;
        unsigned int i;

        for (i = 0; i < hdr->size; ) {
            rec = CONST_PTR_OFFSET(data, i);
            ret = sync_header_update(rec, ctx);
            if (ret <= 0)
                break;

            i += sizeof(*rec) + rec->size;
            if ((i % 4) != 0)
                i += 4 - (i % 4);
        }
        break;
    }
    case MAIL_TRANSACTION_EXT_INTRO: {
        const struct mail_transaction_ext_intro *rec = data;
        unsigned int i;

        for (i = 0; i < hdr->size; ) {
            if (i + sizeof(*rec) > hdr->size) {
                /* should be just extra padding */
                break;
            }

            rec = CONST_PTR_OFFSET(data, i);
            ret = mail_index_sync_ext_intro(ctx, rec);
            if (ret <= 0)
                break;

            i += sizeof(*rec) + rec->name_size;
            if ((i % 4) != 0)
                i += 4 - (i % 4);
        }
        break;
    }
    case MAIL_TRANSACTION_EXT_RESET: {
        const struct mail_transaction_ext_reset *rec = data;
        ret = mail_index_sync_ext_reset(ctx, rec);
        break;
    }
    case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
        const struct mail_transaction_ext_hdr_update *rec = data;
        unsigned int i;

        for (i = 0; i < hdr->size; ) {
            rec = CONST_PTR_OFFSET(data, i);
            ret = mail_index_sync_ext_hdr_update(ctx, rec);
            if (ret <= 0)
                break;

            i += sizeof(*rec) + rec->size;
            if ((i % 4) != 0)
                i += 4 - (i % 4);
        }
        break;
    }
    case MAIL_TRANSACTION_EXT_REC_UPDATE: {
        const struct mail_transaction_ext_rec_update *rec, *end;
        const struct mail_index_ext *ext;
        unsigned int record_size;

        if (ctx->cur_ext_id == (uint32_t)-1) {
            mail_transaction_log_view_set_corrupted(
                ctx->view->log_view,
                "Extension record update update "
                "without intro prefix");
            ret = -1;
            break;
        }

        if (ctx->cur_ext_ignore) {
            ret = 1;
            break;
        }

        ext = array_idx(&ctx->view->map->extensions, ctx->cur_ext_id);
        record_size = sizeof(*rec) + ext->record_size;

        rec = data;
        end = CONST_PTR_OFFSET(data, hdr->size);
        while (rec < end) {
            ret = mail_index_sync_ext_rec_update(ctx, rec);
            if (ret <= 0)
                break;

            rec = CONST_PTR_OFFSET(rec, record_size);
        }
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_UPDATE: {
        const struct mail_transaction_keyword_update *rec = data;

        ret = mail_index_sync_keywords(ctx, hdr, rec);
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_RESET: {
        const struct mail_transaction_keyword_reset *rec = data;

        ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
        break;
    }
    default:
        i_unreached();
    }
    t_pop();

    i_assert(ctx->view->map->records_count ==
         ctx->view->map->hdr.messages_count);
    return ret;
}

void mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
                  struct mail_index_view *view,
                  enum mail_index_sync_handler_type type)
{
    memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
    sync_map_ctx->view = view;
    sync_map_ctx->cur_ext_id = (uint32_t)-1;
    sync_map_ctx->type = type;

    mail_index_sync_init_handlers(sync_map_ctx);
}

void mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
{
    if (sync_map_ctx->expunge_handlers_used)
        mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
    mail_index_sync_deinit_handlers(sync_map_ctx);
}

static void mail_index_sync_remove_recent(struct mail_index_sync_ctx *sync_ctx)
{
    struct mail_index_map *map = sync_ctx->view->map;
    struct mail_index_record *rec;
    unsigned int i;

    for (i = 0; i < map->records_count; i++) {
        rec = MAIL_INDEX_MAP_IDX(map, i);
        if ((rec->flags & MAIL_RECENT) != 0)
            rec->flags &= ~MAIL_RECENT;
    }
}

int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
                 int sync_only_external)
{
    struct mail_index *index = sync_ctx->index;
    struct mail_index_view *view = sync_ctx->view;
    struct mail_index_map *map;
        struct mail_index_sync_map_ctx sync_map_ctx;
    const struct mail_transaction_header *thdr;
    const void *data;
    unsigned int count, old_lock_id;
    uint32_t seq, i, first_append_uid;
    uoff_t offset;
    int ret, had_dirty, skipped, check_ext_offsets;

    mail_index_sync_map_init(&sync_map_ctx, view,
                 MAIL_INDEX_SYNC_HANDLER_INDEX);

    /* we'll have to update view->lock_id to avoid mail_index_view_lock()
       trying to update the file later. */
    old_lock_id = view->lock_id;
    if (mail_index_lock_exclusive(index, &view->lock_id) < 0)
        return -1;
    mail_index_unlock(index, old_lock_id);

    /* NOTE: locking may change index->map so make sure the assignment is
       after locking */
    map = index->map;
    if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
        map->write_to_disk = TRUE;

    if (map->hdr_base != map->hdr_copy_buf->data) {
        buffer_reset(map->hdr_copy_buf);
        buffer_append(map->hdr_copy_buf, map->hdr_base,
                  map->hdr.header_size);
        map->hdr_base = map->hdr_copy_buf->data;
    }
    i_assert(map->hdr.base_header_size >= sizeof(map->hdr));

    mail_index_unmap(index, view->map);
    view->map = map;
    view->map->refcount++;

    had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
    if (had_dirty)
        map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;

    if (sync_ctx->sync_recent) {
        /* mark all messages non-recent */
        mail_index_sync_remove_recent(sync_ctx);
    }

    /* make sure we don't go doing fsck while modifying the index */
    index->sync_update = TRUE;

    first_append_uid = 0;
    check_ext_offsets = TRUE;
    while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
                             &data, &skipped)) > 0) {
        if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
            if (sync_only_external) {
                /* we're syncing only external changes. */
                continue;
            }
        } else if (check_ext_offsets) {
            uint32_t prev_seq;
            uoff_t prev_offset;

            mail_transaction_log_view_get_prev_pos(view->log_view,
                                   &prev_seq,
                                   &prev_offset);
            if (prev_seq < view->map->hdr.log_file_seq ||
                (prev_seq == view->map->hdr.log_file_seq &&
                 prev_offset <
                 view->map->hdr.log_file_ext_offset)) {
                /* we have already synced this change */
                continue;
            }
            check_ext_offsets = FALSE;
        }

        if ((thdr->type & MAIL_TRANSACTION_APPEND) != 0) {
            const struct mail_index_record *rec = data;

            if (first_append_uid == 0)
                first_append_uid = rec->uid;

            map = view->map;
            count = thdr->size / sizeof(*rec);
            if (mail_index_grow(index, map, count) < 0) {
                ret = -1;
                break;
            }
            if (map != index->map) {
                map = index->map;
                mail_index_unmap(view->index, view->map);
                view->map = map;
                view->map->refcount++;
            }
        }

        if (mail_index_sync_record(&sync_map_ctx, thdr, data) < 0) {
            ret = -1;
            break;
        }

        /* mail_index_sync_record() might have changed map to anything.
           make sure we don't accidentally try to use it. */
        map = NULL;
    }
    map = view->map;
        mail_index_sync_map_deinit(&sync_map_ctx);

    index->sync_update = FALSE;

    if (ret < 0) {
        mail_index_view_unlock(view);
        return -1;
    }

    i_assert(map->records_count == map->hdr.messages_count);

    mail_transaction_log_get_head(index->log, &seq, &offset);

    map->hdr.log_file_seq = seq;
    if (!sync_only_external)
        map->hdr.log_file_int_offset = offset;
    map->hdr.log_file_ext_offset = offset;

    if (first_append_uid != 0)
        mail_index_update_day_headers(&map->hdr, first_append_uid);

    if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 &&
        had_dirty) {
        /* do we have dirty flags anymore? */
        const struct mail_index_record *rec;

        for (i = 0; i < map->records_count; i++) {
            rec = MAIL_INDEX_MAP_IDX(map, i);
            if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
                map->hdr.flags |=
                    MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
                break;
            }
        }
    }

    if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
        map->mmap_used_size = index->hdr->header_size +
            map->records_count * map->hdr.record_size;

        memcpy(map->mmap_base, &map->hdr, sizeof(map->hdr));
        memcpy(PTR_OFFSET(map->mmap_base, sizeof(map->hdr)),
               PTR_OFFSET(map->hdr_base, sizeof(map->hdr)),
               map->hdr.header_size - sizeof(map->hdr));
        if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
            mail_index_set_syscall_error(index, "msync()");
            ret = -1;
        }
    }
    i_assert(view->map == index->map);
    view->hdr = map->hdr;

        mail_index_view_unlock(view);
    return ret;
}