src/lib-index/mail-index-transaction-export.c

	mail-index-transaction-export.c revision 01e29d9d23a1844b4582592a473a3b3eac56b36b
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (c) 2003-2015 Dovecot authors, see the included COPYING file */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "lib.h"
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen#include "array.h"
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen#include "mail-index-private.h"
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen#include "mail-index-modseq.h"
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen#include "mail-transaction-log-private.h"
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen#include "mail-index-transaction-private.h"
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstruct mail_index_export_context {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen    struct mail_index_transaction *trans;
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen    struct mail_transaction_log_append_ctx *append_ctx;
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen};
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainenstatic void
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainenlog_append_buffer(struct mail_index_export_context *ctx,
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen          const buffer_t *buf, enum mail_transaction_type type)
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen{
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    mail_transaction_log_append_add(ctx->append_ctx, type,
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen                    buf->data, buf->used);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen}
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainenstatic void log_append_flag_updates(struct mail_index_export_context *ctx,
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen                    struct mail_index_transaction *t)
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen{
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    ARRAY(struct mail_transaction_flag_update) log_updates;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    const struct mail_index_flag_update *updates;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    struct mail_transaction_flag_update *log_update;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    unsigned int i, count;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    updates = array_get(&t->updates, &count);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    if (count == 0)
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen        return;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    i_array_init(&log_updates, count);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    for (i = 0; i < count; i++) {
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen        log_update = array_append_space(&log_updates);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen        log_update->uid1 = updates[i].uid1;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen        log_update->uid2 = updates[i].uid2;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        log_update->add_flags = updates[i].add_flags & 0xff;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        log_update->remove_flags = updates[i].remove_flags & 0xff;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        if ((updates[i].add_flags & MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ) != 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen            log_update->modseq_inc_flag = 1;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen    }
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen    log_append_buffer(ctx, log_updates.arr.buffer,
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen              MAIL_TRANSACTION_FLAG_UPDATE);
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen    array_free(&log_updates);
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen}
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainenstatic const buffer_t *
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainenlog_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen{
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen    buffer_t *buf;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    const unsigned char *data, *mask;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    struct mail_transaction_header_update u;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    uint16_t offset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    int state = 0;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    memset(&u, 0, sizeof(u));
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen    data = prepend ? t->pre_hdr_change : t->post_hdr_change;
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen    mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainen
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainen    buf = buffer_create_dynamic(pool_datastack_create(), 256);
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainen    for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainen        if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
9349a0afffad990e45d3ad33081e1d2d9e68a753Timo Sirainen            if (state == 0) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen                u.offset = offset;
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainen                state++;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen            }
13a8c553f293349248b161ff851743498916e26eTimo Sirainen        } else {
b87761f9bbef949f31dae297e619ac3f5e9c2b2eTimo Sirainen            if (state > 0) {
b87761f9bbef949f31dae297e619ac3f5e9c2b2eTimo Sirainen                u.size = offset - u.offset;
b87761f9bbef949f31dae297e619ac3f5e9c2b2eTimo Sirainen                buffer_append(buf, &u, sizeof(u));
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen                buffer_append(buf, data + u.offset, u.size);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen                state = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen            }
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen        }
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    }
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    return buf;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainenstatic unsigned int
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainenext_hdr_update_get_size(const struct mail_index_transaction_ext_hdr_update *hu)
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen{
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen    unsigned int i;
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen    for (i = hu->alloc_size; i > 0; i--) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen        if (hu->mask[i-1] != 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen            return i;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    }
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    return 0;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen}
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic void log_append_ext_intro(struct mail_index_export_context *ctx,
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen                 uint32_t ext_id, uint32_t reset_id,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen                 unsigned int *hdr_size_r)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen    struct mail_index_transaction *t = ctx->trans;
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen    const struct mail_index_registered_ext *rext;
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen    const struct mail_index_ext *ext;
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen        struct mail_transaction_ext_intro *intro, *resizes;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen    buffer_t *buf;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    uint32_t idx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    unsigned int count;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    i_assert(ext_id != (uint32_t)-1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    if (t->reset ||
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        /* new extension */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        idx = (uint32_t)-1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    rext = array_idx(&t->view->index->extensions, ext_id);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen    if (!array_is_created(&t->ext_resizes)) {
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen        resizes = NULL;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        count = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    } else {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        resizes = array_get_modifiable(&t->ext_resizes, &count);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    buf = buffer_create_dynamic(pool_datastack_create(), 128);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    if (ext_id < count && resizes[ext_id].name_size != 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        /* we're resizing the extension. use the resize struct. */
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen        intro = &resizes[ext_id];
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen        if (idx != (uint32_t)-1) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen            intro->ext_id = idx;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen            intro->name_size = 0;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen        } else {
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen            intro->ext_id = (uint32_t)-1;
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen            intro->name_size = strlen(rext->name);
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen        }
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen        buffer_append(buf, intro, sizeof(*intro));
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen    } else {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen        /* generate a new intro structure */
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen        intro = buffer_append_space_unsafe(buf, sizeof(*intro));
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen        intro->ext_id = idx;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen        if (idx == (uint32_t)-1) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen            intro->hdr_size = rext->hdr_size;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen            intro->record_size = rext->record_size;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen            intro->record_align = rext->record_align;
bfdf0fd7b6186f64cbdcbf1cb2bf9c42a9007b77Timo Sirainen            intro->name_size = strlen(rext->name);
bfdf0fd7b6186f64cbdcbf1cb2bf9c42a9007b77Timo Sirainen        } else {
bfdf0fd7b6186f64cbdcbf1cb2bf9c42a9007b77Timo Sirainen            ext = array_idx(&t->view->index->map->extensions, idx);
bfdf0fd7b6186f64cbdcbf1cb2bf9c42a9007b77Timo Sirainen            intro->hdr_size = ext->hdr_size;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen            intro->record_size = ext->record_size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen            intro->record_align = ext->record_align;
            intro->name_size = 0;
        }
        intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;

        /* handle increasing header size automatically */
        if (array_is_created(&t->ext_hdr_updates) &&
            ext_id < array_count(&t->ext_hdr_updates)) {
            const struct mail_index_transaction_ext_hdr_update *hu;
            unsigned int hdr_update_size;

            hu = array_idx(&t->ext_hdr_updates, ext_id);
            hdr_update_size = ext_hdr_update_get_size(hu);
            if (intro->hdr_size < hdr_update_size)
                intro->hdr_size = hdr_update_size;
        }
    }
    i_assert(intro->record_size != 0 || intro->hdr_size != 0);
    if (reset_id != 0) {
        /* we're going to reset this extension in this transaction */
        intro->reset_id = reset_id;
    } else if (idx != (uint32_t)-1) {
        /* use the existing reset_id */
        const struct mail_index_ext *map_ext =
            array_idx(&t->view->index->map->extensions, idx);
        intro->reset_id = map_ext->reset_id;
    } else {
        /* new extension, reset_id defaults to 0 */
    }
    buffer_append(buf, rext->name, intro->name_size);
    if ((buf->used % 4) != 0)
        buffer_append_zero(buf, 4 - (buf->used % 4));

    if (ctx->append_ctx->new_highest_modseq == 0 &&
        strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
        /* modseq tracking started */
        ctx->append_ctx->new_highest_modseq = 1;
    }

    log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_INTRO);
    *hdr_size_r = intro->hdr_size;
}

static void
log_append_ext_hdr_update(struct mail_index_export_context *ctx,
              const struct mail_index_transaction_ext_hdr_update *hdr,
              unsigned int ext_hdr_size)
{
    buffer_t *buf;
    const unsigned char *data, *mask;
    struct mail_transaction_ext_hdr_update u;
    struct mail_transaction_ext_hdr_update32 u32;
    size_t offset;
    bool started = FALSE, use_32 = hdr->alloc_size >= 65536;

    memset(&u, 0, sizeof(u));
    memset(&u32, 0, sizeof(u32));

    data = hdr->data;
    mask = hdr->mask;

    buf = buffer_create_dynamic(pool_datastack_create(), 256);
    for (offset = 0; offset <= hdr->alloc_size; offset++) {
        if (offset < hdr->alloc_size && mask[offset] != 0) {
            if (!started) {
                u32.offset = offset;
                started = TRUE;
            }
        } else {
            if (started) {
                u32.size = offset - u32.offset;
                if (use_32)
                    buffer_append(buf, &u32, sizeof(u32));
                else {
                    u.offset = u32.offset;
                    u.size = u32.size;
                    buffer_append(buf, &u, sizeof(u));
                }
                i_assert(u32.offset + u32.size <= ext_hdr_size);
                buffer_append(buf, data + u32.offset, u32.size);
                started = FALSE;
            }
        }
    }
    if (buf->used % 4 != 0)
        buffer_append_zero(buf, 4 - buf->used % 4);
    log_append_buffer(ctx, buf, use_32 ? MAIL_TRANSACTION_EXT_HDR_UPDATE32 :
              MAIL_TRANSACTION_EXT_HDR_UPDATE);
}

static void
mail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx)
{
    struct mail_index_transaction *t = ctx->trans;
        const struct mail_transaction_ext_intro *resize;
    const struct mail_index_transaction_ext_hdr_update *hdrs;
    struct mail_transaction_ext_reset ext_reset;
    unsigned int resize_count, ext_count = 0;
    unsigned int hdrs_count, reset_id_count, reset_count, hdr_size;
    uint32_t ext_id, reset_id;
    const struct mail_transaction_ext_reset *reset;
    const uint32_t *reset_ids;
    buffer_t reset_buf;

    if (!array_is_created(&t->ext_resizes)) {
        resize = NULL;
        resize_count = 0;
    } else {
        resize = array_get(&t->ext_resizes, &resize_count);
        if (ext_count < resize_count)
            ext_count = resize_count;
    }

    if (!array_is_created(&t->ext_reset_ids)) {
        reset_ids = NULL;
        reset_id_count = 0;
    } else {
        reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
    }

    if (!array_is_created(&t->ext_resets)) {
        reset = NULL;
        reset_count = 0;
    } else {
        reset = array_get(&t->ext_resets, &reset_count);
        if (ext_count < reset_count)
            ext_count = reset_count;
    }

    if (!array_is_created(&t->ext_hdr_updates)) {
        hdrs = NULL;
        hdrs_count = 0;
    } else {
        hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
        if (ext_count < hdrs_count)
            ext_count = hdrs_count;
    }

    memset(&ext_reset, 0, sizeof(ext_reset));
    buffer_create_from_data(&reset_buf, &ext_reset, sizeof(ext_reset));
    buffer_set_used_size(&reset_buf, sizeof(ext_reset));

    for (ext_id = 0; ext_id < ext_count; ext_id++) {
        if (ext_id < reset_count)
            ext_reset = reset[ext_id];
        else
            ext_reset.new_reset_id = 0;
        if ((ext_id < resize_count && resize[ext_id].name_size) ||
            ext_reset.new_reset_id != 0 ||
            (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
            if (ext_reset.new_reset_id != 0) {
                /* we're going to reset this extension
                   immediately after the intro */
                reset_id = 0;
            } else {
                reset_id = ext_id < reset_id_count ?
                    reset_ids[ext_id] : 0;
            }
            log_append_ext_intro(ctx, ext_id, reset_id, &hdr_size);
        } else {
            hdr_size = 0;
        }
        if (ext_reset.new_reset_id != 0) {
            i_assert(ext_id < reset_id_count &&
                 ext_reset.new_reset_id == reset_ids[ext_id]);
            log_append_buffer(ctx, &reset_buf,
                      MAIL_TRANSACTION_EXT_RESET);
        }
        if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
            T_BEGIN {
                log_append_ext_hdr_update(ctx, &hdrs[ext_id],
                              hdr_size);
            } T_END;
        }
    }
}

static void log_append_ext_recs(struct mail_index_export_context *ctx,
                const ARRAY_TYPE(seq_array_array) *arr,
                enum mail_transaction_type type)
{
    struct mail_index_transaction *t = ctx->trans;
    const ARRAY_TYPE(seq_array) *updates;
    const uint32_t *reset_ids;
    unsigned int ext_id, count, reset_id_count, hdr_size;
    uint32_t reset_id;

    if (!array_is_created(&t->ext_reset_ids)) {
        reset_ids = NULL;
        reset_id_count = 0;
    } else {
        reset_ids = array_get_modifiable(&t->ext_reset_ids,
                         &reset_id_count);
    }

    updates = array_get(arr, &count);
    for (ext_id = 0; ext_id < count; ext_id++) {
        if (!array_is_created(&updates[ext_id]))
            continue;

        reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
        log_append_ext_intro(ctx, ext_id, reset_id, &hdr_size);

        log_append_buffer(ctx, updates[ext_id].arr.buffer, type);
    }
}

static void
log_append_keyword_update(struct mail_index_export_context *ctx,
              buffer_t *tmp_buf, enum modify_type modify_type,
              const char *keyword, const buffer_t *uid_buffer)
{
    struct mail_transaction_keyword_update kt_hdr;

    i_assert(uid_buffer->used > 0);

    memset(&kt_hdr, 0, sizeof(kt_hdr));
    kt_hdr.modify_type = modify_type;
    kt_hdr.name_size = strlen(keyword);

    buffer_set_used_size(tmp_buf, 0);
    buffer_append(tmp_buf, &kt_hdr, sizeof(kt_hdr));
    buffer_append(tmp_buf, keyword, kt_hdr.name_size);
    if ((tmp_buf->used % 4) != 0)
        buffer_append_zero(tmp_buf, 4 - (tmp_buf->used % 4));
    buffer_append(tmp_buf, uid_buffer->data, uid_buffer->used);

    log_append_buffer(ctx, tmp_buf, MAIL_TRANSACTION_KEYWORD_UPDATE);
}

static enum mail_index_fsync_mask
log_append_keyword_updates(struct mail_index_export_context *ctx)
{
        const struct mail_index_transaction_keyword_update *updates;
    const char *const *keywords;
    buffer_t *tmp_buf;
    enum mail_index_fsync_mask change_mask = 0;
    unsigned int i, count, keywords_count;

    tmp_buf = buffer_create_dynamic(pool_datastack_create(), 64);

    keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
                    &keywords_count);
    updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
    i_assert(count <= keywords_count);

    for (i = 0; i < count; i++) {
        if (array_is_created(&updates[i].add_seq) &&
            array_count(&updates[i].add_seq) > 0) {
            change_mask |= MAIL_INDEX_FSYNC_MASK_KEYWORDS;
            log_append_keyword_update(ctx, tmp_buf,
                    MODIFY_ADD, keywords[i],
                    updates[i].add_seq.arr.buffer);
        }
        if (array_is_created(&updates[i].remove_seq) &&
            array_count(&updates[i].remove_seq) > 0) {
            change_mask |= MAIL_INDEX_FSYNC_MASK_KEYWORDS;
            log_append_keyword_update(ctx, tmp_buf,
                    MODIFY_REMOVE, keywords[i],
                    updates[i].remove_seq.arr.buffer);
        }
    }
    return change_mask;
}

void mail_index_transaction_export(struct mail_index_transaction *t,
                   struct mail_transaction_log_append_ctx *append_ctx)
{
    static uint8_t null4[4] = { 0, 0, 0, 0 };
    enum mail_index_fsync_mask change_mask = 0;
    struct mail_index_export_context ctx;

    memset(&ctx, 0, sizeof(ctx));
    ctx.trans = t;
    ctx.append_ctx = append_ctx;

    if (t->index_undeleted) {
        i_assert(!t->index_deleted);
        mail_transaction_log_append_add(ctx.append_ctx,
            MAIL_TRANSACTION_INDEX_UNDELETED, &null4, 4);
    }

    /* send all extension introductions and resizes before appends
       to avoid resize overhead as much as possible */
        mail_transaction_log_append_ext_intros(&ctx);

    if (t->pre_hdr_changed) {
        log_append_buffer(&ctx, log_get_hdr_update_buffer(t, TRUE),
                  MAIL_TRANSACTION_HEADER_UPDATE);
    }
    if (t->attribute_updates != NULL) {
        buffer_append_c(t->attribute_updates, '\0');
        /* need to have 32bit alignment */
        if (t->attribute_updates->used % 4 != 0) {
            buffer_append_zero(t->attribute_updates,
                       4 - t->attribute_updates->used % 4);
        }
        /* append the timestamp and value lengths */
        buffer_append(t->attribute_updates,
                  t->attribute_updates_suffix->data,
                  t->attribute_updates_suffix->used);
        i_assert(t->attribute_updates->used % 4 == 0);
        log_append_buffer(&ctx, t->attribute_updates,
                  MAIL_TRANSACTION_ATTRIBUTE_UPDATE);
    }
    if (array_is_created(&t->appends)) {
        change_mask |= MAIL_INDEX_FSYNC_MASK_APPENDS;
        log_append_buffer(&ctx, t->appends.arr.buffer,
                  MAIL_TRANSACTION_APPEND);
    }

    if (array_is_created(&t->updates)) {
        change_mask |= MAIL_INDEX_FSYNC_MASK_FLAGS;
        log_append_flag_updates(&ctx, t);
    }

    if (array_is_created(&t->ext_rec_updates)) {
        log_append_ext_recs(&ctx, &t->ext_rec_updates,
                    MAIL_TRANSACTION_EXT_REC_UPDATE);
    }
    if (array_is_created(&t->ext_rec_atomics)) {
        log_append_ext_recs(&ctx, &t->ext_rec_atomics,
                    MAIL_TRANSACTION_EXT_ATOMIC_INC);
    }

    if (array_is_created(&t->keyword_updates))
        change_mask |= log_append_keyword_updates(&ctx);
    /* keep modseq updates almost last */
    if (array_is_created(&t->modseq_updates)) {
        log_append_buffer(&ctx, t->modseq_updates.arr.buffer,
                  MAIL_TRANSACTION_MODSEQ_UPDATE);
    }

    if (array_is_created(&t->expunges)) {
        /* non-external expunges are only requests, ignore them when
           checking fsync_mask */
        if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
            change_mask |= MAIL_INDEX_FSYNC_MASK_EXPUNGES;
        log_append_buffer(&ctx, t->expunges.arr.buffer,
                  MAIL_TRANSACTION_EXPUNGE_GUID);
    }

    if (t->post_hdr_changed) {
        log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
                  MAIL_TRANSACTION_HEADER_UPDATE);
    }

    if (t->index_deleted) {
        i_assert(!t->index_undeleted);
        mail_transaction_log_append_add(ctx.append_ctx,
                        MAIL_TRANSACTION_INDEX_DELETED,
                        &null4, 4);
    }

    append_ctx->index_sync_transaction = t->sync_transaction;
    append_ctx->tail_offset_changed = t->tail_offset_changed;
    append_ctx->want_fsync =
        (t->view->index->fsync_mask & change_mask) != 0 ||
        (t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;
}