src/lib-index/mail-index-transaction-export.c

	mail-index-transaction-export.c revision 0c909e3461607eadcd66f4eac69b7f34e37fccf1
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose#include "lib.h"
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose#include "array.h"
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose#include "mail-index-private.h"
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose#include "mail-index-modseq.h"
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose#include "mail-transaction-log-private.h"
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose#include "mail-index-transaction-private.h"
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bosestruct mail_index_export_context {
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose    struct mail_index_transaction *trans;
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose    struct mail_transaction_log_append_ctx *append_ctx;
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose};
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bosestatic void
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Boselog_append_buffer(struct mail_index_export_context *ctx,
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose          const buffer_t *buf, enum mail_transaction_type type)
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose{
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose    mail_transaction_log_append_add(ctx->append_ctx, type,
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose                    buf->data, buf->used);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose}
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bosestatic const buffer_t *
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Boselog_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose{
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    buffer_t *buf;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    const unsigned char *data, *mask;
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose    struct mail_transaction_header_update u;
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose    uint16_t offset;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    int state = 0;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    memset(&u, 0, sizeof(u));
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    data = prepend ? t->pre_hdr_change : t->post_hdr_change;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    buf = buffer_create_dynamic(pool_datastack_create(), 256);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            if (state == 0) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                u.offset = offset;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                state++;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        } else {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            if (state > 0) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                u.size = offset - u.offset;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                buffer_append(buf, &u, sizeof(u));
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                buffer_append(buf, data + u.offset, u.size);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                state = 0;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        }
2ca45dfa0144fea08bd360dafded57bc90111e68Sumit Bose    }
2ca45dfa0144fea08bd360dafded57bc90111e68Sumit Bose    return buf;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose}
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bosestatic void log_append_ext_intro(struct mail_index_export_context *ctx,
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                 uint32_t ext_id, uint32_t reset_id)
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose{
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    struct mail_index_transaction *t = ctx->trans;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    const struct mail_index_registered_ext *rext;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        struct mail_transaction_ext_intro *intro, *resizes;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    buffer_t *buf;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    uint32_t idx;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    unsigned int count;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    i_assert(ext_id != (uint32_t)-1);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    if (t->reset ||
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        /* new extension */
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        idx = (uint32_t)-1;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    rext = array_idx(&t->view->index->extensions, ext_id);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    if (!array_is_created(&t->ext_resizes)) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        resizes = NULL;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        count = 0;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    } else {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        resizes = array_get_modifiable(&t->ext_resizes, &count);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    buf = buffer_create_dynamic(pool_datastack_create(), 128);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    if (ext_id < count && resizes[ext_id].name_size != 0) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        /* we're resizing the extension. use the resize struct. */
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro = &resizes[ext_id];
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        i_assert(intro->ext_id == idx);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro->name_size = idx != (uint32_t)-1 ? 0 :
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            strlen(rext->name);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        buffer_append(buf, intro, sizeof(*intro));
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    } else {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        /* generate a new intro structure */
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro = buffer_append_space_unsafe(buf, sizeof(*intro));
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro->ext_id = idx;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro->hdr_size = rext->hdr_size;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro->record_size = rext->record_size;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro->record_align = rext->record_align;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro->name_size = idx != (uint32_t)-1 ? 0 :
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            strlen(rext->name);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    }
2ca45dfa0144fea08bd360dafded57bc90111e68Sumit Bose    if (reset_id != 0) {
2ca45dfa0144fea08bd360dafded57bc90111e68Sumit Bose        /* we're going to reset this extension in this transaction */
2ca45dfa0144fea08bd360dafded57bc90111e68Sumit Bose        intro->reset_id = reset_id;
2ca45dfa0144fea08bd360dafded57bc90111e68Sumit Bose    } else if (idx != (uint32_t)-1) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        /* use the existing reset_id */
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        const struct mail_index_ext *map_ext =
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            array_idx(&t->view->index->map->extensions, idx);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        intro->reset_id = map_ext->reset_id;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    } else {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        /* new extension, reset_id defaults to 0 */
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    buffer_append(buf, rext->name, intro->name_size);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    if ((buf->used % 4) != 0)
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        buffer_append_zero(buf, 4 - (buf->used % 4));
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    if (ctx->append_ctx->new_highest_modseq == 0 &&
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        /* modseq tracking started */
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        ctx->append_ctx->new_highest_modseq = 1;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_INTRO);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose}
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bosestatic void
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Boselog_append_ext_hdr_update(struct mail_index_export_context *ctx,
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            const struct mail_index_transaction_ext_hdr_update *hdr)
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose{
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    buffer_t *buf;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    const unsigned char *data, *mask;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    struct mail_transaction_ext_hdr_update u;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    uint16_t offset;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    bool started = FALSE;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    memset(&u, 0, sizeof(u));
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    data = hdr->data;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    mask = hdr->mask;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    buf = buffer_create_dynamic(pool_datastack_create(), 256);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    for (offset = 0; offset <= hdr->alloc_size; offset++) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        if (offset < hdr->alloc_size && mask[offset] != 0) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            if (!started) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                u.offset = offset;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                started = TRUE;
cc98e19b424861c2a7fd91e0d657d82c1dbf3059Pavel Reichl            }
cc98e19b424861c2a7fd91e0d657d82c1dbf3059Pavel Reichl        } else {
cc98e19b424861c2a7fd91e0d657d82c1dbf3059Pavel Reichl            if (started) {
cc98e19b424861c2a7fd91e0d657d82c1dbf3059Pavel Reichl                u.size = offset - u.offset;
cc98e19b424861c2a7fd91e0d657d82c1dbf3059Pavel Reichl                buffer_append(buf, &u, sizeof(u));
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                buffer_append(buf, data + u.offset, u.size);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose                started = FALSE;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    if (buf->used % 4 != 0)
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        buffer_append_zero(buf, 4 - buf->used % 4);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_HDR_UPDATE);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose}
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bosestatic void
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bosemail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx)
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose{
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    struct mail_index_transaction *t = ctx->trans;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        const struct mail_transaction_ext_intro *resize;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    const struct mail_index_transaction_ext_hdr_update *hdrs;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    struct mail_transaction_ext_reset ext_reset;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    unsigned int resize_count, ext_count = 0;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    unsigned int hdrs_count, reset_id_count, reset_count;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    uint32_t ext_id, reset_id;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    const struct mail_transaction_ext_reset *reset;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    const uint32_t *reset_ids;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    buffer_t reset_buf;
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose    if (!array_is_created(&t->ext_resizes)) {
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose        resize = NULL;
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose        resize_count = 0;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    } else {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        resize = array_get(&t->ext_resizes, &resize_count);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        if (ext_count < resize_count)
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose            ext_count = resize_count;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    if (!array_is_created(&t->ext_reset_ids)) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        reset_ids = NULL;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        reset_id_count = 0;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    } else {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    }
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    if (!array_is_created(&t->ext_resets)) {
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose        reset = NULL;
8a5e793a0576250da80371e53aa3e7eba15cdb63Sumit Bose        reset_count = 0;
6b5044001e4b0a0caf971a2cf5f27674e0d270f4Sumit Bose    } else {
        reset = array_get(&t->ext_resets, &reset_count);
        if (ext_count < reset_count)
            ext_count = reset_count;
    }

    if (!array_is_created(&t->ext_hdr_updates)) {
        hdrs = NULL;
        hdrs_count = 0;
    } else {
        hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
        if (ext_count < hdrs_count)
            ext_count = hdrs_count;
    }

    memset(&ext_reset, 0, sizeof(ext_reset));
    buffer_create_data(&reset_buf, &ext_reset, sizeof(ext_reset));
    buffer_set_used_size(&reset_buf, sizeof(ext_reset));

    for (ext_id = 0; ext_id < ext_count; ext_id++) {
        if (ext_id < reset_count)
            ext_reset = reset[ext_id];
        else
            ext_reset.new_reset_id = 0;
        if ((ext_id < resize_count && resize[ext_id].name_size) ||
            ext_reset.new_reset_id != 0 ||
            (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
            if (ext_reset.new_reset_id != 0) {
                /* we're going to reset this extension
                   immediately after the intro */
                reset_id = 0;
            } else {
                reset_id = ext_id < reset_id_count ?
                    reset_ids[ext_id] : 0;
            }
            log_append_ext_intro(ctx, ext_id, reset_id);
        }
        if (ext_reset.new_reset_id != 0) {
            i_assert(ext_id < reset_id_count &&
                 ext_reset.new_reset_id == reset_ids[ext_id]);
            log_append_buffer(ctx, &reset_buf,
                      MAIL_TRANSACTION_EXT_RESET);
        }
        if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
            T_BEGIN {
                log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
            } T_END;
        }
    }
}

static void log_append_ext_recs(struct mail_index_export_context *ctx,
                const ARRAY_TYPE(seq_array_array) *arr,
                enum mail_transaction_type type)
{
    struct mail_index_transaction *t = ctx->trans;
    const ARRAY_TYPE(seq_array) *updates;
    const uint32_t *reset_ids;
    unsigned int ext_id, count, reset_id_count;
    uint32_t reset_id;

    if (!array_is_created(&t->ext_reset_ids)) {
        reset_ids = NULL;
        reset_id_count = 0;
    } else {
        reset_ids = array_get_modifiable(&t->ext_reset_ids,
                         &reset_id_count);
    }

    updates = array_get(arr, &count);
    for (ext_id = 0; ext_id < count; ext_id++) {
        if (!array_is_created(&updates[ext_id]))
            continue;

        reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
        log_append_ext_intro(ctx, ext_id, reset_id);

        log_append_buffer(ctx, updates[ext_id].arr.buffer, type);
    }
}

static void
log_append_keyword_update(struct mail_index_export_context *ctx,
              buffer_t *tmp_buf, enum modify_type modify_type,
              const char *keyword, const buffer_t *buffer)
{
    struct mail_transaction_keyword_update kt_hdr;

    memset(&kt_hdr, 0, sizeof(kt_hdr));
    kt_hdr.modify_type = modify_type;
    kt_hdr.name_size = strlen(keyword);

    buffer_set_used_size(tmp_buf, 0);
    buffer_append(tmp_buf, &kt_hdr, sizeof(kt_hdr));
    buffer_append(tmp_buf, keyword, kt_hdr.name_size);
    if ((tmp_buf->used % 4) != 0)
        buffer_append_zero(tmp_buf, 4 - (tmp_buf->used % 4));
    buffer_append(tmp_buf, buffer->data, buffer->used);

    log_append_buffer(ctx, tmp_buf, MAIL_TRANSACTION_KEYWORD_UPDATE);
}

static enum mail_index_sync_type
log_append_keyword_updates(struct mail_index_export_context *ctx)
{
        const struct mail_index_transaction_keyword_update *updates;
    const char *const *keywords;
    buffer_t *tmp_buf;
    enum mail_index_sync_type change_mask = 0;
    unsigned int i, count, keywords_count;

    tmp_buf = buffer_create_dynamic(pool_datastack_create(), 64);

    keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
                    &keywords_count);
    updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
    i_assert(count <= keywords_count);

    for (i = 0; i < count; i++) {
        if (array_is_created(&updates[i].add_seq)) {
            change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
            log_append_keyword_update(ctx, tmp_buf,
                    MODIFY_ADD, keywords[i],
                    updates[i].add_seq.arr.buffer);
        }
        if (array_is_created(&updates[i].remove_seq)) {
            change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
            log_append_keyword_update(ctx, tmp_buf,
                    MODIFY_REMOVE, keywords[i],
                    updates[i].remove_seq.arr.buffer);
        }
    }
    return change_mask;
}

void mail_index_transaction_export(struct mail_index_transaction *t,
                   struct mail_transaction_log_append_ctx *append_ctx)
{
    enum mail_index_sync_type change_mask = 0;
    struct mail_index_export_context ctx;

    memset(&ctx, 0, sizeof(ctx));
    ctx.trans = t;
    ctx.append_ctx = append_ctx;

    /* send all extension introductions and resizes before appends
       to avoid resize overhead as much as possible */
        mail_transaction_log_append_ext_intros(&ctx);

    if (t->pre_hdr_changed) {
        log_append_buffer(&ctx, log_get_hdr_update_buffer(t, TRUE),
                  MAIL_TRANSACTION_HEADER_UPDATE);
    }
    if (array_is_created(&t->appends)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_APPEND;
        log_append_buffer(&ctx, t->appends.arr.buffer,
                  MAIL_TRANSACTION_APPEND);
    }
    if (array_is_created(&t->updates)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS;
        log_append_buffer(&ctx, t->updates.arr.buffer,
                  MAIL_TRANSACTION_FLAG_UPDATE);
    }

    if (array_is_created(&t->ext_rec_updates)) {
        log_append_ext_recs(&ctx, &t->ext_rec_updates,
                    MAIL_TRANSACTION_EXT_REC_UPDATE);
    }
    if (array_is_created(&t->ext_rec_atomics)) {
        log_append_ext_recs(&ctx, &t->ext_rec_atomics,
                    MAIL_TRANSACTION_EXT_ATOMIC_INC);
    }

    /* keyword resets before updates */
    if (array_is_created(&t->keyword_resets)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
        log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
                  MAIL_TRANSACTION_KEYWORD_RESET);
    }
    if (array_is_created(&t->keyword_updates))
        change_mask |= log_append_keyword_updates(&ctx);

    if (array_is_created(&t->expunges)) {
        /* non-external expunges are only requests, ignore them when
           checking fsync_mask */
        if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
            change_mask |= MAIL_INDEX_SYNC_TYPE_EXPUNGE;
        log_append_buffer(&ctx, t->expunges.arr.buffer,
                  MAIL_TRANSACTION_EXPUNGE_GUID);
    }

    if (t->post_hdr_changed) {
        log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
                  MAIL_TRANSACTION_HEADER_UPDATE);
    }

    /* Update the tail offsets only when committing the sync transaction.
       Other transactions may not know the latest tail offset and might
       end up shrinking it. (Alternatively the shrinking tail offsets could
       just be ignored, which would probably work fine too.) */
    append_ctx->append_sync_offset = t->sync_transaction;

    append_ctx->want_fsync =
        (t->view->index->fsync_mask & change_mask) != 0 ||
        (t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;
}