src/lib-index/mail-transaction-log-append.c

	mail-transaction-log-append.c revision 0d0451206a91e9f96e522075dce28a89adc2325d
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "lib.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "ioloop.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "array.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "buffer.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "write-full.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "mail-index-private.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "mail-index-view-private.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "mail-index-transaction-private.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen#include "mail-transaction-log-private.h"
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainenstatic int log_append_buffer(struct mail_transaction_log_file *file,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                 const buffer_t *buf, const buffer_t *hdr_buf,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                 enum mail_transaction_type type, int external)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    struct mail_transaction_header hdr;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const void *data, *hdr_data;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    size_t size, hdr_data_size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    uoff_t offset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    uint32_t hdr_size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    data = buffer_get_data(buf, &size);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (size == 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        return 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    i_assert((size % 4) == 0);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (hdr_buf != NULL) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        hdr_data = buffer_get_data(hdr_buf, &hdr_data_size);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        i_assert((hdr_data_size % 4) == 0);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        hdr_data = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        hdr_data_size = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    memset(&hdr, 0, sizeof(hdr));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    hdr.type = type;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (type == MAIL_TRANSACTION_EXPUNGE)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (external)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        hdr.type |= MAIL_TRANSACTION_EXTERNAL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    hdr_size = mail_index_uint32_to_offset(sizeof(hdr) + size +
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                           hdr_data_size);
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen    if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        do {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            offset = file->sync_offset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (file->first_append_size == 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                /* size will be written later once everything
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                   is in disk */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                file->first_append_size = hdr_size;
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen            } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                hdr.size = hdr_size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (pwrite_full(file->fd, &hdr, sizeof(hdr),
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                    offset) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                break;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            offset += sizeof(hdr);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (hdr_data_size > 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                if (pwrite_full(file->fd, hdr_data,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                        hdr_data_size, offset) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                    break;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                offset += hdr_data_size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (pwrite_full(file->fd, data, size, offset) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                break;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            file->sync_offset = offset + size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            return 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        } while (0);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* write failure. */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (!ENOSPACE(errno)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            mail_index_file_set_syscall_error(file->log->index,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                              file->filepath,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                              "pwrite_full()");
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            return -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* not enough space. fallback to in-memory indexes. */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (mail_index_move_to_memory(file->log->index) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            return -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        i_assert(MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    hdr.size = hdr_size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    i_assert(file->buffer_offset + file->buffer->used ==
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen         file->sync_offset);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buffer_append(file->buffer, &hdr, sizeof(hdr));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buffer_append(file->buffer, hdr_data, hdr_data_size);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buffer_append(file->buffer, data, size);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    file->sync_offset = file->buffer_offset + file->buffer->used;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    return 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic const buffer_t *
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenlog_get_hdr_update_buffer(struct mail_index_transaction *t, int prepend)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buffer_t *buf;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const unsigned char *data, *mask;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    struct mail_transaction_header_update u;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    uint16_t offset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    int state = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    memset(&u, 0, sizeof(u));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    data = prepend ? t->pre_hdr_change : t->post_hdr_change;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buf = buffer_create_dynamic(pool_datastack_create(), 256);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (state == 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                u.offset = offset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                state++;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (state > 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                u.size = offset - u.offset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                buffer_append(buf, &u, sizeof(u));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                buffer_append(buf, data + u.offset, u.size);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                state = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    return buf;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic int log_append_ext_intro(struct mail_transaction_log_file *file,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                struct mail_index_transaction *t,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                uint32_t ext_id, uint32_t reset_id)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const struct mail_index_registered_ext *rext;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        struct mail_transaction_ext_intro *intro;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buffer_t *buf;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    uint32_t idx;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    unsigned int count;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!mail_index_map_get_ext_idx(t->view->map, ext_id, &idx)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* new extension */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        idx = (uint32_t)-1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    rext = array_idx(&t->view->index->extensions, ext_id);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!array_is_created(&t->ext_resizes)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        count = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro = array_get_modifyable(&t->ext_resizes, &count);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buf = buffer_create_dynamic(pool_datastack_create(), 128);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (ext_id < count && intro[ext_id].name_size != 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* we're resizing it */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro += ext_id;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        i_assert(intro->ext_id == idx);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro->name_size = idx != (uint32_t)-1 ? 0 :
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            strlen(rext->name);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        buffer_append(buf, intro, sizeof(*intro));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* generate a new intro structure */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro = buffer_append_space_unsafe(buf, sizeof(*intro));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro->ext_id = idx;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro->hdr_size = rext->hdr_size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro->record_size = rext->record_size;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro->record_align = rext->record_align;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro->name_size = idx != (uint32_t)-1 ? 0 :
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            strlen(rext->name);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (reset_id != 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* we're going to reset this extension in this transaction */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro->reset_id = reset_id;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else if (idx != (uint32_t)-1) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* use the existing reset_id */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        const struct mail_index_ext *map_ext =
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            array_idx(&t->view->map->extensions, idx);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        intro->reset_id = map_ext->reset_id;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        /* new extension, reset_id defaults to 0 */
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buffer_append(buf, rext->name, intro->name_size);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if ((buf->used % 4) != 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        buffer_append_zero(buf, 4 - (buf->used % 4));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    return log_append_buffer(file, buf, NULL, MAIL_TRANSACTION_EXT_INTRO,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                 t->external);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic int
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenmail_transaction_log_append_ext_intros(struct mail_transaction_log_file *file,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                       struct mail_index_transaction *t)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        const struct mail_transaction_ext_intro *resize;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    struct mail_transaction_ext_reset ext_reset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    unsigned int update_count, resize_count, reset_count, ext_count;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    uint32_t ext_id;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const uint32_t *reset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const array_t *update;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buffer_t *buf;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!array_is_created(&t->ext_rec_updates)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        update = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        update_count = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        update = array_get(&t->ext_rec_updates, &update_count);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!array_is_created(&t->ext_resizes)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        resize = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        resize_count = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        resize = array_get(&t->ext_resizes, &resize_count);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!array_is_created(&t->ext_resets)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        reset = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        reset_count = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        reset = array_get(&t->ext_resets, &reset_count);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    memset(&ext_reset, 0, sizeof(ext_reset));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buf = buffer_create_data(pool_datastack_create(),
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                 &ext_reset, sizeof(ext_reset));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    buffer_set_used_size(buf, sizeof(ext_reset));
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    ext_count = I_MAX(I_MAX(update_count, resize_count), reset_count);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    for (ext_id = 0; ext_id < ext_count; ext_id++) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        ext_reset.new_reset_id =
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            ext_id < reset_count && reset[ext_id] != 0 ?
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            reset[ext_id] : 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if ((ext_id < resize_count && resize[ext_id].name_size) ||
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            (ext_id < update_count &&
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen             array_is_created(&update[ext_id])) ||
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            ext_reset.new_reset_id != 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (log_append_ext_intro(file, t, ext_id, 0) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                return -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (ext_reset.new_reset_id != 0) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            if (log_append_buffer(file, buf, NULL,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                          MAIL_TRANSACTION_EXT_RESET,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                          t->external) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                return -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    return 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen}
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainenstatic int log_append_ext_rec_updates(struct mail_transaction_log_file *file,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                      struct mail_index_transaction *t)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen{
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    array_t *updates;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    const uint32_t *reset;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    unsigned int ext_id, count, reset_count;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    uint32_t reset_id;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!array_is_created(&t->ext_rec_updates)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        updates = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        count = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        updates = array_get_modifyable(&t->ext_rec_updates, &count);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    if (!array_is_created(&t->ext_resets)) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        reset = NULL;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        reset_count = 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    } else {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        reset = array_get_modifyable(&t->ext_resets, &reset_count);
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    }
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen    for (ext_id = 0; ext_id < count; ext_id++) {
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (!array_is_created(&updates[ext_id]))
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            continue;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        reset_id = ext_id < reset_count && reset[ext_id] != 0 ?
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            reset[ext_id] : 0;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (log_append_ext_intro(file, t, ext_id, reset_id) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            return -1;
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen        if (log_append_buffer(file, updates[ext_id].buffer, NULL,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                      MAIL_TRANSACTION_EXT_REC_UPDATE,
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen                      t->external) < 0)
5806683c1c3f5b1997e92a023c0fe39912d4df5dTimo Sirainen            return -1;
    }
    return 0;
}

static int
log_append_keyword_update(struct mail_transaction_log_file *file,
              struct mail_index_transaction *t,
              buffer_t *hdr_buf, enum modify_type modify_type,
              const char *keyword, const buffer_t *buffer)
{
    struct mail_transaction_keyword_update kt_hdr;

    memset(&kt_hdr, 0, sizeof(kt_hdr));
    kt_hdr.modify_type = modify_type;
    kt_hdr.name_size = strlen(keyword);

    buffer_set_used_size(hdr_buf, 0);
    buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
    buffer_append(hdr_buf, keyword, kt_hdr.name_size);
    if ((hdr_buf->used % 4) != 0)
        buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));

    if (t->hide_transaction) {
        mail_index_view_add_synced_transaction(t->view,
            file->hdr.file_seq, file->sync_offset);
    }

    return log_append_buffer(file, buffer, hdr_buf,
                 MAIL_TRANSACTION_KEYWORD_UPDATE, t->external);
}

static int log_append_keyword_updates(struct mail_transaction_log_file *file,
                      struct mail_index_transaction *t)
{
        const struct mail_index_transaction_keyword_update *updates;
    const char *const *keywords;
    buffer_t *hdr_buf;
    unsigned int i, count, keywords_count;

    hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);

    keywords = array_get_modifyable(&t->view->index->keywords,
                    &keywords_count);
    updates = array_get_modifyable(&t->keyword_updates, &count);
    i_assert(count <= keywords_count);

    for (i = 0; i < count; i++) {
        if (array_is_created(&updates[i].add_seq)) {
            if (log_append_keyword_update(file, t, hdr_buf,
                    MODIFY_ADD, keywords[i],
                    updates[i].add_seq.buffer) < 0)
                return -1;
        }
        if (array_is_created(&updates[i].remove_seq)) {
            if (log_append_keyword_update(file, t, hdr_buf,
                    MODIFY_REMOVE, keywords[i],
                    updates[i].remove_seq.buffer) < 0)
                return -1;
        }
    }

    return 0;
}

#define ARE_ALL_TRANSACTIONS_IN_INDEX(log, idx_hdr) \
    ((log)->head->hdr.file_seq == (idx_hdr)->log_file_seq && \
     (log)->head->sync_offset == (idx_hdr)->log_file_int_offset && \
     (log)->head->sync_offset == (idx_hdr)->log_file_ext_offset)

int mail_transaction_log_append(struct mail_index_transaction *t,
                uint32_t *log_file_seq_r,
                uoff_t *log_file_offset_r)
{
    struct mail_index_view *view = t->view;
    struct mail_index *index;
    struct mail_transaction_log *log;
    struct mail_transaction_log_file *file;
    struct mail_index_header idx_hdr;
    uoff_t append_offset;
    unsigned int old_log_syncs_pos;
    unsigned int lock_id;
    int ret;

    index = mail_index_view_get_index(view);
    log = index->log;

    if (!t->log_updates) {
        /* nothing to append */
        *log_file_seq_r = 0;
        *log_file_offset_r = 0;
        return 0;
    }

    if (log->index->log_locked) {
        i_assert(t->external);
    } else {
        if (mail_transaction_log_lock_head(log) < 0)
            return -1;

        /* update sync_offset */
        if (mail_transaction_log_file_map(log->head,
                          log->head->sync_offset,
                          (uoff_t)-1) < 0) {
            mail_transaction_log_file_unlock(log->head);
            return -1;
        }
    }

    if (log->head->sync_offset > MAIL_TRANSACTION_LOG_ROTATE_SIZE &&
        (time_t)log->head->hdr.create_stamp <
        ioloop_time - MAIL_TRANSACTION_LOG_ROTATE_TIME &&
        ARE_ALL_TRANSACTIONS_IN_INDEX(log, index->hdr)) {
        /* we might want to rotate, but check first that everything is
           synced in index. */
        if (mail_index_lock_shared(log->index, TRUE, &lock_id) < 0) {
            if (!log->index->log_locked)
                mail_transaction_log_file_unlock(log->head);
            return -1;
        }

        /* we need the latest log_file_*_offsets. It's important to
           use this function instead of mail_index_map() as it may
           have generated them by reading log files. */
        if (mail_index_get_latest_header(index, &idx_hdr) <= 0) {
            mail_index_unlock(index, lock_id);
            if (!log->index->log_locked)
                mail_transaction_log_file_unlock(log->head);
            return -1;
        }
        mail_index_unlock(log->index, lock_id);

        if (ARE_ALL_TRANSACTIONS_IN_INDEX(log, &idx_hdr)) {
            if (mail_transaction_log_rotate(log, TRUE) < 0) {
                /* that didn't work. well, try to continue
                   anyway */
            }
        }
    }

    file = log->head;
    file->first_append_size = 0;
    append_offset = file->sync_offset;

    old_log_syncs_pos = !array_is_created(&view->log_syncs) ? 0 :
        array_count(&view->log_syncs);

    ret = 0;

    /* send all extension introductions and resizes before appends
       to avoid resize overhead as much as possible */
        ret = mail_transaction_log_append_ext_intros(file, t);

    if (t->pre_hdr_changed && ret == 0) {
        ret = log_append_buffer(file,
                    log_get_hdr_update_buffer(t, TRUE),
                    NULL, MAIL_TRANSACTION_HEADER_UPDATE,
                    t->external);
    }
    if (array_is_created(&t->appends) && ret == 0) {
        if (t->hide_transaction) {
            mail_index_view_add_synced_transaction(view,
                file->hdr.file_seq, file->sync_offset);
        }
        ret = log_append_buffer(file, t->appends.buffer, NULL,
                    MAIL_TRANSACTION_APPEND, t->external);
    }
    if (array_is_created(&t->updates) && ret == 0) {
        if (t->hide_transaction) {
            mail_index_view_add_synced_transaction(view,
                file->hdr.file_seq, file->sync_offset);
        }
        ret = log_append_buffer(file, t->updates.buffer, NULL,
                    MAIL_TRANSACTION_FLAG_UPDATE,
                    t->external);
    }

    if (array_is_created(&t->ext_rec_updates) && ret == 0)
        ret = log_append_ext_rec_updates(file, t);

    /* keyword resets before updates */
    if (array_is_created(&t->keyword_resets) && ret == 0) {
        if (t->hide_transaction) {
            mail_index_view_add_synced_transaction(view,
                file->hdr.file_seq, file->sync_offset);
        }
        ret = log_append_buffer(file, t->keyword_resets.buffer, NULL,
                    MAIL_TRANSACTION_KEYWORD_RESET,
                    t->external);
    }
    if (array_is_created(&t->keyword_updates) && ret == 0)
        ret = log_append_keyword_updates(file, t);

    if (array_is_created(&t->expunges) && ret == 0) {
        /* Expunges cannot be hidden */
        ret = log_append_buffer(file, t->expunges.buffer, NULL,
                    MAIL_TRANSACTION_EXPUNGE, t->external);
    }

    if (t->post_hdr_changed && ret == 0) {
        ret = log_append_buffer(file,
                    log_get_hdr_update_buffer(t, FALSE),
                    NULL, MAIL_TRANSACTION_HEADER_UPDATE,
                    t->external);
    }

    if (ret == 0 && file->first_append_size != 0) {
        if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
            /* synced - rewrite first record's header */
            ret = pwrite_full(file->fd, &file->first_append_size,
                      sizeof(uint32_t), append_offset);
            if (ret < 0) {
                mail_index_file_set_syscall_error(log->index,
                    file->filepath, "pwrite()");
            }
        } else {
            /* changed into in-memory buffer in the middle */
            buffer_write(file->buffer,
                     append_offset - file->buffer_offset,
                     &file->first_append_size,
                     sizeof(file->first_append_size));
        }
    }

    if (ret < 0) {
        if (array_is_created(&view->log_syncs)) {
            /* revert changes to log_syncs */
            array_delete(&view->log_syncs, old_log_syncs_pos,
                     array_count(&view->log_syncs) -
                     old_log_syncs_pos);
        }
        file->sync_offset = append_offset;
    }

    *log_file_seq_r = file->hdr.file_seq;
    *log_file_offset_r = file->sync_offset;

    if (!log->index->log_locked)
        mail_transaction_log_file_unlock(file);
    return ret;
}