src/lib-index/mail-cache-transaction.c

	mail-cache-transaction.c revision 4bbee99b3aef449a9a2a11a5b5cf1ca486915c49
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi/* Copyright (C) 2003-2004 Timo Sirainen */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "lib.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "buffer.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "file-set-size.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "read-full.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "write-full.h"
96ad5d81ee4a2cc66a4ae19893efc8aa6d06fae7jailletc#include "mail-cache-private.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "mail-index-transaction-private.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
d29d9ab4614ff992b0e8de6e2b88d52b6f1f153erbowen#include <stddef.h>
2e545ce2450a9953665f701bb05350f0d3f26275nd#include <sys/stat.h>
d29d9ab4614ff992b0e8de6e2b88d52b6f1f153erbowen
d29d9ab4614ff992b0e8de6e2b88d52b6f1f153erbowenstruct mail_cache_transaction_ctx {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mail_cache *cache;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mail_cache_view *view;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mail_index_transaction *trans;
af33a4994ae2ff15bc67d19ff1a7feb906745bf8rbowen
3f08db06526d6901aa08c110b5bc7dde6bc39905nd    buffer_t *cache_data, *cache_data_seq;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    uint32_t prev_seq;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    size_t prev_pos;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
3f08db06526d6901aa08c110b5bc7dde6bc39905nd        buffer_t *reservations;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    uint32_t reserved_space_offset, reserved_space;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    uint32_t last_grow_size;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    unsigned int changes:1;
11495c9f0bd33e51a25b4d532beadfbcf9b944a3nilgun};
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic const unsigned char *null4[] = { 0, 0, 0, 0 };
f086b4b402fa9a2fefc7dda85de2a3cc1cd0a654rjung
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistruct mail_cache_transaction_ctx *
645cf915f6bc22be17750bc5bb34ade8de6744dfndmail_cache_get_transaction(struct mail_cache_view *view,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi               struct mail_index_transaction *t)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mail_cache_transaction_ctx *ctx;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (t->cache_trans_ctx != NULL)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        return t->cache_trans_ctx;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    ctx = i_new(struct mail_cache_transaction_ctx, 1);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    ctx->cache = view->cache;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    ctx->view = view;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    ctx->trans = t;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    ctx->reservations =
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        buffer_create_dynamic(system_pool, 256, (size_t)-1);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    i_assert(view->transaction == NULL);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    view->transaction = ctx;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    t->cache_trans_ctx = ctx;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    return ctx;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi}
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic void mail_cache_transaction_free(struct mail_cache_transaction_ctx *ctx)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    ctx->view->transaction = NULL;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    ctx->view->trans_seq1 = ctx->view->trans_seq2 = 0;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (ctx->cache_data != NULL)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        buffer_free(ctx->cache_data);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (ctx->cache_data_seq != NULL)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        buffer_free(ctx->cache_data_seq);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    buffer_free(ctx->reservations);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    i_free(ctx);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun}
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic int mail_cache_grow_file(struct mail_cache *cache, size_t size)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct stat st;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    uoff_t new_fsize, grow_size;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    i_assert(cache->locked);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    /* grow the file */
1f1b6bf13313fdd14a45e52e553d3ff28689b717coar    new_fsize = cache->hdr_copy.used_file_size + size;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (grow_size < 16384)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        grow_size = 16384;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    new_fsize += grow_size;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    new_fsize &= ~1023;
1f1b6bf13313fdd14a45e52e553d3ff28689b717coar
1f1b6bf13313fdd14a45e52e553d3ff28689b717coar    if (fstat(cache->fd, &st) < 0) {
1f1b6bf13313fdd14a45e52e553d3ff28689b717coar        mail_cache_set_syscall_error(cache, "fstat()");
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        return -1;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if ((uoff_t)st.st_size < new_fsize) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        if (file_set_size(cache->fd, new_fsize) < 0) {
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun            mail_cache_set_syscall_error(cache, "file_set_size()");
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            return -1;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    return 0;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi}
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic int mail_cache_unlink_hole(struct mail_cache *cache, size_t size,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi                  struct mail_cache_hole_header *hole_r)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mail_cache_header *hdr = &cache->hdr_copy;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mail_cache_hole_header hole;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    uint32_t offset, prev_offset;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    i_assert(cache->locked);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    offset = hdr->hole_offset; prev_offset = 0;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    while (offset != 0) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        if (pread_full(cache->fd, &hole, sizeof(hole), offset) <= 0) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            mail_cache_set_syscall_error(cache, "pread_full()");
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            return FALSE;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        if (hole.magic != MAIL_CACHE_HOLE_HEADER_MAGIC) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            mail_cache_set_corrupted(cache,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi                "Invalid magic in hole header");
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            return FALSE;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        if (hole.size >= size)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            break;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        offset = hole.next_offset;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (offset == 0)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        return FALSE;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    if (prev_offset == 0)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        hdr->hole_offset = hole.next_offset;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    else {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        if (pwrite_full(cache->fd, &hole.next_offset,
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun                sizeof(hole.next_offset), prev_offset) < 0) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            mail_cache_set_syscall_error(cache, "pwrite_full()");
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            return FALSE;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    hdr->deleted_space -= hole.size;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    hole_r->next_offset = offset;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    hole_r->size = hole.size;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    return TRUE;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun}
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgunstatic void
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgunmail_cache_transaction_add_reservation(struct mail_cache_transaction_ctx *ctx)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun{
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    buffer_append(ctx->reservations, &ctx->reserved_space_offset,
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun              sizeof(ctx->reserved_space_offset));
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    buffer_append(ctx->reservations, &ctx->reserved_space,
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun              sizeof(ctx->reserved_space));
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun}
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgunstatic int
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgunmail_cache_transaction_reserve_more(struct mail_cache_transaction_ctx *ctx,
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun                    size_t size, int commit)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun{
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    struct mail_cache *cache = ctx->cache;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    struct mail_cache_header *hdr = &cache->hdr_copy;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    struct mail_cache_hole_header hole;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    uint32_t *buf;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    i_assert(cache->locked);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    if (mail_cache_unlink_hole(cache, size, &hole)) {
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        /* found a large enough hole. */
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        ctx->reserved_space_offset = hole.next_offset;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        ctx->reserved_space = hole.size;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        mail_cache_transaction_add_reservation(ctx);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        return 0;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
11495c9f0bd33e51a25b4d532beadfbcf9b944a3nilgun        /* mail_cache_unlink_hole() could have noticed corruption */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        return -1;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
f086b4b402fa9a2fefc7dda85de2a3cc1cd0a654rjung
727872d18412fc021f03969b8641810d8896820bhumbedooh    if ((uoff_t)hdr->used_file_size + size > (uint32_t)-1) {
0d0ba3a410038e179b695446bb149cce6264e0abnd        mail_index_set_error(cache->index, "Cache file too large: %s",
727872d18412fc021f03969b8641810d8896820bhumbedooh                     cache->filepath);
cc7e1025de9ac63bd4db6fe7f71c158b2cf09fe4humbedooh        return -1;
0d0ba3a410038e179b695446bb149cce6264e0abnd    }
cc7e1025de9ac63bd4db6fe7f71c158b2cf09fe4humbedooh
727872d18412fc021f03969b8641810d8896820bhumbedooh    if (!commit) {
0d0ba3a410038e179b695446bb149cce6264e0abnd        size = (size + ctx->last_grow_size) * 2;
0d0ba3a410038e179b695446bb149cce6264e0abnd        if ((uoff_t)hdr->used_file_size + size > (uint32_t)-1)
0d0ba3a410038e179b695446bb149cce6264e0abnd            size = (uint32_t)-1;
ac082aefa89416cbdc9a1836eaf3bed9698201c8humbedooh        ctx->last_grow_size = size;
0d0ba3a410038e179b695446bb149cce6264e0abnd    }
0d0ba3a410038e179b695446bb149cce6264e0abnd
0d0ba3a410038e179b695446bb149cce6264e0abnd    if (mail_cache_grow_file(ctx->cache, size) < 0)
727872d18412fc021f03969b8641810d8896820bhumbedooh        return -1;
0d0ba3a410038e179b695446bb149cce6264e0abnd
0d0ba3a410038e179b695446bb149cce6264e0abnd    if (ctx->reserved_space_offset + ctx->reserved_space ==
30471a4650391f57975f60bbb6e4a90be7b284bfhumbedooh        hdr->used_file_size) {
205f749042ed530040a4f0080dbcb47ceae8a374rjung        /* we can simply grow it */
af33a4994ae2ff15bc67d19ff1a7feb906745bf8rbowen        ctx->reserved_space = size - ctx->reserved_space;
0d0ba3a410038e179b695446bb149cce6264e0abnd
7fec19672a491661b2fe4b29f685bc7f4efa64d4nd        /* grow reservation. it's probably the last one in the buffer,
7fec19672a491661b2fe4b29f685bc7f4efa64d4nd           but it's not guarateed because we might have used holes
7fec19672a491661b2fe4b29f685bc7f4efa64d4nd           as well */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        buf = buffer_get_modifyable_data(ctx->reservations, &size);
        size /= sizeof(uint32_t);
        i_assert(size >= 2);

        do {
            size -= 2;
            if (buf[size] == ctx->reserved_space_offset) {
                buf[size+1] = ctx->reserved_space;
                break;
            }
        } while (size >= 2);
    } else {
        ctx->reserved_space_offset = hdr->used_file_size;
        ctx->reserved_space = size;
        mail_cache_transaction_add_reservation(ctx);
    }

    cache->hdr_modified = TRUE;
    hdr->used_file_size = ctx->reserved_space_offset + ctx->reserved_space;
    return 0;
}

static void
mail_cache_free_space(struct mail_cache *cache, uint32_t offset, uint32_t size)
{
    struct mail_cache_hole_header hole;

    i_assert(cache->locked);

    if (offset + size == cache->hdr_copy.used_file_size) {
        /* we can just set used_file_size back */
        cache->hdr_modified = TRUE;
        cache->hdr_copy.used_file_size = offset;
    } else if (size >= MAIL_CACHE_MIN_HOLE_SIZE) {
        /* set it up as a hole */
        hole.next_offset = cache->hdr_copy.hole_offset;
        hole.size = size;
        hole.magic = MAIL_CACHE_HOLE_HEADER_MAGIC;

        if (pwrite_full(cache->fd, &hole, sizeof(hole), offset) < 0) {
            mail_cache_set_syscall_error(cache, "pwrite_full()");
            return;
        }

        cache->hdr_copy.deleted_space += size;
        cache->hdr_copy.hole_offset = offset;
        cache->hdr_modified = TRUE;
    }
}

static void
mail_cache_transaction_free_space(struct mail_cache_transaction_ctx *ctx)
{
    int locked = ctx->cache->locked;

    if (ctx->reserved_space == 0)
        return;

    if (!locked) {
        if (mail_cache_lock(ctx->cache) <= 0)
            return;
    }

    mail_cache_free_space(ctx->cache, ctx->reserved_space_offset,
                  ctx->reserved_space);

    if (!locked)
        mail_cache_unlock(ctx->cache);
}

static uint32_t
mail_cache_transaction_get_space(struct mail_cache_transaction_ctx *ctx,
                 size_t min_size, size_t max_size,
                 size_t *available_space_r, int commit)
{
    int locked = ctx->cache->locked;
    uint32_t offset;
    size_t size;
    int ret;

    i_assert((min_size & 3) == 0);
    i_assert((max_size & 3) == 0);

    if (min_size > ctx->reserved_space) {
        if (!locked) {
            if (mail_cache_lock(ctx->cache) <= 0)
                return -1;
        }
        ret = mail_cache_transaction_reserve_more(ctx, max_size,
                              commit);
        if (!locked)
            mail_cache_unlock(ctx->cache);

        if (ret < 0)
            return 0;

        size = max_size;
    } else {
        size = I_MIN(max_size, ctx->reserved_space);
    }

    offset = ctx->reserved_space_offset;
    ctx->reserved_space_offset += size;
    ctx->reserved_space -= size;
    if (available_space_r != NULL)
        *available_space_r = size;
    i_assert((size & 3) == 0);

    if (size == max_size && commit) {
        /* final commit - see if we can free the rest of the
           reserved space */
        mail_cache_transaction_free_space(ctx);
    }

    return offset;
}

static int
mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
{
    struct mail_cache *cache = ctx->cache;
    const struct mail_cache_record *rec, *tmp_rec;
    const uint32_t *seq;
    uint32_t write_offset, old_offset, rec_pos;
    size_t size, max_size, seq_idx, seq_limit, seq_count;
    int commit;

    commit = ctx->prev_seq == 0;
    if (commit) {
        /* committing, remove the last dummy record */
        buffer_set_used_size(ctx->cache_data, ctx->prev_pos);
    }

    rec = buffer_get_data(ctx->cache_data, &size);
    i_assert(ctx->prev_pos <= size);

    seq = buffer_get_data(ctx->cache_data_seq, &seq_count);
    seq_count /= sizeof(*seq);
    seq_limit = 0;

    for (seq_idx = 0, rec_pos = 0; rec_pos < ctx->prev_pos;) {
        max_size = ctx->prev_pos - rec_pos;
        write_offset = mail_cache_transaction_get_space(ctx, rec->size,
                                max_size,
                                &max_size,
                                commit);
        if (write_offset == 0) {
            /* nothing to write / error */
            return ctx->prev_pos == 0 ? 0 : -1;
        }

        if (rec_pos + max_size < ctx->prev_pos) {
            /* see how much we can really write there */
            tmp_rec = rec;
            for (size = 0; size + tmp_rec->size <= max_size; ) {
                seq_limit++;
                size += tmp_rec->size;
                tmp_rec = CONST_PTR_OFFSET(tmp_rec,
                               tmp_rec->size);
            }
            max_size = size;
        } else {
            seq_limit = seq_count;
        }

        /* write it to file */
        if (pwrite_full(cache->fd, rec, max_size, write_offset) < 0) {
            mail_cache_set_syscall_error(cache, "pwrite_full()");
            return -1;
        }

        /* write the cache_offsets to index file. records' prev_offset
           is updated to point to old cache record when index is being
           synced. */
        for (; seq_idx < seq_limit; seq_idx++) {
            mail_index_update_cache(ctx->trans, seq[seq_idx],
                        cache->hdr->file_seq,
                        write_offset, &old_offset);
            if (old_offset != 0) {
                /* we added records for this message multiple
                   times in this same uncommitted transaction.
                   only the new one will be written to
                   transaction log, we need to do the linking
                   ourself here. */
                if (mail_cache_link(cache, old_offset,
                            write_offset) < 0)
                    return -1;
            }

            write_offset += rec->size;
            rec_pos += rec->size;
            rec = CONST_PTR_OFFSET(rec, rec->size);
        }
    }

    /* drop the written data from buffer */
    buffer_copy(ctx->cache_data, 0,
            ctx->cache_data, ctx->prev_pos, (size_t)-1);
    buffer_set_used_size(ctx->cache_data,
                 buffer_get_used_size(ctx->cache_data) -
                 ctx->prev_pos);
    ctx->prev_pos = 0;

    buffer_set_used_size(ctx->cache_data_seq, 0);
    return 0;
}

static void
mail_cache_transaction_switch_seq(struct mail_cache_transaction_ctx *ctx)
{
    struct mail_cache_record *rec, new_rec;
    void *data;
    size_t size;

    if (ctx->prev_seq != 0) {
        /* fix record size */
        data = buffer_get_modifyable_data(ctx->cache_data, &size);
        rec = PTR_OFFSET(data, ctx->prev_pos);
        rec->size = size - ctx->prev_pos;
        i_assert(rec->size != 0);

        buffer_append(ctx->cache_data_seq, &ctx->prev_seq,
                  sizeof(ctx->prev_seq));
        ctx->prev_pos = size;
    } else if (ctx->cache_data == NULL) {
        ctx->cache_data =
            buffer_create_dynamic(system_pool, 32768, (size_t)-1);
        ctx->cache_data_seq =
            buffer_create_dynamic(system_pool, 256, (size_t)-1);
    }

    memset(&new_rec, 0, sizeof(new_rec));
    buffer_append(ctx->cache_data, &new_rec, sizeof(new_rec));

    ctx->prev_seq = 0;
    ctx->changes = TRUE;
}

int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
{
    struct mail_cache *cache = ctx->cache;
    int ret = 0;

    if (!ctx->changes) {
        mail_cache_transaction_free(ctx);
        return 0;
    }

    if (mail_cache_lock(cache) <= 0) {
        mail_cache_transaction_rollback(ctx);
        return -1;
    }

    if (ctx->prev_seq != 0)
                mail_cache_transaction_switch_seq(ctx);

    if (mail_cache_transaction_flush(ctx) < 0)
        ret = -1;

    /* make sure everything's written before updating offsets */
    if (fdatasync(cache->fd) < 0) {
        mail_cache_set_syscall_error(cache, "fdatasync()");
        ret = -1;
    }

    mail_cache_unlock(cache);
    mail_cache_transaction_free(ctx);
    return ret;
}

void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx)
{
    struct mail_cache *cache = ctx->cache;
    const uint32_t *buf;
    size_t size;

    if (mail_cache_lock(cache) > 0) {
        mail_cache_transaction_free_space(ctx);

        buf = buffer_get_data(ctx->reservations, &size);
        i_assert(size % sizeof(uint32_t)*2 == 0);
        size /= sizeof(*buf);

        if (size > 0) {
            /* free flushed data as well. do it from end to
               beginning so we have a better chance of updating
               used_file_size instead of adding holes */
            do {
                size -= 2;
                mail_cache_free_space(ctx->cache, buf[size],
                              buf[size+1]);
            } while (size > 0);
        }
        mail_cache_unlock(cache);
    }

    mail_cache_transaction_free(ctx);
}

static int
mail_cache_header_write_fields(struct mail_cache_transaction_ctx *ctx)
{
    struct mail_cache *cache = ctx->cache;
    buffer_t *buffer;
    const void *data;
    size_t size;
    uint32_t offset, hdr_offset;
    int ret = 0;

    if (mail_cache_lock(cache) <= 0)
        return -1;

    t_push();
    buffer = buffer_create_dynamic(pool_datastack_create(),
                       256, (size_t)-1);
    mail_cache_header_fields_get(cache, buffer);
    data = buffer_get_data(buffer, &size);

    offset = mail_cache_transaction_get_space(ctx, size, size, &size, TRUE);
    if (offset == 0)
        ret = -1;
    else if (pwrite_full(cache->fd, data, size, offset) < 0) {
        mail_cache_set_syscall_error(cache, "pwrite_full()");
        ret = -1;
    } else if (fdatasync(cache->fd) < 0) {
        mail_cache_set_syscall_error(cache, "fdatasync()");
        ret = -1;
    } else if (mail_cache_header_fields_get_next_offset(cache,
                                &hdr_offset) < 0)
        ret = -1;
    else {
        /* after it's guaranteed to be in disk, update header offset */
        offset = mail_cache_uint32_to_offset(offset);
        if (pwrite_full(cache->fd, &offset, sizeof(offset),
                hdr_offset) < 0) {
            mail_cache_set_syscall_error(cache, "pwrite_full()");
            ret = -1;
        } else {
            /* we'll need to fix mappings. */
            if (mail_cache_header_fields_read(cache) < 0)
                ret = -1;
        }
    }
    t_pop();

    mail_cache_unlock(cache);
    return ret;
}

void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
            unsigned int field, const void *data, size_t data_size)
{
    uint32_t file_field, data_size32;
    unsigned int fixed_size;
    size_t full_size;

    i_assert(field < ctx->cache->fields_count);
    i_assert(data_size < (uint32_t)-1);

    if (ctx->cache->fields[field].field.decision ==
        (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED))
        return;

    file_field = ctx->cache->field_file_map[field];
    if (file_field == (uint32_t)-1) {
        /* we'll have to add this field to headers */
        if (mail_cache_header_write_fields(ctx) < 0)
            return;

        file_field = ctx->cache->field_file_map[field];
        i_assert(file_field != (uint32_t)-1);
    }

    mail_cache_decision_add(ctx->view, seq, field);

    fixed_size = ctx->cache->fields[field].field.field_size;
    i_assert(fixed_size == (unsigned int)-1 || fixed_size == data_size);

    data_size32 = (uint32_t)data_size;

    if (ctx->prev_seq != seq) {
        mail_cache_transaction_switch_seq(ctx);
        ctx->prev_seq = seq;

        /* remember roughly what we have modified, so cache lookups can
           look into transactions to see changes. */
        if (seq < ctx->view->trans_seq1 || ctx->view->trans_seq1 == 0)
            ctx->view->trans_seq1 = seq;
        if (seq > ctx->view->trans_seq2)
            ctx->view->trans_seq2 = seq;
    }

    full_size = (data_size + 3) & ~3;
    if (fixed_size == (unsigned int)-1)
        full_size += sizeof(data_size32);

    if (buffer_get_used_size(ctx->cache_data) + full_size >
        buffer_get_size(ctx->cache_data)) {
        /* time to flush our buffer */
        if (mail_cache_transaction_flush(ctx) < 0)
            return;
    }

    buffer_append(ctx->cache_data, &file_field, sizeof(file_field));
    if (fixed_size == (unsigned int)-1) {
        buffer_append(ctx->cache_data, &data_size32,
                  sizeof(data_size32));
    }

    buffer_append(ctx->cache_data, data, data_size);
    if ((data_size & 3) != 0)
                buffer_append(ctx->cache_data, null4, 4 - (data_size & 3));
}

int mail_cache_transaction_lookup(struct mail_cache_transaction_ctx *ctx,
                  uint32_t seq, uint32_t *offset_r)
{
    return mail_index_update_cache_lookup(ctx->trans, seq, offset_r);
}


int mail_cache_link(struct mail_cache *cache, uint32_t old_offset,
            uint32_t new_offset)
{
    i_assert(cache->locked);

    if (new_offset + sizeof(struct mail_cache_record) >
        cache->hdr_copy.used_file_size) {
        mail_cache_set_corrupted(cache,
            "Cache record offset %u points outside file",
            new_offset);
        return -1;
    }

    new_offset += offsetof(struct mail_cache_record, prev_offset);
    if (pwrite_full(cache->fd, &old_offset,
            sizeof(old_offset), new_offset) < 0) {
        mail_cache_set_syscall_error(cache, "pwrite_full()");
        return -1;
    }

    cache->hdr_copy.continued_record_count++;
    cache->hdr_modified = TRUE;
    return 0;
}

int mail_cache_delete(struct mail_cache *cache, uint32_t offset)
{
    struct mail_cache_record *cache_rec;

    i_assert(cache->locked);

    cache_rec = mail_cache_get_record(cache, offset);
    if (cache_rec == NULL)
        return 0;

    /* we'll only update the deleted_space in header. we can't really
       do any actual deleting as other processes might still be using
       the data. also it's actually useful as some index views are still
       able to ask cached data from messages that have already been
       expunged. */
    do {
        cache->hdr_copy.deleted_space += cache_rec->size;
        cache_rec =
            mail_cache_get_record(cache, cache_rec->prev_offset);
    } while (cache_rec != NULL);

    cache->hdr_modified = TRUE;
    return 0;
}