index/dbox-multi/mdbox-save.c

	mdbox-save.c revision 9047d770bfbb93ab6af5363dedb2d01363877243
45312f52ff3a3d4c137447be4c7556500c2f8bf2Timo Sirainen/* Copyright (c) 2007-2011 Dovecot authors, see the included COPYING file */
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "lib.h"
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "array.h"
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "fdatasync-path.h"
65988f5a8abed57e9894fec77105941e046d3490Timo Sirainen#include "hex-binary.h"
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen#include "hex-dec.h"
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "str.h"
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "istream.h"
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "istream-crlf.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "ostream.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "write-full.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "index-mail.h"
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen#include "mail-copy.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "dbox-save.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "mdbox-storage.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "mdbox-map.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "mdbox-file.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include "mdbox-sync.h"
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen#include <stdlib.h>
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenstruct dbox_save_mail {
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    struct dbox_file_append_context *file_append;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    uint32_t seq;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    uint32_t append_offset;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen};
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenstruct mdbox_save_context {
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    struct dbox_save_context ctx;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    struct mdbox_mailbox *mbox;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    struct mdbox_sync_context *sync_ctx;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    struct dbox_file_append_context *cur_file_append;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    struct mdbox_map_append_context *append_ctx;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    ARRAY_TYPE(uint32_t) copy_map_uids;
1a1fcdbe27a8cee9a4c453a6b2f625a5be572a32Timo Sirainen    struct mdbox_map_atomic_context *atomic;
1a1fcdbe27a8cee9a4c453a6b2f625a5be572a32Timo Sirainen    struct mdbox_map_transaction_context *map_trans;
1a1fcdbe27a8cee9a4c453a6b2f625a5be572a32Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    ARRAY_DEFINE(mails, struct dbox_save_mail);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen};
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenstatic struct dbox_file *
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenmdbox_copy_file_get_file(struct mailbox_transaction_context *t,
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen             uint32_t seq, uoff_t *offset_r)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen{
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    struct mdbox_save_context *ctx =
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen        (struct mdbox_save_context *)t->save_ctx;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    const struct mdbox_mail_index_record *rec;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    const void *data;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    bool expunged;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    uint32_t file_id;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    mail_index_lookup_ext(t->view, seq, ctx->mbox->ext_id,
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen                  &data, &expunged);
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    rec = data;
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen    if (mdbox_map_lookup(ctx->mbox->storage->map, rec->map_uid,
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen                 &file_id, offset_r) < 0)
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen        i_unreached();
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen    return mdbox_file_init(ctx->mbox->storage, file_id);
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen}
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainenstruct dbox_file *
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainenmdbox_save_file_get_file(struct mailbox_transaction_context *t,
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen             uint32_t seq, uoff_t *offset_r)
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen{
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    struct mdbox_save_context *ctx =
7c424aa51c956c628e3512055841aa2f9eef4833Timo Sirainen        (struct mdbox_save_context *)t->save_ctx;
f923659c0e5298263d80622c99f4dc4132b4675bTimo Sirainen    const struct dbox_save_mail *mails, *mail;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    unsigned int count;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    mails = array_get(&ctx->mails, &count);
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen    i_assert(count > 0);
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    i_assert(seq >= mails[0].seq);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    mail = &mails[seq - mails[0].seq];
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    i_assert(mail->seq == seq);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    if (mail->file_append == NULL) {
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen        /* copied mail */
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen        return mdbox_copy_file_get_file(t, seq, offset_r);
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    }
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen    /* saved mail */
b014857be9961acf2d37ef7b76d941b20cc8c2d1Timo Sirainen    if (dbox_file_append_flush(mail->file_append) < 0)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen        ctx->ctx.failed = TRUE;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    mail->file_append->file->refcount++;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    *offset_r = mail->append_offset;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    return mail->file_append->file;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen}
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainenstruct mail_save_context *
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainenmdbox_save_alloc(struct mailbox_transaction_context *t)
fab050cbfdf3da692441d2e2fb4b2a4c6ac9e0daTimo Sirainen{
fab050cbfdf3da692441d2e2fb4b2a4c6ac9e0daTimo Sirainen    struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)t->box;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    struct mdbox_save_context *ctx =
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen        (struct mdbox_save_context *)t->save_ctx;
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen    if (ctx != NULL) {
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen        /* use the existing allocated structure */
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen        ctx->ctx.failed = FALSE;
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen        ctx->ctx.finished = FALSE;
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen        ctx->ctx.cur_file = NULL;
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen        ctx->ctx.dbox_output = NULL;
2a3fc652e13a574ca14ff2405b5c29a59232db49Timo Sirainen        ctx->cur_file_append = NULL;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen        return &ctx->ctx.ctx;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    }
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    ctx = i_new(struct mdbox_save_context, 1);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    ctx->ctx.ctx.transaction = t;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    ctx->ctx.trans = t->itrans;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    ctx->mbox = mbox;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    ctx->atomic = mdbox_map_atomic_begin(mbox->storage->map);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    ctx->append_ctx = mdbox_map_append_begin(ctx->atomic);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    i_array_init(&ctx->mails, 32);
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    t->save_ctx = &ctx->ctx.ctx;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen    return t->save_ctx;
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen}
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainenint mdbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
812ac1e2570c600a086c09b24d250224a822a97dTimo Sirainen{
d65a556a5ec078cd7f1d0060adb16fc860d66b27Timo Sirainen    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
d65a556a5ec078cd7f1d0060adb16fc860d66b27Timo Sirainen    struct dbox_save_mail *save_mail;
d65a556a5ec078cd7f1d0060adb16fc860d66b27Timo Sirainen    uoff_t mail_size, append_offset;
e4ded29bff0662a590c2439ef2df8cda8a7cdd9bTimo Sirainen
e4ded29bff0662a590c2439ef2df8cda8a7cdd9bTimo Sirainen    /* get the size of the mail to be saved, if possible */
847aeef259d42e2f14cf126699e28291e6e1fb53Timo Sirainen    if (i_stream_get_size(input, TRUE, &mail_size) <= 0) {
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen        const struct stat *st;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen        /* we couldn't find out the exact size. fallback to non-exact,
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen           maybe it'll give something useful. the mail size is used
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen           only to figure out if it's causing mdbox file to grow
345253fb28498b2e0a60f4a2a8644c65feee7e75Timo Sirainen           too large. */
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen        st = i_stream_stat(input, FALSE);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen        mail_size = st->st_size > 0 ? st->st_size : 0;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    }
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    if (mdbox_map_append_next(ctx->append_ctx, mail_size, 0,
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen                  &ctx->cur_file_append,
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen                  &ctx->ctx.dbox_output) < 0) {
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen        ctx->ctx.failed = TRUE;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen        return -1;
a4ee24a4d5eefa80bbefc5acba16587ae36c3b5bTimo Sirainen    }
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    i_assert(ctx->ctx.dbox_output->offset <= (uint32_t)-1);
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen    append_offset = ctx->ctx.dbox_output->offset;
a93de780c3b78cfaace287026e468f3c3e34683aTimo Sirainen
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen    ctx->ctx.cur_file = ctx->cur_file_append->file;
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen    dbox_save_begin(&ctx->ctx, input);
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen    save_mail = array_append_space(&ctx->mails);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    save_mail->file_append = ctx->cur_file_append;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    save_mail->seq = ctx->ctx.seq;
345253fb28498b2e0a60f4a2a8644c65feee7e75Timo Sirainen    save_mail->append_offset = append_offset;
65988f5a8abed57e9894fec77105941e046d3490Timo Sirainen    return ctx->ctx.failed ? -1 : 0;
65988f5a8abed57e9894fec77105941e046d3490Timo Sirainen}
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainenstatic int mdbox_save_mail_write_metadata(struct mdbox_save_context *ctx,
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen                      struct dbox_save_mail *mail)
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen{
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    struct dbox_file *file = mail->file_append->file;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    struct dbox_message_header dbox_msg_hdr;
a4ee24a4d5eefa80bbefc5acba16587ae36c3b5bTimo Sirainen    uoff_t message_size;
a4ee24a4d5eefa80bbefc5acba16587ae36c3b5bTimo Sirainen    guid_128_t guid_128;
af6c7862e6160ffaecec458f4cec43b94272ad57Timo Sirainen
af6c7862e6160ffaecec458f4cec43b94272ad57Timo Sirainen    i_assert(file->msg_header_size == sizeof(dbox_msg_hdr));
af6c7862e6160ffaecec458f4cec43b94272ad57Timo Sirainen
af6c7862e6160ffaecec458f4cec43b94272ad57Timo Sirainen    message_size = ctx->ctx.dbox_output->offset -
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen        mail->append_offset - mail->file_append->file->msg_header_size;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    dbox_save_write_metadata(&ctx->ctx.ctx, ctx->ctx.dbox_output,
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen                 message_size, ctx->mbox->box.name, guid_128);
055f4599bba1874fa1148a8fa488517fa077619cTimo Sirainen    /* save the 128bit GUID to index so if the map index gets corrupted
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen       we can still find the message */
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen                  ctx->mbox->guid_ext_id, guid_128, NULL);
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen    dbox_msg_header_fill(&dbox_msg_hdr, message_size);
345253fb28498b2e0a60f4a2a8644c65feee7e75Timo Sirainen    if (o_stream_pwrite(ctx->ctx.dbox_output, &dbox_msg_hdr,
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen                sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen        dbox_file_set_syscall_error(file, "pwrite()");
a93de780c3b78cfaace287026e468f3c3e34683aTimo Sirainen        return -1;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    }
bc2d4f1c18222a3bd2a6b2b8b5f6abb560a865b3Timo Sirainen    return 0;
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen}
345253fb28498b2e0a60f4a2a8644c65feee7e75Timo Sirainen
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainenstatic int mdbox_save_finish_write(struct mail_save_context *_ctx)
a93de780c3b78cfaace287026e468f3c3e34683aTimo Sirainen{
72388282bf6718c39af34cfcf51438910f9d62daTimo Sirainen    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
    struct dbox_save_mail *mail;

    ctx->ctx.finished = TRUE;
    if (ctx->ctx.dbox_output == NULL)
        return -1;

    dbox_save_end(&ctx->ctx);
    index_mail_cache_parse_deinit(_ctx->dest_mail,
                      _ctx->received_date, !ctx->ctx.failed);

    mail = array_idx_modifiable(&ctx->mails, array_count(&ctx->mails) - 1);
    if (!ctx->ctx.failed) T_BEGIN {
        if (mdbox_save_mail_write_metadata(ctx, mail) < 0)
            ctx->ctx.failed = TRUE;
        else
            mdbox_map_append_finish(ctx->append_ctx);
    } T_END;

    if (mail->file_append->file->input != NULL) {
        /* if we try to read the saved mail before unlocking file,
           make sure the input stream doesn't have stale data */
        i_stream_sync(mail->file_append->file->input);
    }
    i_stream_unref(&ctx->ctx.input);

    if (ctx->ctx.failed) {
        mail_index_expunge(ctx->ctx.trans, ctx->ctx.seq);
        mdbox_map_append_abort(ctx->append_ctx);
        array_delete(&ctx->mails, array_count(&ctx->mails) - 1, 1);
        return -1;
    }
    return 0;
}

int mdbox_save_finish(struct mail_save_context *ctx)
{
    int ret;

    ret = mdbox_save_finish_write(ctx);
    index_save_context_free(ctx);
    return ret;
}

void mdbox_save_cancel(struct mail_save_context *_ctx)
{
    struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;

    ctx->failed = TRUE;
    (void)mdbox_save_finish(_ctx);
}

static void
mdbox_save_set_map_uids(struct mdbox_save_context *ctx,
            uint32_t first_map_uid, uint32_t last_map_uid)
{
    struct mdbox_mailbox *mbox = ctx->mbox;
    struct mail_index_view *view = ctx->ctx.ctx.transaction->view;
    const struct mdbox_mail_index_record *old_rec;
    struct mdbox_mail_index_record rec;
    const struct dbox_save_mail *mails;
    unsigned int i, count;
    const void *data;
    bool expunged;
    uint32_t next_map_uid = first_map_uid;

    mdbox_update_header(mbox, ctx->ctx.trans, NULL);

    memset(&rec, 0, sizeof(rec));
    rec.save_date = ioloop_time;
    mails = array_get(&ctx->mails, &count);
    for (i = 0; i < count; i++) {
        mail_index_lookup_ext(view, mails[i].seq, mbox->ext_id,
                      &data, &expunged);
        old_rec = data;
        if (old_rec != NULL && old_rec->map_uid != 0) {
            /* message was copied. keep the existing map uid */
            continue;
        }

        rec.map_uid = next_map_uid++;
        mail_index_update_ext(ctx->ctx.trans, mails[i].seq,
                      mbox->ext_id, &rec, NULL);
    }
    i_assert(next_map_uid == last_map_uid + 1);
}

int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
    struct mailbox_transaction_context *_t = _ctx->transaction;
    const struct mail_index_header *hdr;
    uint32_t first_map_uid, last_map_uid;

    i_assert(ctx->ctx.finished);

    /* make sure the map gets locked */
    if (mdbox_map_atomic_lock(ctx->atomic) < 0) {
        mdbox_transaction_save_rollback(_ctx);
        return -1;
    }

    /* assign map UIDs for newly saved messages. they're written to
       transaction log immediately within this function, but the map
       is left locked. */
    if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid,
                         &last_map_uid) < 0) {
        mdbox_transaction_save_rollback(_ctx);
        return -1;
    }

    /* lock the mailbox after map to avoid deadlocks. if we've noticed
       any corruption, deal with it later, otherwise we won't have
       up-to-date atomic->sync_view */
    if (mdbox_sync_begin(ctx->mbox, MDBOX_SYNC_FLAG_NO_PURGE |
                 MDBOX_SYNC_FLAG_FORCE |
                 MDBOX_SYNC_FLAG_FSYNC |
                 MDBOX_SYNC_FLAG_NO_REBUILD, ctx->atomic,
                 &ctx->sync_ctx) < 0) {
        mdbox_transaction_save_rollback(_ctx);
        return -1;
    }

    /* assign UIDs for new messages */
    hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
    mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid,
                      &_t->changes->saved_uids);

    /* save map UIDs to mailbox index */
    if (first_map_uid != 0)
        mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid);

    /* increase map's refcount for copied mails */
    if (array_is_created(&ctx->copy_map_uids)) {
        ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE);
        if (mdbox_map_update_refcounts(ctx->map_trans,
                           &ctx->copy_map_uids, 1) < 0) {
            mdbox_transaction_save_rollback(_ctx);
            return -1;
        }
    }

    if (ctx->ctx.mail != NULL)
        mail_free(&ctx->ctx.mail);

    _t->changes->uid_validity = hdr->uid_validity;
    return 0;
}

void mdbox_transaction_save_commit_post(struct mail_save_context *_ctx,
                    struct mail_index_transaction_commit_result *result)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
    struct mail_storage *storage = _ctx->transaction->box->storage;

    _ctx->transaction = NULL; /* transaction is already freed */

    mail_index_sync_set_commit_result(ctx->sync_ctx->index_sync_ctx,
                      result);

    /* finish writing the mailbox APPENDs */
    if (mdbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) {
        /* commit refcount increases for copied mails */
        if (ctx->map_trans != NULL) {
            if (mdbox_map_transaction_commit(ctx->map_trans) < 0)
                mdbox_map_atomic_set_failed(ctx->atomic);
        }
        /* flush file append writes */
        if (mdbox_map_append_commit(ctx->append_ctx) < 0)
            mdbox_map_atomic_set_failed(ctx->atomic);
    }
    mdbox_map_append_free(&ctx->append_ctx);
    /* update the sync tail offset, everything else
       was already written at this point. */
    (void)mdbox_map_atomic_finish(&ctx->atomic);

    if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) {
        const char *box_path = mailbox_get_path(&ctx->mbox->box);

        if (fdatasync_path(box_path) < 0) {
            mail_storage_set_critical(storage,
                "fdatasync_path(%s) failed: %m", box_path);
        }
    }
    mdbox_transaction_save_rollback(_ctx);
}

void mdbox_transaction_save_rollback(struct mail_save_context *_ctx)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;

    if (!ctx->ctx.finished)
        mdbox_save_cancel(&ctx->ctx.ctx);
    if (ctx->append_ctx != NULL)
        mdbox_map_append_free(&ctx->append_ctx);
    if (ctx->map_trans != NULL)
        mdbox_map_transaction_free(&ctx->map_trans);
    if (ctx->atomic != NULL)
        (void)mdbox_map_atomic_finish(&ctx->atomic);
    if (array_is_created(&ctx->copy_map_uids))
        array_free(&ctx->copy_map_uids);

    if (ctx->sync_ctx != NULL)
        (void)mdbox_sync_finish(&ctx->sync_ctx, FALSE);

    if (ctx->ctx.mail != NULL)
        mail_free(&ctx->ctx.mail);
    array_free(&ctx->mails);
    i_free(ctx);
}

int mdbox_copy(struct mail_save_context *_ctx, struct mail *mail)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
    struct dbox_save_mail *save_mail;
    struct mdbox_mailbox *src_mbox;
    struct mdbox_mail_index_record rec;
    const void *data;
    bool expunged;

    ctx->ctx.finished = TRUE;

    if (mail->box->storage != _ctx->transaction->box->storage ||
        _ctx->transaction->box->disable_reflink_copy_to ||
        _ctx->guid != NULL)
        return mail_storage_copy(_ctx, mail);
    src_mbox = (struct mdbox_mailbox *)mail->box;

    memset(&rec, 0, sizeof(rec));
    rec.save_date = ioloop_time;
    if (mdbox_mail_lookup(src_mbox, mail->transaction->view, mail->seq,
                  &rec.map_uid) < 0)
        return -1;

    /* remember the map_uid so we can later increase its refcount */
    if (!array_is_created(&ctx->copy_map_uids))
        i_array_init(&ctx->copy_map_uids, 32);
    array_append(&ctx->copy_map_uids, &rec.map_uid, 1);

    /* add message to mailbox index */
    dbox_save_add_to_index(&ctx->ctx);
    mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
                  ctx->mbox->ext_id, &rec, NULL);

    mail_index_lookup_ext(mail->transaction->view, mail->seq,
                  src_mbox->guid_ext_id, &data, &expunged);
    if (data != NULL) {
        mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
                      ctx->mbox->guid_ext_id, data, NULL);
    }
    index_copy_cache_fields(_ctx, mail, ctx->ctx.seq);

    save_mail = array_append_space(&ctx->mails);
    save_mail->seq = ctx->ctx.seq;

    if (_ctx->dest_mail != NULL)
        mail_set_seq_saving(_ctx->dest_mail, ctx->ctx.seq);
    return 0;
}