index/dbox-single/sdbox-save.c

	sdbox-save.c revision 9047d770bfbb93ab6af5363dedb2d01363877243
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen/* Copyright (c) 2007-2011 Dovecot authors, see the included COPYING file */
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen#include "lib.h"
767431e5084a037c4dbefdf30ebfa03c84b1f449Timo Sirainen#include "array.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "fdatasync-path.h"
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen#include "hex-binary.h"
5fb3f13537dffd15a31e997da133a721c0728af8Timo Sirainen#include "hex-dec.h"
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen#include "str.h"
97437f768d1a3e6134fed1971202803fd250eef2Timo Sirainen#include "istream.h"
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen#include "istream-crlf.h"
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen#include "ostream.h"
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen#include "write-full.h"
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen#include "index-mail.h"
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainen#include "mail-copy.h"
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen#include "dbox-attachment.h"
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen#include "dbox-save.h"
e09c7dc961cb9cab04ec7cc79215c2f6318fbde0Timo Sirainen#include "sdbox-storage.h"
23878bd03d1de531e3261a25598beec621351910Timo Sirainen#include "sdbox-file.h"
23878bd03d1de531e3261a25598beec621351910Timo Sirainen#include "sdbox-sync.h"
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen#include <stdlib.h>
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainen
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainenstruct sdbox_save_context {
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainen    struct dbox_save_context ctx;
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    struct sdbox_mailbox *mbox;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    struct sdbox_sync_context *sync_ctx;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    struct dbox_file_append_context *append_ctx;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    uint32_t first_saved_seq;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    ARRAY_DEFINE(files, struct dbox_file *);
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen};
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainenstruct dbox_file *
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainensdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq)
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen{
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    struct sdbox_save_context *ctx =
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen        (struct sdbox_save_context *)t->save_ctx;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    struct dbox_file *const *files;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    unsigned int count;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    i_assert(seq >= ctx->first_saved_seq);
7358272563d8ef77366447708ab0e58c0cff4151Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    files = array_get(&ctx->files, &count);
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    i_assert(count > 0);
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    i_assert(seq - ctx->first_saved_seq < count);
7358272563d8ef77366447708ab0e58c0cff4151Timo Sirainen
7358272563d8ef77366447708ab0e58c0cff4151Timo Sirainen    return files[seq - ctx->first_saved_seq];
97437f768d1a3e6134fed1971202803fd250eef2Timo Sirainen}
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainenstruct mail_save_context *
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainensdbox_save_alloc(struct mailbox_transaction_context *t)
23878bd03d1de531e3261a25598beec621351910Timo Sirainen{
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)t->box;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    struct sdbox_save_context *ctx =
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen        (struct sdbox_save_context *)t->save_ctx;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    if (ctx != NULL) {
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen        /* use the existing allocated structure */
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen        ctx->ctx.failed = FALSE;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen        ctx->ctx.finished = FALSE;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen        ctx->ctx.cur_file = NULL;
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen        ctx->ctx.dbox_output = NULL;
0f9a8663b0ff6fe30389d02284a2b002c40914ebTimo Sirainen        return &ctx->ctx.ctx;
bd417d416988d11a6b555b9aa57779e7ed976951Timo Sirainen    }
a9efdb661eb7a8a33aacfdcc3486dcc675a21543Timo Sirainen
a9efdb661eb7a8a33aacfdcc3486dcc675a21543Timo Sirainen    ctx = i_new(struct sdbox_save_context, 1);
fab850a6aee4aaef4f4795bd7946807a3ba45041Timo Sirainen    ctx->ctx.ctx.transaction = t;
bd417d416988d11a6b555b9aa57779e7ed976951Timo Sirainen    ctx->ctx.trans = t->itrans;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    ctx->mbox = mbox;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    i_array_init(&ctx->files, 32);
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    t->save_ctx = &ctx->ctx.ctx;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    return t->save_ctx;
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen}
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainenvoid sdbox_save_add_file(struct mail_save_context *_ctx, struct dbox_file *file)
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen{
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen    struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen
97437f768d1a3e6134fed1971202803fd250eef2Timo Sirainen    if (ctx->first_saved_seq == 0)
97437f768d1a3e6134fed1971202803fd250eef2Timo Sirainen        ctx->first_saved_seq = ctx->ctx.seq;
97afa073e3e1e0301dc41173ec34beb08edcce50Timo Sirainen    array_append(&ctx->files, &file, 1);
97afa073e3e1e0301dc41173ec34beb08edcce50Timo Sirainen}
b8b085f7bc6f1c0367802a9f00062bbbd981690dTimo Sirainen
b8b085f7bc6f1c0367802a9f00062bbbd981690dTimo Sirainenint sdbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
b8b085f7bc6f1c0367802a9f00062bbbd981690dTimo Sirainen{
94ba4820927b906b333e39445c1508a29387c3aaTimo Sirainen    struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainen    struct dbox_file *file;
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    int ret;
23878bd03d1de531e3261a25598beec621351910Timo Sirainen
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    file = sdbox_file_create(ctx->mbox);
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    ctx->append_ctx = dbox_file_append_init(file);
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    ret = dbox_file_get_append_stream(ctx->append_ctx,
23878bd03d1de531e3261a25598beec621351910Timo Sirainen                      &ctx->ctx.dbox_output);
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    if (ret <= 0) {
23878bd03d1de531e3261a25598beec621351910Timo Sirainen        i_assert(ret != 0);
23878bd03d1de531e3261a25598beec621351910Timo Sirainen        dbox_file_append_rollback(&ctx->append_ctx);
23878bd03d1de531e3261a25598beec621351910Timo Sirainen        dbox_file_unref(&file);
23878bd03d1de531e3261a25598beec621351910Timo Sirainen        ctx->ctx.failed = TRUE;
23878bd03d1de531e3261a25598beec621351910Timo Sirainen        return -1;
ecbbdf594f9329fc15a182bd6c7c4a7fb144ed74Timo Sirainen    }
ecbbdf594f9329fc15a182bd6c7c4a7fb144ed74Timo Sirainen    ctx->ctx.cur_file = file;
b8b085f7bc6f1c0367802a9f00062bbbd981690dTimo Sirainen    dbox_save_begin(&ctx->ctx, input);
94ba4820927b906b333e39445c1508a29387c3aaTimo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    sdbox_save_add_file(_ctx, file);
    return ctx->ctx.failed ? -1 : 0;
}

static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx,
                     struct dbox_file *file)
{
    struct sdbox_file *sfile = (struct sdbox_file *)file;
    const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr;
    const struct mail_attachment_extref *extrefs;
    struct dbox_message_header dbox_msg_hdr;
    uoff_t message_size;
    guid_128_t guid_128;
    unsigned int i, count;

    i_assert(file->msg_header_size == sizeof(dbox_msg_hdr));

    message_size = ctx->dbox_output->offset -
        file->msg_header_size - file->file_header_size;

    dbox_save_write_metadata(&ctx->ctx, ctx->dbox_output,
                 message_size, NULL, guid_128);
    dbox_msg_header_fill(&dbox_msg_hdr, message_size);
    if (o_stream_pwrite(ctx->dbox_output, &dbox_msg_hdr,
                sizeof(dbox_msg_hdr),
                file->file_header_size) < 0) {
        dbox_file_set_syscall_error(file, "pwrite()");
        return -1;
    }

    /* remember the attachment paths until commit time */
    extrefs_arr = index_attachment_save_get_extrefs(&ctx->ctx);
    if (extrefs_arr != NULL)
        extrefs = array_get(extrefs_arr, &count);
    else {
        extrefs = NULL;
        count = 0;
    }
    if (count > 0) {
        sfile->attachment_pool =
            pool_alloconly_create("sdbox attachment paths", 512);
        p_array_init(&sfile->attachment_paths,
                 sfile->attachment_pool, count);
        for (i = 0; i < count; i++) {
            const char *path = p_strdup(sfile->attachment_pool,
                            extrefs[i].path);
            array_append(&sfile->attachment_paths, &path, 1);
        }
    }
    return 0;
}

static int dbox_save_finish_write(struct mail_save_context *_ctx)
{
    struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
    struct dbox_file *const *files;

    ctx->ctx.finished = TRUE;
    if (ctx->ctx.dbox_output == NULL)
        return -1;

    if (_ctx->save_date != (time_t)-1) {
        /* we can't change ctime, but we can add the date to cache */
        struct index_mail *mail = (struct index_mail *)_ctx->dest_mail;
        uint32_t t = _ctx->save_date;

        index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t));
    }

    index_mail_cache_parse_deinit(_ctx->dest_mail,
                      _ctx->received_date, !ctx->ctx.failed);

    files = array_idx_modifiable(&ctx->files, array_count(&ctx->files) - 1);

    dbox_save_end(&ctx->ctx);
    if (!ctx->ctx.failed) T_BEGIN {
        if (dbox_save_mail_write_metadata(&ctx->ctx, *files) < 0)
            ctx->ctx.failed = TRUE;
    } T_END;

    if (ctx->ctx.failed) {
        mail_index_expunge(ctx->ctx.trans, ctx->ctx.seq);
        dbox_file_append_rollback(&ctx->append_ctx);
    } else {
        dbox_file_append_checkpoint(ctx->append_ctx);
        if (dbox_file_append_commit(&ctx->append_ctx) < 0)
            ctx->ctx.failed = TRUE;
    }

    i_stream_unref(&ctx->ctx.input);
    dbox_file_close(*files);
    ctx->ctx.dbox_output = NULL;

    return ctx->ctx.failed ? -1 : 0;
}

int sdbox_save_finish(struct mail_save_context *ctx)
{
    int ret;

    ret = dbox_save_finish_write(ctx);
    index_save_context_free(ctx);
    return ret;
}

void sdbox_save_cancel(struct mail_save_context *_ctx)
{
    struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;

    ctx->failed = TRUE;
    (void)sdbox_save_finish(_ctx);
}

static int dbox_save_assign_uids(struct sdbox_save_context *ctx,
                 const ARRAY_TYPE(seq_range) *uids)
{
    struct dbox_file *const *files;
    struct seq_range_iter iter;
    unsigned int i, count, n = 0;
    uint32_t uid;
    bool ret;

    seq_range_array_iter_init(&iter, uids);
    files = array_get(&ctx->files, &count);
    for (i = 0; i < count; i++) {
        struct sdbox_file *sfile = (struct sdbox_file *)files[i];

        ret = seq_range_array_iter_nth(&iter, n++, &uid);
        i_assert(ret);
        if (sdbox_file_assign_uid(sfile, uid) < 0)
            return -1;
    }
    i_assert(!seq_range_array_iter_nth(&iter, n, &uid));
    return 0;
}

static void dbox_save_unref_files(struct sdbox_save_context *ctx)
{
    struct dbox_file **files;
    unsigned int i, count;

    files = array_get_modifiable(&ctx->files, &count);
    for (i = 0; i < count; i++) {
        if (ctx->ctx.failed) {
            struct sdbox_file *sfile =
                (struct sdbox_file *)files[i];

            (void)sdbox_file_unlink_aborted_save(sfile);
        }
        dbox_file_unref(&files[i]);
    }
    array_free(&ctx->files);
}

int sdbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
{
    struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
    struct mailbox_transaction_context *_t = _ctx->transaction;
    const struct mail_index_header *hdr;

    i_assert(ctx->ctx.finished);

    if (array_count(&ctx->files) == 0) {
        /* the mail must be freed in the commit_pre() */
        if (ctx->ctx.mail != NULL)
            mail_free(&ctx->ctx.mail);
        return 0;
    }

    if (sdbox_sync_begin(ctx->mbox, SDBOX_SYNC_FLAG_FORCE |
                 SDBOX_SYNC_FLAG_FSYNC, &ctx->sync_ctx) < 0) {
        sdbox_transaction_save_rollback(_ctx);
        return -1;
    }

    /* assign UIDs for new messages */
    hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
    mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid,
                      &_t->changes->saved_uids);
    if (dbox_save_assign_uids(ctx, &_t->changes->saved_uids) < 0) {
        sdbox_transaction_save_rollback(_ctx);
        return -1;
    }

    if (ctx->ctx.mail != NULL)
        mail_free(&ctx->ctx.mail);

    _t->changes->uid_validity = hdr->uid_validity;
    return 0;
}

void sdbox_transaction_save_commit_post(struct mail_save_context *_ctx,
                    struct mail_index_transaction_commit_result *result)
{
    struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
    struct mail_storage *storage = _ctx->transaction->box->storage;

    _ctx->transaction = NULL; /* transaction is already freed */

    if (array_count(&ctx->files) == 0) {
        sdbox_transaction_save_rollback(_ctx);
        return;
    }

    mail_index_sync_set_commit_result(ctx->sync_ctx->index_sync_ctx,
                      result);

    if (sdbox_sync_finish(&ctx->sync_ctx, TRUE) < 0)
        ctx->ctx.failed = TRUE;

    if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) {
        const char *box_path = mailbox_get_path(&ctx->mbox->box);

        if (fdatasync_path(box_path) < 0) {
            mail_storage_set_critical(storage,
                "fdatasync_path(%s) failed: %m", box_path);
        }
    }
    sdbox_transaction_save_rollback(_ctx);
}

void sdbox_transaction_save_rollback(struct mail_save_context *_ctx)
{
    struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;

    if (!ctx->ctx.finished)
        sdbox_save_cancel(_ctx);
    dbox_save_unref_files(ctx);

    if (ctx->sync_ctx != NULL)
        (void)sdbox_sync_finish(&ctx->sync_ctx, FALSE);

    if (ctx->ctx.mail != NULL)
        mail_free(&ctx->ctx.mail);
    i_free(ctx);
}