index/dbox-multi/mdbox-save.c

	mdbox-save.c revision 5e702db5540b2303e25554dee21bbf35a4813381
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "lib.h"
49e358eebea107aad9919dcc4bd88cee8519ba2eTimo Sirainen#include "array.h"
49e358eebea107aad9919dcc4bd88cee8519ba2eTimo Sirainen#include "fdatasync-path.h"
49e358eebea107aad9919dcc4bd88cee8519ba2eTimo Sirainen#include "hex-binary.h"
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen#include "hex-dec.h"
dd62b77c932d1b518f2a3e4bf80e36542becc256Timo Sirainen#include "str.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "istream.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "istream-crlf.h"
03f5c621d06d6b6d77a145196c9633a7aa64dc78Timo Sirainen#include "ostream.h"
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen#include "write-full.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "index-mail.h"
411d6baa37f31d90730e90c4a28c43e1974bbe58Timo Sirainen#include "mail-copy.h"
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen#include "dbox-save.h"
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen#include "mdbox-storage.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "mdbox-map.h"
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen#include "mdbox-file.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "mdbox-sync.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
2526d52441ef368215ab6bf04fd0356d3b09d235Timo Sirainen#include <stdlib.h>
2526d52441ef368215ab6bf04fd0356d3b09d235Timo Sirainen
fe363b433b8038a69b55169da9dca27892ad7d18Timo Sirainenstruct dbox_save_mail {
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    struct dbox_file_append_context *file_append;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen    uint32_t seq;
fe363b433b8038a69b55169da9dca27892ad7d18Timo Sirainen    uint32_t append_offset;
fe363b433b8038a69b55169da9dca27892ad7d18Timo Sirainen};
dd62b77c932d1b518f2a3e4bf80e36542becc256Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainenstruct mdbox_save_context {
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen    struct dbox_save_context ctx;
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen    struct mdbox_mailbox *mbox;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    struct mdbox_sync_context *sync_ctx;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    struct dbox_file_append_context *cur_file_append;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    struct dbox_map_append_context *append_ctx;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    ARRAY_TYPE(uint32_t) copy_map_uids;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    struct dbox_map_transaction_context *map_trans;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    ARRAY_DEFINE(mails, struct dbox_save_mail);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen};
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainenstruct dbox_file *
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainenmdbox_save_file_get_file(struct mailbox_transaction_context *t,
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainen             uint32_t seq, uoff_t *offset_r)
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainen{
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    struct mdbox_save_context *ctx =
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        (struct mdbox_save_context *)t->save_ctx;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    const struct dbox_save_mail *mails, *mail;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    unsigned int count;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    mails = array_get(&ctx->mails, &count);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    i_assert(count > 0);
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    i_assert(seq >= mails[0].seq);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    mail = &mails[seq - mails[0].seq];
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    i_assert(mail->seq == seq);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    if (dbox_file_append_flush(mail->file_append) < 0)
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen        ctx->ctx.failed = TRUE;
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen    *offset_r = mail->append_offset;
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen    return mail->file_append->file;
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen}
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainenstruct mail_save_context *
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainenmdbox_save_alloc(struct mailbox_transaction_context *t)
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen{
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen    struct index_transaction_context *it =
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen        (struct index_transaction_context *)t;
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen    struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)t->box;
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen    struct mdbox_save_context *ctx =
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen        (struct mdbox_save_context *)t->save_ctx;
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen    i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen
89e195dfb5c4b0efd9b9f459771a4467674e5b1fTimo Sirainen    if (ctx != NULL) {
08fb191d6148feb3ed14e2d6c625cd248dd1c1d4Timo Sirainen        /* use the existing allocated structure */
08fb191d6148feb3ed14e2d6c625cd248dd1c1d4Timo Sirainen        ctx->ctx.finished = FALSE;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen        return &ctx->ctx.ctx;
89e195dfb5c4b0efd9b9f459771a4467674e5b1fTimo Sirainen    }
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen
89e195dfb5c4b0efd9b9f459771a4467674e5b1fTimo Sirainen    ctx = i_new(struct mdbox_save_context, 1);
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen    ctx->ctx.ctx.transaction = t;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    ctx->ctx.trans = it->trans;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    ctx->mbox = mbox;
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen    ctx->append_ctx = dbox_map_append_begin(mbox->storage->map, 0);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    i_array_init(&ctx->mails, 32);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    t->save_ctx = &ctx->ctx.ctx;
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen    return t->save_ctx;
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen}
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainenint mdbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
65cb456a072219fa35b55695d476b0bf51e2d735Timo Sirainen{
65cb456a072219fa35b55695d476b0bf51e2d735Timo Sirainen    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen    struct dbox_save_mail *save_mail;
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen    uoff_t mail_size, append_offset;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    /* get the size of the mail to be saved, if possible */
    if (i_stream_get_size(input, TRUE, &mail_size) <= 0) {
        const struct stat *st;

        /* we couldn't find out the exact size. fallback to non-exact,
           maybe it'll give something useful. the mail size is used
           only to figure out if it's causing mdbox file to grow
           too large. */
        st = i_stream_stat(input, FALSE);
        mail_size = st->st_size > 0 ? st->st_size : 0;
    }
    if (dbox_map_append_next(ctx->append_ctx, mail_size,
                 &ctx->cur_file_append,
                 &ctx->ctx.cur_output) < 0) {
        ctx->ctx.failed = TRUE;
        return -1;
    }
    i_assert(ctx->ctx.cur_output->offset <= (uint32_t)-1);
    append_offset = ctx->ctx.cur_output->offset;

    ctx->ctx.cur_file = ctx->cur_file_append->file;
    dbox_save_begin(&ctx->ctx, input);

    save_mail = array_append_space(&ctx->mails);
    save_mail->file_append = ctx->cur_file_append;
    save_mail->seq = ctx->ctx.seq;
    save_mail->append_offset = append_offset;
    return ctx->ctx.failed ? -1 : 0;
}

static int mdbox_save_mail_write_metadata(struct mdbox_save_context *ctx,
                      struct dbox_save_mail *mail)
{
    struct dbox_file *file = mail->file_append->file;
    struct dbox_message_header dbox_msg_hdr;
    uoff_t message_size;
    uint8_t guid_128[MAIL_GUID_128_SIZE];

    i_assert(file->msg_header_size == sizeof(dbox_msg_hdr));

    message_size = ctx->ctx.cur_output->offset -
        mail->append_offset - mail->file_append->file->msg_header_size;

    dbox_save_write_metadata(&ctx->ctx.ctx, ctx->ctx.cur_output,
                 ctx->mbox->ibox.box.name, guid_128);
    /* save the 128bit GUID to index so if the map index gets corrupted
       we can still find the message */
    mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
                  ctx->mbox->guid_ext_id, guid_128, NULL);

    dbox_msg_header_fill(&dbox_msg_hdr, message_size);
    if (o_stream_pwrite(ctx->ctx.cur_output, &dbox_msg_hdr,
                sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
        dbox_file_set_syscall_error(file, "pwrite()");
        return -1;
    }
    return 0;
}

static int mdbox_save_finish_write(struct mail_save_context *_ctx)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
    struct dbox_save_mail *mails;

    ctx->ctx.finished = TRUE;
    if (ctx->ctx.cur_output == NULL)
        return -1;

    index_mail_cache_parse_deinit(_ctx->dest_mail,
                      _ctx->received_date, !ctx->ctx.failed);

    mails = array_idx_modifiable(&ctx->mails, array_count(&ctx->mails) - 1);
    if (!ctx->ctx.failed) T_BEGIN {
        if (mdbox_save_mail_write_metadata(ctx, mails) < 0)
            ctx->ctx.failed = TRUE;
        else
            dbox_map_append_finish(ctx->append_ctx);
    } T_END;

    i_stream_unref(&ctx->ctx.input);

    if (ctx->ctx.failed) {
        array_delete(&ctx->mails, array_count(&ctx->mails) - 1, 1);
        return -1;
    }
    return 0;
}

int mdbox_save_finish(struct mail_save_context *ctx)
{
    int ret;

    ret = mdbox_save_finish_write(ctx);
    index_save_context_free(ctx);
    return ret;
}

void mdbox_save_cancel(struct mail_save_context *_ctx)
{
    struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;

    ctx->failed = TRUE;
    (void)mdbox_save_finish(_ctx);
}

int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
    struct mailbox_transaction_context *_t = _ctx->transaction;
    struct mdbox_mailbox *mbox = ctx->mbox;
    const struct mail_index_header *hdr;
    uint32_t first_map_uid, last_map_uid;

    i_assert(ctx->ctx.finished);

    /* lock the mailbox before map to avoid deadlocks */
    if (mdbox_sync_begin(mbox, MDBOX_SYNC_FLAG_NO_PURGE |
                 MDBOX_SYNC_FLAG_FORCE |
                 MDBOX_SYNC_FLAG_FSYNC, &ctx->sync_ctx) < 0) {
        mdbox_transaction_save_rollback(_ctx);
        return -1;
    }

    /* get map UIDs for messages saved to multi-files. they're written
       to transaction log immediately within this function, but the map
       is left locked. */
    if (dbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid,
                        &last_map_uid) < 0) {
        mdbox_transaction_save_rollback(_ctx);
        return -1;
    }

    /* assign UIDs for new messages */
    hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
    mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid,
                      &_t->changes->saved_uids);

    /* add map_uids for all messages saved to multi-files */
    if (first_map_uid != 0) {
        struct mdbox_mail_index_record rec;
        const struct dbox_save_mail *mails;
        unsigned int i, count;
        uint32_t next_map_uid = first_map_uid;

        mdbox_update_header(mbox, ctx->ctx.trans, NULL);

        memset(&rec, 0, sizeof(rec));
        rec.save_date = ioloop_time;
        mails = array_get(&ctx->mails, &count);
        for (i = 0; i < count; i++) {
            rec.map_uid = next_map_uid++;
            mail_index_update_ext(ctx->ctx.trans, mails[i].seq,
                          mbox->ext_id, &rec, NULL);
        }
        i_assert(next_map_uid == last_map_uid + 1);
    }

    /* increase map's refcount for copied mails */
    if (array_is_created(&ctx->copy_map_uids)) {
        ctx->map_trans =
            dbox_map_transaction_begin(mbox->storage->map, FALSE);
        if (dbox_map_update_refcounts(ctx->map_trans,
                          &ctx->copy_map_uids, 1) < 0) {
            mdbox_transaction_save_rollback(_ctx);
            return -1;
        }
    }

    if (ctx->ctx.mail != NULL)
        mail_free(&ctx->ctx.mail);

    _t->changes->uid_validity = hdr->uid_validity;
    return 0;
}

void mdbox_transaction_save_commit_post(struct mail_save_context *_ctx,
                    struct mail_index_transaction_commit_result *result)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;

    _ctx->transaction = NULL; /* transaction is already freed */

    mail_index_sync_set_commit_result(ctx->sync_ctx->index_sync_ctx,
                      result);

    /* finish writing the mailbox APPENDs */
    if (mdbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) {
        if (ctx->map_trans != NULL)
            (void)dbox_map_transaction_commit(ctx->map_trans);
        /* commit only updates the sync tail offset, everything else
           was already written at this point. */
        (void)dbox_map_append_commit(ctx->append_ctx);
    }
    dbox_map_append_free(&ctx->append_ctx);

    if (!ctx->mbox->storage->storage.storage.set->fsync_disable) {
        if (fdatasync_path(ctx->mbox->ibox.box.path) < 0) {
            i_error("fdatasync_path(%s) failed: %m",
                ctx->mbox->ibox.box.path);
        }
    }
    mdbox_transaction_save_rollback(_ctx);
}

void mdbox_transaction_save_rollback(struct mail_save_context *_ctx)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;

    if (!ctx->ctx.finished)
        mdbox_save_cancel(&ctx->ctx.ctx);
    if (ctx->append_ctx != NULL)
        dbox_map_append_free(&ctx->append_ctx);
    if (ctx->map_trans != NULL)
        dbox_map_transaction_free(&ctx->map_trans);
    if (array_is_created(&ctx->copy_map_uids))
        array_free(&ctx->copy_map_uids);

    if (ctx->sync_ctx != NULL)
        (void)mdbox_sync_finish(&ctx->sync_ctx, FALSE);

    if (ctx->ctx.mail != NULL)
        mail_free(&ctx->ctx.mail);
    array_free(&ctx->mails);
    i_free(ctx);
}

int mdbox_copy(struct mail_save_context *_ctx, struct mail *mail)
{
    struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
    struct mdbox_mailbox *src_mbox;
    struct mdbox_mail_index_record rec;
    const void *data;
    bool expunged;

    ctx->ctx.finished = TRUE;

    if (mail->box->storage != _ctx->transaction->box->storage)
        return mail_storage_copy(_ctx, mail);
    src_mbox = (struct mdbox_mailbox *)mail->box;

    memset(&rec, 0, sizeof(rec));
    rec.save_date = ioloop_time;
    if (mdbox_mail_lookup(src_mbox, src_mbox->ibox.view, mail->seq,
                  &rec.map_uid) < 0)
        return -1;

    /* remember the map_uid so we can later increase its refcount */
    if (!array_is_created(&ctx->copy_map_uids))
        i_array_init(&ctx->copy_map_uids, 32);
    array_append(&ctx->copy_map_uids, &rec.map_uid, 1);

    /* add message to mailbox index */
    dbox_save_add_to_index(&ctx->ctx);
    mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
                  ctx->mbox->ext_id, &rec, NULL);

    mail_index_lookup_ext(src_mbox->ibox.view, mail->seq,
                  src_mbox->guid_ext_id, &data, &expunged);
    if (data != NULL) {
        mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
                      ctx->mbox->guid_ext_id, data, NULL);
    }
    return 0;
}