src/lib-index/mail-transaction-log-append.c

	mail-transaction-log-append.c revision 61cfcb75224b54bf20e864e85e1a343c72a84066
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "lib.h"
2423da95ee20fd4b3c260c1389cf2952d25f099cTimo Sirainen#include "array.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "buffer.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "write-full.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "mail-index-private.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "mail-index-view-private.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "mail-index-modseq.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "mail-index-transaction-private.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "mail-transaction-log-private.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainenstruct log_append_context {
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen    struct mail_transaction_log_file *file;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    struct mail_index_transaction *trans;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    buffer_t *output;
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen    uint64_t modseq;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    uint32_t first_append_size;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    bool sync_includes_this;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen};
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainenstatic void log_append_buffer(struct log_append_context *ctx,
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen                  const buffer_t *buf, const buffer_t *hdr_buf,
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen                  enum mail_transaction_type type)
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen{
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen    struct mail_transaction_header hdr;
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen    uint32_t hdr_size;
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen    size_t hdr_pos;
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen    i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    i_assert((buf->used % 4) == 0);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    i_assert(hdr_buf == NULL || (hdr_buf->used % 4) == 0);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (buf->used == 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        return;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    memset(&hdr, 0, sizeof(hdr));
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen    hdr.type = type;
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen    if (type == MAIL_TRANSACTION_EXPUNGE)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        hdr.type |= MAIL_TRANSACTION_EXTERNAL;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    hdr.size = sizeof(hdr) + buf->used +
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        (hdr_buf == NULL ? 0 : hdr_buf->used);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    hdr_pos = ctx->output->used;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    buffer_append(ctx->output, &hdr, sizeof(hdr));
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    if (hdr_buf != NULL)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    buffer_append(ctx->output, buf->data, buf->used);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (mail_transaction_header_has_modseq(buf->data,
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen            CONST_PTR_OFFSET(buf->data, sizeof(hdr)), ctx->modseq))
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen        ctx->modseq++;
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    /* update the size */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    hdr_size = mail_index_uint32_to_offset(hdr.size);
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen    if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) &&
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen        ctx->first_append_size == 0) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        /* size will be written later once everything
61e84692827b6a64912343f515c984853021483aTimo Sirainen           is in disk */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        ctx->first_append_size = hdr_size;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        hdr.size = 0;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    } else {
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen        hdr.size = hdr_size;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    }
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    buffer_write(ctx->output, hdr_pos, &hdr, sizeof(hdr));
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen}
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainenstatic int log_buffer_move_to_memory(struct log_append_context *ctx)
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen{
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    struct mail_transaction_log_file *file = ctx->file;
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen    /* first we need to truncate this latest write so that log syncing
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen       doesn't break */
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen    if (ftruncate(file->fd, file->sync_offset) < 0) {
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                          file->filepath,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                          "ftruncate()");
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen    }
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen    if (mail_index_move_to_memory(file->log->index) < 0)
2f30b72d49fbff0c4096125c139e4bdfef45669cTimo Sirainen        return -1;
0b25846ba794ce19536a24d4065beaf2a0bd9464Timo Sirainen    i_assert(MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen    i_assert(file->buffer_offset + file->buffer->used ==
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen         file->sync_offset);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    buffer_append_buf(file->buffer, ctx->output, 0, (size_t)-1);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    buffer_write(file->buffer, file->sync_offset - file->buffer_offset,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen             &ctx->first_append_size, sizeof(uint32_t));
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    file->sync_offset = file->buffer_offset + file->buffer->used;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    return 0;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen}
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic int log_buffer_write(struct log_append_context *ctx, bool want_fsync)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen{
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    struct mail_transaction_log_file *file = ctx->file;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        buffer_append_buf(file->buffer, ctx->output, 0, (size_t)-1);
bbd0a870f8639767e4e4011d2aedadac08d5c66fTimo Sirainen        file->sync_offset = file->buffer_offset + file->buffer->used;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        return 0;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    i_assert(ctx->first_append_size != 0);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (pwrite_full(file->fd, ctx->output->data, ctx->output->used,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            file->sync_offset) < 0) {
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen        /* write failure, fallback to in-memory indexes. */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen                          file->filepath,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                          "pwrite_full()");
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        return log_buffer_move_to_memory(ctx);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen    }
106b804c819443791f1324f8bbe34429eeea6a13Timo Sirainen
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen    i_assert(!ctx->sync_includes_this ||
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen         file->sync_offset + ctx->output->used ==
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen         file->max_tail_offset);
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen    /* now that the whole transaction has been written, rewrite the first
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen       record's size so the transaction becomes visible */
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen    if (pwrite_full(file->fd, &ctx->first_append_size,
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen            sizeof(uint32_t), file->sync_offset) < 0) {
3fe67ec75ccae1230bb9eb9f16affc48377f6441Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen                          file->filepath,
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen                          "pwrite_full()");
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen        return log_buffer_move_to_memory(ctx);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    if ((want_fsync && !file->log->index->fsync_disable) ||
4d84348ffcbb60de566108562c95ad64629e7a53Timo Sirainen        file->log->index->nfs_flush) {
4d84348ffcbb60de566108562c95ad64629e7a53Timo Sirainen        if (fdatasync(file->fd) < 0) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            mail_index_file_set_syscall_error(file->log->index,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                              file->filepath,
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen                              "fdatasync()");
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen            return log_buffer_move_to_memory(ctx);
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen        }
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    /* FIXME: when we're relying on O_APPEND and someone else wrote a
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen       transaction, we'll need to wait for it to commit its transaction.
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen       if it crashes before doing that, we'll need to overwrite it with
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen       a dummy record */
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    file->sync_offset += ctx->output->used;
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    return 0;
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic const buffer_t *
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainenlog_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen{
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    buffer_t *buf;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    const unsigned char *data, *mask;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    struct mail_transaction_header_update u;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    uint16_t offset;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    int state = 0;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    memset(&u, 0, sizeof(u));
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    data = prepend ? t->pre_hdr_change : t->post_hdr_change;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    buf = buffer_create_dynamic(pool_datastack_create(), 256);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen        if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen            if (state == 0) {
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen                u.offset = offset;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen                state++;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen            }
4645cc6c911a95991d7af43b40f88e99506ea5e9Timo Sirainen        } else {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            if (state > 0) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen                u.size = offset - u.offset;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen                buffer_append(buf, &u, sizeof(u));
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen                buffer_append(buf, data + u.offset, u.size);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen                state = 0;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen            }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    return buf;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen}
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainenstatic void
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainenext_reset_update_atomic(struct mail_index_transaction *t,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen            uint32_t ext_id, uint32_t expected_reset_id)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen{
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen    const struct mail_index_ext *map_ext;
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen    struct mail_transaction_ext_reset *reset;
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen    uint32_t idx, reset_id;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    if (!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen        /* new extension */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        reset_id = 1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    } else {
96f2533c48ce5def0004931606a2fdf275578880Timo Sirainen        map_ext = array_idx(&t->view->index->map->extensions, idx);
96f2533c48ce5def0004931606a2fdf275578880Timo Sirainen        reset_id = map_ext->reset_id + 1;
4d84348ffcbb60de566108562c95ad64629e7a53Timo Sirainen    }
4d84348ffcbb60de566108562c95ad64629e7a53Timo Sirainen    if (reset_id != expected_reset_id) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        /* ignore this extension update */
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen        mail_index_ext_set_reset_id(t, ext_id, 0);
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen        return;
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen    }
0185427dd52fddec6fc76d6e99c7659620d4366eTimo Sirainen
0185427dd52fddec6fc76d6e99c7659620d4366eTimo Sirainen    if (reset_id == 0)
9c66dd5c3e2ba484a5c7cdb98a139bda31fdfaa4Timo Sirainen        reset_id++;
9c66dd5c3e2ba484a5c7cdb98a139bda31fdfaa4Timo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen    array_idx_set(&t->ext_reset_ids, ext_id, &reset_id);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen    /* reseting existing data is optional */
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen    if (array_is_created(&t->ext_resets)) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        reset = array_idx_modifiable(&t->ext_resets, ext_id);
43d32cbe60fdaef2699d99f1ca259053e9350411Timo Sirainen        if (reset->new_reset_id == (uint32_t)-1)
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen            reset->new_reset_id = reset_id;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
static void
transaction_update_atomic_reset_ids(struct mail_index_transaction *t)
{
    const uint32_t *expected_reset_ids;
    unsigned int ext_id, count;

    if (!array_is_created(&t->ext_reset_atomic))
        return;

    expected_reset_ids = array_get(&t->ext_reset_atomic, &count);
    for (ext_id = 0; ext_id < count; ext_id++) {
        if (expected_reset_ids[ext_id] != 0) {
            ext_reset_update_atomic(t, ext_id,
                        expected_reset_ids[ext_id]);
        }
    }
}

static void log_append_ext_intro(struct log_append_context *ctx,
                 uint32_t ext_id, uint32_t reset_id)
{
    struct mail_index_transaction *t = ctx->trans;
    const struct mail_index_registered_ext *rext;
        struct mail_transaction_ext_intro *intro;
    buffer_t *buf;
    uint32_t idx;
    unsigned int count;

    i_assert(ext_id != (uint32_t)-1);

    if (t->reset ||
        !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
        /* new extension */
        idx = (uint32_t)-1;
    }

    rext = array_idx(&t->view->index->extensions, ext_id);
    if (!array_is_created(&t->ext_resizes)) {
        intro = NULL;
        count = 0;
    } else {
        intro = array_get_modifiable(&t->ext_resizes, &count);
    }

    buf = buffer_create_dynamic(pool_datastack_create(), 128);
    if (ext_id < count && intro[ext_id].name_size != 0) {
        /* we're resizing it */
        intro += ext_id;

        i_assert(intro->ext_id == idx);
        intro->name_size = idx != (uint32_t)-1 ? 0 :
            strlen(rext->name);
        buffer_append(buf, intro, sizeof(*intro));
    } else {
        /* generate a new intro structure */
        intro = buffer_append_space_unsafe(buf, sizeof(*intro));
        intro->ext_id = idx;
        intro->hdr_size = rext->hdr_size;
        intro->record_size = rext->record_size;
        intro->record_align = rext->record_align;
        intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
        intro->name_size = idx != (uint32_t)-1 ? 0 :
            strlen(rext->name);
    }
    if (reset_id != 0) {
        /* we're going to reset this extension in this transaction */
        intro->reset_id = reset_id;
    } else if (idx != (uint32_t)-1) {
        /* use the existing reset_id */
        const struct mail_index_ext *map_ext =
            array_idx(&t->view->index->map->extensions, idx);
        intro->reset_id = map_ext->reset_id;
    } else {
        /* new extension, reset_id defaults to 0 */
    }
    buffer_append(buf, rext->name, intro->name_size);
    if ((buf->used % 4) != 0)
        buffer_append_zero(buf, 4 - (buf->used % 4));

    if (ctx->file->sync_highest_modseq == 0 &&
        strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
        /* modseq tracking started */
        ctx->file->sync_highest_modseq = 1;
    }

    log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
}

static void
log_append_ext_hdr_update(struct log_append_context *ctx,
            const struct mail_index_transaction_ext_hdr_update *hdr)
{
    buffer_t *buf;
    const unsigned char *data, *mask;
    struct mail_transaction_ext_hdr_update u;
    uint16_t offset;
    bool started = FALSE;

    memset(&u, 0, sizeof(u));

    data = hdr->data;
    mask = hdr->mask;

    buf = buffer_create_dynamic(pool_datastack_create(), 256);
    for (offset = 0; offset <= hdr->alloc_size; offset++) {
        if (offset < hdr->alloc_size && mask[offset] != 0) {
            if (!started) {
                u.offset = offset;
                started = TRUE;
            }
        } else {
            if (started) {
                u.size = offset - u.offset;
                buffer_append(buf, &u, sizeof(u));
                buffer_append(buf, data + u.offset, u.size);
                started = FALSE;
            }
        }
    }
    if (buf->used % 4 != 0)
        buffer_append_zero(buf, 4 - buf->used % 4);
    log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_HDR_UPDATE);
}

static void
mail_transaction_log_append_ext_intros(struct log_append_context *ctx)
{
    struct mail_index_transaction *t = ctx->trans;
        const struct mail_transaction_ext_intro *resize;
    const struct mail_index_transaction_ext_hdr_update *hdrs;
    struct mail_transaction_ext_reset ext_reset;
    unsigned int update_count, resize_count, ext_count = 0;
    unsigned int hdrs_count, reset_id_count, reset_count;
    uint32_t ext_id, reset_id;
    const struct mail_transaction_ext_reset *reset;
    const uint32_t *reset_ids;
    const ARRAY_TYPE(seq_array) *update;
    buffer_t *buf;

    if (!array_is_created(&t->ext_rec_updates)) {
        update = NULL;
        update_count = 0;
    } else {
        update = array_get(&t->ext_rec_updates, &update_count);
        ext_count = update_count;
    }

    if (!array_is_created(&t->ext_resizes)) {
        resize = NULL;
        resize_count = 0;
    } else {
        resize = array_get(&t->ext_resizes, &resize_count);
        if (ext_count < resize_count)
            ext_count = resize_count;
    }

    if (!array_is_created(&t->ext_reset_ids)) {
        reset_ids = NULL;
        reset_id_count = 0;
    } else {
        reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
    }

    if (!array_is_created(&t->ext_resets)) {
        reset = NULL;
        reset_count = 0;
    } else {
        reset = array_get(&t->ext_resets, &reset_count);
        if (ext_count < reset_count)
            ext_count = reset_count;
    }

    if (!array_is_created(&t->ext_hdr_updates)) {
        hdrs = NULL;
        hdrs_count = 0;
    } else {
        hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
        if (ext_count < hdrs_count)
            ext_count = hdrs_count;
    }

    memset(&ext_reset, 0, sizeof(ext_reset));
    buf = buffer_create_data(pool_datastack_create(),
                 &ext_reset, sizeof(ext_reset));
    buffer_set_used_size(buf, sizeof(ext_reset));

    for (ext_id = 0; ext_id < ext_count; ext_id++) {
        if (ext_id < reset_count)
            ext_reset = reset[ext_id];
        else
            ext_reset.new_reset_id = 0;
        if ((ext_id < resize_count && resize[ext_id].name_size) ||
            (ext_id < update_count &&
             array_is_created(&update[ext_id])) ||
            ext_reset.new_reset_id != 0 ||
            (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
            reset_id = ext_id < reset_id_count &&
                ext_reset.new_reset_id == 0 ?
                reset_ids[ext_id] : 0;
            log_append_ext_intro(ctx, ext_id, reset_id);
        }
        if (ext_reset.new_reset_id != 0) {
            i_assert(ext_id < reset_id_count &&
                 ext_reset.new_reset_id == reset_ids[ext_id]);
            log_append_buffer(ctx, buf, NULL,
                      MAIL_TRANSACTION_EXT_RESET);
        }
        if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
            T_BEGIN {
                log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
            } T_END;
        }
    }
}

static void log_append_ext_rec_updates(struct log_append_context *ctx)
{
    struct mail_index_transaction *t = ctx->trans;
    ARRAY_TYPE(seq_array) *updates;
    const uint32_t *reset_ids;
    unsigned int ext_id, count, reset_id_count;
    uint32_t reset_id;

    if (!array_is_created(&t->ext_rec_updates)) {
        updates = NULL;
        count = 0;
    } else {
        updates = array_get_modifiable(&t->ext_rec_updates, &count);
    }

    if (!array_is_created(&t->ext_reset_ids)) {
        reset_ids = NULL;
        reset_id_count = 0;
    } else {
        reset_ids = array_get_modifiable(&t->ext_reset_ids,
                         &reset_id_count);
    }

    for (ext_id = 0; ext_id < count; ext_id++) {
        if (!array_is_created(&updates[ext_id]))
            continue;

        reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
        log_append_ext_intro(ctx, ext_id, reset_id);

        log_append_buffer(ctx, updates[ext_id].arr.buffer, NULL,
                  MAIL_TRANSACTION_EXT_REC_UPDATE);
    }
}

static void
log_append_keyword_update(struct log_append_context *ctx,
              buffer_t *hdr_buf, enum modify_type modify_type,
              const char *keyword, const buffer_t *buffer)
{
    struct mail_transaction_keyword_update kt_hdr;

    memset(&kt_hdr, 0, sizeof(kt_hdr));
    kt_hdr.modify_type = modify_type;
    kt_hdr.name_size = strlen(keyword);

    buffer_set_used_size(hdr_buf, 0);
    buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
    buffer_append(hdr_buf, keyword, kt_hdr.name_size);
    if ((hdr_buf->used % 4) != 0)
        buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));

    log_append_buffer(ctx, buffer, hdr_buf,
              MAIL_TRANSACTION_KEYWORD_UPDATE);
}

static enum mail_index_sync_type
log_append_keyword_updates(struct log_append_context *ctx)
{
        const struct mail_index_transaction_keyword_update *updates;
    const char *const *keywords;
    buffer_t *hdr_buf;
    enum mail_index_sync_type change_mask = 0;
    unsigned int i, count, keywords_count;

    hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);

    keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
                    &keywords_count);
    updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
    i_assert(count <= keywords_count);

    for (i = 0; i < count; i++) {
        if (array_is_created(&updates[i].add_seq)) {
            change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
            log_append_keyword_update(ctx, hdr_buf,
                    MODIFY_ADD, keywords[i],
                    updates[i].add_seq.arr.buffer);
        }
        if (array_is_created(&updates[i].remove_seq)) {
            change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
            log_append_keyword_update(ctx, hdr_buf,
                    MODIFY_REMOVE, keywords[i],
                    updates[i].remove_seq.arr.buffer);
        }
    }
    return change_mask;
}

static void log_append_sync_offset_if_needed(struct log_append_context *ctx)
{
    struct mail_transaction_header_update *u;
    buffer_t *buf;
    uint32_t offset;

    if (ctx->file->max_tail_offset == ctx->file->sync_offset) {
        /* FIXME: when we remove exclusive log locking, we
           can't rely on this. then write non-changed offset + check
           real offset + rewrite the new offset if other transactions
           weren't written in the middle */
        ctx->file->max_tail_offset += ctx->output->used +
            sizeof(struct mail_transaction_header) +
            sizeof(*u) + sizeof(offset);
        ctx->sync_includes_this = TRUE;
    }
    offset = ctx->file->max_tail_offset;

    if (ctx->file->saved_tail_offset == offset)
        return;
    i_assert(offset > ctx->file->saved_tail_offset);

    buf = buffer_create_static_hard(pool_datastack_create(),
                    sizeof(*u) + sizeof(offset));
    u = buffer_append_space_unsafe(buf, sizeof(*u));
    u->offset = offsetof(struct mail_index_header, log_file_tail_offset);
    u->size = sizeof(offset);
    buffer_append(buf, &offset, sizeof(offset));

    log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_HEADER_UPDATE);
}

#define TRANSACTION_HAS_CHANGES(t) \
    ((t)->log_updates || (t)->log_ext_updates || \
     (array_is_created(&(t)->updates) && array_count(&(t)->updates) > 0))

static int
mail_transaction_log_append_locked(struct mail_index_transaction *t,
                   uint32_t *log_file_seq_r,
                   uoff_t *log_file_offset_r)
{
    enum mail_index_sync_type change_mask = 0;
    struct mail_index_view *view = t->view;
    struct mail_index *index;
    struct mail_transaction_log *log;
    struct mail_transaction_log_file *file;
    struct log_append_context ctx;
    uoff_t append_offset;
    bool want_fsync;

    index = mail_index_view_get_index(view);
    log = index->log;

    if (t->reset) {
        /* Reset the whole index, preserving only indexid. Begin by
           rotating the log. We don't care if we skip some non-synced
           transactions. */
        if (mail_transaction_log_rotate(log, TRUE) < 0)
            return -1;

        if (!TRANSACTION_HAS_CHANGES(t)) {
            /* we only wanted to reset */
            return 0;
        }
    }

    if (!index->log_locked) {
        /* update sync_offset */
        if (mail_transaction_log_file_map(log->head,
                          log->head->sync_offset,
                          (uoff_t)-1) <= 0)
            return -1;
    }

    if (array_is_created(&t->ext_reset_atomic) || t->max_modseq != 0) {
        if (mail_index_map(t->view->index,
                   MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
            return -1;
    }
    if (array_is_created(&t->ext_reset_atomic))
        transaction_update_atomic_reset_ids(t);
    if (t->max_modseq != 0)
        mail_index_transaction_check_conflicts(t);
    if (!TRANSACTION_HAS_CHANGES(t)) {
        /* we aborted all changes, nothing else to do */
        return 0;
    }
    /* finally convert all sequences to UIDs before we write them,
       but after we've checked and removed conflicts */
    mail_index_transaction_convert_to_uids(t);

    file = log->head;

    i_assert(file->sync_offset >= file->buffer_offset);

    memset(&ctx, 0, sizeof(ctx));
    ctx.file = file;
    ctx.trans = t;
    ctx.output = buffer_create_dynamic(default_pool, 1024);
    ctx.modseq = file->sync_highest_modseq;

    /* send all extension introductions and resizes before appends
       to avoid resize overhead as much as possible */
        mail_transaction_log_append_ext_intros(&ctx);

    if (t->pre_hdr_changed) {
        log_append_buffer(&ctx,
                  log_get_hdr_update_buffer(t, TRUE),
                  NULL, MAIL_TRANSACTION_HEADER_UPDATE);
    }
    if (array_is_created(&t->appends)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_APPEND;
        log_append_buffer(&ctx, t->appends.arr.buffer, NULL,
                  MAIL_TRANSACTION_APPEND);
    }
    if (array_is_created(&t->updates)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS;
        log_append_buffer(&ctx, t->updates.arr.buffer, NULL,
                  MAIL_TRANSACTION_FLAG_UPDATE);
    }

    if (array_is_created(&t->ext_rec_updates))
        log_append_ext_rec_updates(&ctx);

    /* keyword resets before updates */
    if (array_is_created(&t->keyword_resets)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
        log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
                  NULL, MAIL_TRANSACTION_KEYWORD_RESET);
    }
    if (array_is_created(&t->keyword_updates))
        change_mask |= log_append_keyword_updates(&ctx);

    if (array_is_created(&t->expunges)) {
        /* non-external expunges are only requests, ignore them when
           checking fsync_mask */
        if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
            change_mask |= MAIL_INDEX_SYNC_TYPE_EXPUNGE;
        log_append_buffer(&ctx, t->expunges.arr.buffer, NULL,
                  MAIL_TRANSACTION_EXPUNGE);
    }

    if (t->post_hdr_changed) {
        log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
                  NULL, MAIL_TRANSACTION_HEADER_UPDATE);
    }

    /* NOTE: mailbox sync offset update must be the last change.
       it may update the sync offset to include this transaction, so it
       needs to know this transaction's size */
    if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
        log_append_sync_offset_if_needed(&ctx);

    if (file->sync_offset < file->last_size) {
        /* there is some garbage at the end of the transaction log
           (eg. previous write failed). remove it so reader doesn't
           break because of it. */
        buffer_set_used_size(file->buffer,
                     file->sync_offset - file->buffer_offset);
        if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
            if (ftruncate(file->fd, file->sync_offset) < 0) {
                mail_index_file_set_syscall_error(index,
                    file->filepath, "ftruncate()");
            }
        }
    }

    want_fsync = (view->index->fsync_mask & change_mask) != 0;
    append_offset = file->sync_offset;
    if (log_buffer_write(&ctx, want_fsync) < 0) {
        buffer_free(&ctx.output);
        return -1;
    }
    file->sync_highest_modseq = ctx.modseq;
    buffer_free(&ctx.output);

    if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0) {
        /* mark the area covered by this transaction hidden */
        mail_index_view_add_hidden_transaction(view, file->hdr.file_seq,
            append_offset, file->sync_offset - append_offset);
    }

    *log_file_seq_r = file->hdr.file_seq;
    *log_file_offset_r = file->sync_offset;
    return 0;
}

int mail_transaction_log_append(struct mail_index_transaction *t,
                uint32_t *log_file_seq_r,
                uoff_t *log_file_offset_r)
{
    struct mail_index *index;
    int ret;

    *log_file_seq_r = 0;
    *log_file_offset_r = 0;

    if (!TRANSACTION_HAS_CHANGES(t) && !t->reset) {
        /* nothing to append */
        return 0;
    }

    index = mail_index_view_get_index(t->view);
    if (index->log_locked) {
        i_assert((t->flags &
              MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0);
    } else {
        if (mail_transaction_log_lock_head(index->log) < 0)
            return -1;
    }

    ret = mail_transaction_log_append_locked(t, log_file_seq_r,
                         log_file_offset_r);

    if (!index->log_locked)
        mail_transaction_log_file_unlock(index->log->head);
    return ret;
}