src/lib-index/mail-transaction-log-append.c

	mail-transaction-log-append.c revision 8562da433e7659ce8ad6044ff5d8f44869c8cb73
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "lib.h"
cf1797248b02eadfd7d63aabc0b64678a4239540Timo Sirainen#include "array.h"
cf1797248b02eadfd7d63aabc0b64678a4239540Timo Sirainen#include "buffer.h"
2423da95ee20fd4b3c260c1389cf2952d25f099cTimo Sirainen#include "write-full.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "mail-index-private.h"
a8fe899601735459641edae975c0fa08be8482e2Timo Sirainen#include "mail-index-view-private.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "mail-index-modseq.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "mail-index-transaction-private.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "mail-transaction-log-private.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainenstruct log_append_context {
29543188462c9348f365ec29115d777ffe4769d3Timo Sirainen    struct mail_transaction_log_file *file;
29543188462c9348f365ec29115d777ffe4769d3Timo Sirainen    struct mail_index_transaction *trans;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    buffer_t *output;
fa780a18c41881036af582f7a3473d6399e9d34dTimo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen    uint64_t modseq;
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen    uint32_t first_append_size;
bdb026e2dc8a7c77585ed5ba489f0056df8074d4Timo Sirainen    bool sync_includes_this;
bdb026e2dc8a7c77585ed5ba489f0056df8074d4Timo Sirainen};
bdb026e2dc8a7c77585ed5ba489f0056df8074d4Timo Sirainen
bdb026e2dc8a7c77585ed5ba489f0056df8074d4Timo Sirainenstatic void log_append_buffer(struct log_append_context *ctx,
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen                  const buffer_t *buf, const buffer_t *hdr_buf,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                  enum mail_transaction_type type)
9393445a6dabd17ce62ebfc12fd73545b0065824Timo Sirainen{
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen    struct mail_transaction_header hdr;
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen    uint32_t hdr_size;
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    size_t hdr_pos;
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    i_assert((buf->used % 4) == 0);
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    i_assert(hdr_buf == NULL || (hdr_buf->used % 4) == 0);
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    if (buf->used == 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        return;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
917498e6f84969d2b93410c1e479735abe8e0ed7Timo Sirainen    memset(&hdr, 0, sizeof(hdr));
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    hdr.type = type;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (type == MAIL_TRANSACTION_EXPUNGE)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        hdr.type |= MAIL_TRANSACTION_EXTERNAL;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    hdr.size = sizeof(hdr) + buf->used +
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        (hdr_buf == NULL ? 0 : hdr_buf->used);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    hdr_pos = ctx->output->used;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    buffer_append(ctx->output, &hdr, sizeof(hdr));
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen    if (hdr_buf != NULL)
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen        buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    buffer_append(ctx->output, buf->data, buf->used);
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen    if (mail_transaction_header_has_modseq(buf->data,
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen            CONST_PTR_OFFSET(buf->data, sizeof(hdr)), ctx->modseq))
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        ctx->modseq++;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen    /* update the size */
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen    hdr_size = mail_index_uint32_to_offset(hdr.size);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) &&
61e84692827b6a64912343f515c984853021483aTimo Sirainen        ctx->first_append_size == 0) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        /* size will be written later once everything
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen           is in disk */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        ctx->first_append_size = hdr_size;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen        hdr.size = 0;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    } else {
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen        hdr.size = hdr_size;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    }
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    buffer_write(ctx->output, hdr_pos, &hdr, sizeof(hdr));
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen}
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainenstatic int log_buffer_move_to_memory(struct log_append_context *ctx)
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen{
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen    struct mail_transaction_log_file *file = ctx->file;
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen    /* first we need to truncate this latest write so that log syncing
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen       doesn't break */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (ftruncate(file->fd, file->sync_offset) < 0) {
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
2f30b72d49fbff0c4096125c139e4bdfef45669cTimo Sirainen                          file->filepath,
0b25846ba794ce19536a24d4065beaf2a0bd9464Timo Sirainen                          "ftruncate()");
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen    }
71e88fae3be360e9a93b3398e743f99a6f05d2edTimo Sirainen
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    if (mail_index_move_to_memory(file->log->index) < 0)
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen        return -1;
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    i_assert(MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    i_assert(file->buffer_offset + file->buffer->used ==
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen         file->sync_offset);
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    buffer_append_buf(file->buffer, ctx->output, 0, (size_t)-1);
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    buffer_write(file->buffer, file->sync_offset - file->buffer_offset,
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen             &ctx->first_append_size, sizeof(uint32_t));
58a89627905e3590381cdd5eb931b9537c4b4ea6Timo Sirainen    file->sync_offset = file->buffer_offset + file->buffer->used;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    return 0;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic int log_buffer_write(struct log_append_context *ctx, bool want_fsync)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen{
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    struct mail_transaction_log_file *file = ctx->file;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen        buffer_append_buf(file->buffer, ctx->output, 0, (size_t)-1);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen        file->sync_offset = file->buffer_offset + file->buffer->used;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen        return 0;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
bbd0a870f8639767e4e4011d2aedadac08d5c66fTimo Sirainen    i_assert(ctx->first_append_size != 0);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (pwrite_full(file->fd, ctx->output->data, ctx->output->used,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            file->sync_offset) < 0) {
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen        /* write failure, fallback to in-memory indexes. */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                          file->filepath,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                          "pwrite_full()");
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen        return log_buffer_move_to_memory(ctx);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    i_assert(!ctx->sync_includes_this ||
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen         file->sync_offset + ctx->output->used ==
76b91bac787101e6b0075122ab6478dd98c8a884Timo Sirainen         file->max_tail_offset);
76b91bac787101e6b0075122ab6478dd98c8a884Timo Sirainen
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen    /* now that the whole transaction has been written, rewrite the first
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen       record's size so the transaction becomes visible */
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen    if (pwrite_full(file->fd, &ctx->first_append_size,
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen            sizeof(uint32_t), file->sync_offset) < 0) {
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen        mail_index_file_set_syscall_error(file->log->index,
3fe67ec75ccae1230bb9eb9f16affc48377f6441Timo Sirainen                          file->filepath,
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen                          "pwrite_full()");
847caf605dc11acfb1861586b558d9cca4a85cb0Timo Sirainen        return log_buffer_move_to_memory(ctx);
847caf605dc11acfb1861586b558d9cca4a85cb0Timo Sirainen    }
847caf605dc11acfb1861586b558d9cca4a85cb0Timo Sirainen
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen    if ((want_fsync && !file->log->index->fsync_disable) ||
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen        file->log->index->nfs_flush) {
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen        if (fdatasync(file->fd) < 0) {
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen            mail_index_file_set_syscall_error(file->log->index,
8ca217bf3aa23c7922d0d4aa44fcd2320416d61cMartti Rannanjärvi                              file->filepath,
8ca217bf3aa23c7922d0d4aa44fcd2320416d61cMartti Rannanjärvi                              "fdatasync()");
8ca217bf3aa23c7922d0d4aa44fcd2320416d61cMartti Rannanjärvi            return log_buffer_move_to_memory(ctx);
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen        }
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
4654cf737f538f5de032b8c9908913f121917366Timo Sirainen    /* FIXME: when we're relying on O_APPEND and someone else wrote a
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen       transaction, we'll need to wait for it to commit its transaction.
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen       if it crashes before doing that, we'll need to overwrite it with
4d84348ffcbb60de566108562c95ad64629e7a53Timo Sirainen       a dummy record */
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen
49c48631cfd07017d5f93d83713fffe4f13730c4Timo Sirainen    file->sync_offset += ctx->output->used;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    return 0;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen}
8ca217bf3aa23c7922d0d4aa44fcd2320416d61cMartti Rannanjärvi
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic const buffer_t *
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenlog_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen{
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    buffer_t *buf;
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    const unsigned char *data, *mask;
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    struct mail_transaction_header_update u;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    uint16_t offset;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    int state = 0;
7ff6268cc35102675d73d44d680bed13d0709f7bTimo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    memset(&u, 0, sizeof(u));
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
f8a67af9b7bde79c186e6b82ea200d7fcf85571bTimo Sirainen    data = prepend ? t->pre_hdr_change : t->post_hdr_change;
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen
eb1572d7c44ebc7b0b039d085c3dbab2ef7043ddTimo Sirainen    buf = buffer_create_dynamic(pool_datastack_create(), 256);
ed354926406e28254b581f821bb052f38d9c14e8Timo Sirainen    for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen        if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen            if (state == 0) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                u.offset = offset;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                state++;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen            }
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen        } else {
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen            if (state > 0) {
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen                u.size = offset - u.offset;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen                buffer_append(buf, &u, sizeof(u));
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen                buffer_append(buf, data + u.offset, u.size);
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen                state = 0;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen            }
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen        }
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    }
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen    return buf;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic void
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenext_reset_update_atomic(struct mail_index_transaction *t,
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen            uint32_t ext_id, uint32_t expected_reset_id)
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen{
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen    const struct mail_index_ext *map_ext;
33525312d3f45995686aa0b538dea1cd6eb936e2Timo Sirainen    struct mail_transaction_ext_reset *reset;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen    uint32_t idx, reset_id;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
4645cc6c911a95991d7af43b40f88e99506ea5e9Timo Sirainen    if (!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
d3e5a14ea363264dcc7640ca7226249d0c27a793Timo Sirainen        /* new extension */
d3e5a14ea363264dcc7640ca7226249d0c27a793Timo Sirainen        reset_id = 1;
d3e5a14ea363264dcc7640ca7226249d0c27a793Timo Sirainen    } else {
a4e2101473cfd7ce960fc49b3ce097c3f89ec2adTimo Sirainen        map_ext = array_idx(&t->view->index->map->extensions, idx);
a4e2101473cfd7ce960fc49b3ce097c3f89ec2adTimo Sirainen        reset_id = map_ext->reset_id + 1;
a4e2101473cfd7ce960fc49b3ce097c3f89ec2adTimo Sirainen    }
a4e2101473cfd7ce960fc49b3ce097c3f89ec2adTimo Sirainen    if (reset_id != expected_reset_id) {
d3e5a14ea363264dcc7640ca7226249d0c27a793Timo Sirainen        /* ignore this extension update */
a4e2101473cfd7ce960fc49b3ce097c3f89ec2adTimo Sirainen        mail_index_ext_set_reset_id(t, ext_id, 0);
a4e2101473cfd7ce960fc49b3ce097c3f89ec2adTimo Sirainen        return;
a4e2101473cfd7ce960fc49b3ce097c3f89ec2adTimo Sirainen    }
a4e2101473cfd7ce960fc49b3ce097c3f89ec2adTimo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    if (reset_id == 0)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        reset_id++;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    array_idx_set(&t->ext_reset_ids, ext_id, &reset_id);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    /* reseting existing data is optional */
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen    if (array_is_created(&t->ext_resets)) {
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen        reset = array_idx_modifiable(&t->ext_resets, ext_id);
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen        if (reset->new_reset_id == (uint32_t)-1)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen            reset->new_reset_id = reset_id;
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen    }
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen}
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainenstatic void
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainentransaction_update_atomic_reset_ids(struct mail_index_transaction *t)
24acd68c82dc137b88bb3ba3258b9d1f7ebcb44dTimo Sirainen{
34b724d1d7e50b1ab24267a3b6fc089b1147c1abAki Tuomi    const uint32_t *expected_reset_ids;
34b724d1d7e50b1ab24267a3b6fc089b1147c1abAki Tuomi    unsigned int ext_id, count;
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (!array_is_created(&t->ext_reset_atomic))
77b5fd56e5a06d624f3ab92198272287333114f4Timo Sirainen        return;
77b5fd56e5a06d624f3ab92198272287333114f4Timo Sirainen
77b5fd56e5a06d624f3ab92198272287333114f4Timo Sirainen    expected_reset_ids = array_get(&t->ext_reset_atomic, &count);
96f2533c48ce5def0004931606a2fdf275578880Timo Sirainen    for (ext_id = 0; ext_id < count; ext_id++) {
96f2533c48ce5def0004931606a2fdf275578880Timo Sirainen        if (expected_reset_ids[ext_id] != 0) {
5069b6adc4acb0efb3c6e87e778b820bae9bae9bTimo Sirainen            ext_reset_update_atomic(t, ext_id,
5069b6adc4acb0efb3c6e87e778b820bae9bae9bTimo Sirainen                        expected_reset_ids[ext_id]);
5069b6adc4acb0efb3c6e87e778b820bae9bae9bTimo Sirainen        }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen}
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainenstatic void log_append_ext_intro(struct log_append_context *ctx,
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen                 uint32_t ext_id, uint32_t reset_id)
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_index_transaction *t = ctx->trans;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen    const struct mail_index_registered_ext *rext;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen        struct mail_transaction_ext_intro *intro;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    buffer_t *buf;
43d32cbe60fdaef2699d99f1ca259053e9350411Timo Sirainen    uint32_t idx;
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen    unsigned int count;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    i_assert(ext_id != (uint32_t)-1);
b5c6ce2ab8dc1a5817e8adc989d21a9f603a6673Aki Tuomi
b5c6ce2ab8dc1a5817e8adc989d21a9f603a6673Aki Tuomi    if (t->reset ||
b5c6ce2ab8dc1a5817e8adc989d21a9f603a6673Aki Tuomi        !mail_index_map_get_ext_idx(t->view->map, ext_id, &idx)) {
b5c6ce2ab8dc1a5817e8adc989d21a9f603a6673Aki Tuomi        /* new extension */
847caf605dc11acfb1861586b558d9cca4a85cb0Timo Sirainen        idx = (uint32_t)-1;
847caf605dc11acfb1861586b558d9cca4a85cb0Timo Sirainen    }
b5c6ce2ab8dc1a5817e8adc989d21a9f603a6673Aki Tuomi
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    rext = array_idx(&t->view->index->extensions, ext_id);
    if (!array_is_created(&t->ext_resizes)) {
        intro = NULL;
        count = 0;
    } else {
        intro = array_get_modifiable(&t->ext_resizes, &count);
    }

    buf = buffer_create_dynamic(pool_datastack_create(), 128);
    if (ext_id < count && intro[ext_id].name_size != 0) {
        /* we're resizing it */
        intro += ext_id;

        i_assert(intro->ext_id == idx);
        intro->name_size = idx != (uint32_t)-1 ? 0 :
            strlen(rext->name);
        buffer_append(buf, intro, sizeof(*intro));
    } else {
        /* generate a new intro structure */
        intro = buffer_append_space_unsafe(buf, sizeof(*intro));
        intro->ext_id = idx;
        intro->hdr_size = rext->hdr_size;
        intro->record_size = rext->record_size;
        intro->record_align = rext->record_align;
        intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
        intro->name_size = idx != (uint32_t)-1 ? 0 :
            strlen(rext->name);
    }
    if (reset_id != 0) {
        /* we're going to reset this extension in this transaction */
        intro->reset_id = reset_id;
    } else if (idx != (uint32_t)-1) {
        /* use the existing reset_id */
        const struct mail_index_ext *map_ext =
            array_idx(&t->view->map->extensions, idx);
        intro->reset_id = map_ext->reset_id;
    } else {
        /* new extension, reset_id defaults to 0 */
    }
    buffer_append(buf, rext->name, intro->name_size);
    if ((buf->used % 4) != 0)
        buffer_append_zero(buf, 4 - (buf->used % 4));

    if (ctx->file->sync_highest_modseq == 0 &&
        strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
        /* modseq tracking started */
        ctx->file->sync_highest_modseq = 1;
    }

    log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
}

static void
log_append_ext_hdr_update(struct log_append_context *ctx,
            const struct mail_index_transaction_ext_hdr_update *hdr)
{
    buffer_t *buf;
    const unsigned char *data, *mask;
    struct mail_transaction_ext_hdr_update u;
    uint16_t offset;
    bool started = FALSE;

    memset(&u, 0, sizeof(u));

    data = hdr->data;
    mask = hdr->mask;

    buf = buffer_create_dynamic(pool_datastack_create(), 256);
    for (offset = 0; offset <= hdr->alloc_size; offset++) {
        if (offset < hdr->alloc_size && mask[offset] != 0) {
            if (!started) {
                u.offset = offset;
                started = TRUE;
            }
        } else {
            if (started) {
                u.size = offset - u.offset;
                buffer_append(buf, &u, sizeof(u));
                buffer_append(buf, data + u.offset, u.size);
                started = FALSE;
            }
        }
    }
    if (buf->used % 4 != 0)
        buffer_append_zero(buf, 4 - buf->used % 4);
    log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_HDR_UPDATE);
}

static void
mail_transaction_log_append_ext_intros(struct log_append_context *ctx)
{
    struct mail_index_transaction *t = ctx->trans;
        const struct mail_transaction_ext_intro *resize;
    const struct mail_index_transaction_ext_hdr_update *hdrs;
    struct mail_transaction_ext_reset ext_reset;
    unsigned int update_count, resize_count, ext_count = 0;
    unsigned int hdrs_count, reset_id_count, reset_count;
    uint32_t ext_id, reset_id;
    const struct mail_transaction_ext_reset *reset;
    const uint32_t *reset_ids;
    const ARRAY_TYPE(seq_array) *update;
    buffer_t *buf;

    if (!array_is_created(&t->ext_rec_updates)) {
        update = NULL;
        update_count = 0;
    } else {
        update = array_get(&t->ext_rec_updates, &update_count);
        ext_count = update_count;
    }

    if (!array_is_created(&t->ext_resizes)) {
        resize = NULL;
        resize_count = 0;
    } else {
        resize = array_get(&t->ext_resizes, &resize_count);
        if (ext_count < resize_count)
            ext_count = resize_count;
    }

    if (!array_is_created(&t->ext_reset_ids)) {
        reset_ids = NULL;
        reset_id_count = 0;
    } else {
        reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
    }

    if (!array_is_created(&t->ext_resets)) {
        reset = NULL;
        reset_count = 0;
    } else {
        reset = array_get(&t->ext_resets, &reset_count);
        if (ext_count < reset_count)
            ext_count = reset_count;
    }

    if (!array_is_created(&t->ext_hdr_updates)) {
        hdrs = NULL;
        hdrs_count = 0;
    } else {
        hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
        if (ext_count < hdrs_count)
            ext_count = hdrs_count;
    }

    memset(&ext_reset, 0, sizeof(ext_reset));
    buf = buffer_create_data(pool_datastack_create(),
                 &ext_reset, sizeof(ext_reset));
    buffer_set_used_size(buf, sizeof(ext_reset));

    for (ext_id = 0; ext_id < ext_count; ext_id++) {
        if (ext_id < reset_count)
            ext_reset = reset[ext_id];
        else
            ext_reset.new_reset_id = 0;
        if ((ext_id < resize_count && resize[ext_id].name_size) ||
            (ext_id < update_count &&
             array_is_created(&update[ext_id])) ||
            ext_reset.new_reset_id != 0 ||
            (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
            reset_id = ext_id < reset_id_count &&
                ext_reset.new_reset_id == 0 ?
                reset_ids[ext_id] : 0;
            log_append_ext_intro(ctx, ext_id, reset_id);
        }
        if (ext_reset.new_reset_id != 0) {
            i_assert(ext_id < reset_id_count &&
                 ext_reset.new_reset_id == reset_ids[ext_id]);
            log_append_buffer(ctx, buf, NULL,
                      MAIL_TRANSACTION_EXT_RESET);
        }
        if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
            T_BEGIN {
                log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
            } T_END;
        }
    }
}

static void log_append_ext_rec_updates(struct log_append_context *ctx)
{
    struct mail_index_transaction *t = ctx->trans;
    ARRAY_TYPE(seq_array) *updates;
    const uint32_t *reset_ids;
    unsigned int ext_id, count, reset_id_count;
    uint32_t reset_id;

    if (!array_is_created(&t->ext_rec_updates)) {
        updates = NULL;
        count = 0;
    } else {
        updates = array_get_modifiable(&t->ext_rec_updates, &count);
    }

    if (!array_is_created(&t->ext_reset_ids)) {
        reset_ids = NULL;
        reset_id_count = 0;
    } else {
        reset_ids = array_get_modifiable(&t->ext_reset_ids,
                         &reset_id_count);
    }

    for (ext_id = 0; ext_id < count; ext_id++) {
        if (!array_is_created(&updates[ext_id]))
            continue;

        reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
        log_append_ext_intro(ctx, ext_id, reset_id);

        log_append_buffer(ctx, updates[ext_id].arr.buffer, NULL,
                  MAIL_TRANSACTION_EXT_REC_UPDATE);
    }
}

static void
log_append_keyword_update(struct log_append_context *ctx,
              buffer_t *hdr_buf, enum modify_type modify_type,
              const char *keyword, const buffer_t *buffer)
{
    struct mail_transaction_keyword_update kt_hdr;

    memset(&kt_hdr, 0, sizeof(kt_hdr));
    kt_hdr.modify_type = modify_type;
    kt_hdr.name_size = strlen(keyword);

    buffer_set_used_size(hdr_buf, 0);
    buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
    buffer_append(hdr_buf, keyword, kt_hdr.name_size);
    if ((hdr_buf->used % 4) != 0)
        buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));

    log_append_buffer(ctx, buffer, hdr_buf,
              MAIL_TRANSACTION_KEYWORD_UPDATE);
}

static enum mail_index_sync_type
log_append_keyword_updates(struct log_append_context *ctx)
{
        const struct mail_index_transaction_keyword_update *updates;
    const char *const *keywords;
    buffer_t *hdr_buf;
    enum mail_index_sync_type change_mask = 0;
    unsigned int i, count, keywords_count;

    hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);

    keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
                    &keywords_count);
    updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
    i_assert(count <= keywords_count);

    for (i = 0; i < count; i++) {
        if (array_is_created(&updates[i].add_seq)) {
            change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
            log_append_keyword_update(ctx, hdr_buf,
                    MODIFY_ADD, keywords[i],
                    updates[i].add_seq.arr.buffer);
        }
        if (array_is_created(&updates[i].remove_seq)) {
            change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
            log_append_keyword_update(ctx, hdr_buf,
                    MODIFY_REMOVE, keywords[i],
                    updates[i].remove_seq.arr.buffer);
        }
    }
    return change_mask;
}

static void log_append_sync_offset_if_needed(struct log_append_context *ctx)
{
    struct mail_transaction_header_update *u;
    buffer_t *buf;
    uint32_t offset;

    if (ctx->file->max_tail_offset == ctx->file->sync_offset) {
        /* FIXME: when we remove exclusive log locking, we
           can't rely on this. then write non-changed offset + check
           real offset + rewrite the new offset if other transactions
           weren't written in the middle */
        ctx->file->max_tail_offset += ctx->output->used +
            sizeof(struct mail_transaction_header) +
            sizeof(*u) + sizeof(offset);
        ctx->sync_includes_this = TRUE;
    }
    offset = ctx->file->max_tail_offset;

    if (ctx->file->saved_tail_offset == offset)
        return;
    i_assert(offset > ctx->file->saved_tail_offset);

    buf = buffer_create_static_hard(pool_datastack_create(),
                    sizeof(*u) + sizeof(offset));
    u = buffer_append_space_unsafe(buf, sizeof(*u));
    u->offset = offsetof(struct mail_index_header, log_file_tail_offset);
    u->size = sizeof(offset);
    buffer_append(buf, &offset, sizeof(offset));

    log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_HEADER_UPDATE);
}

#define TRANSACTION_HAS_CHANGES(t) \
    ((t)->log_updates || (t)->log_ext_updates || \
     (array_is_created(&(t)->updates) && array_count(&(t)->updates) > 0))

static int
mail_transaction_log_append_locked(struct mail_index_transaction *t,
                   uint32_t *log_file_seq_r,
                   uoff_t *log_file_offset_r)
{
    enum mail_index_sync_type change_mask = 0;
    struct mail_index_view *view = t->view;
    struct mail_index *index;
    struct mail_transaction_log *log;
    struct mail_transaction_log_file *file;
    struct log_append_context ctx;
    uoff_t append_offset;
    bool want_fsync;

    index = mail_index_view_get_index(view);
    log = index->log;

    if (t->reset) {
        /* Reset the whole index, preserving only indexid. Begin by
           rotating the log. We don't care if we skip some non-synced
           transactions. */
        if (mail_transaction_log_rotate(log, TRUE) < 0)
            return -1;

        if (!TRANSACTION_HAS_CHANGES(t)) {
            /* we only wanted to reset */
            return 0;
        }
    }

    if (!index->log_locked) {
        /* update sync_offset */
        if (mail_transaction_log_file_map(log->head,
                          log->head->sync_offset,
                          (uoff_t)-1) <= 0)
            return -1;
    }

    if (array_is_created(&t->ext_reset_atomic) || t->max_modseq != 0) {
        if (mail_index_map(t->view->index,
                   MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
            return -1;
    }
    if (array_is_created(&t->ext_reset_atomic))
        transaction_update_atomic_reset_ids(t);
    if (t->max_modseq != 0)
        mail_index_transaction_check_conflicts(t);
    if (!TRANSACTION_HAS_CHANGES(t)) {
        /* we aborted all changes, nothing else to do */
        return 0;
    }
    /* finally convert all sequences to UIDs before we write them,
       but after we've checked and removed conflicts */
    mail_index_transaction_convert_to_uids(t);

    file = log->head;

    i_assert(file->sync_offset >= file->buffer_offset);

    memset(&ctx, 0, sizeof(ctx));
    ctx.file = file;
    ctx.trans = t;
    ctx.output = buffer_create_dynamic(default_pool, 1024);
    ctx.modseq = file->sync_highest_modseq;

    /* send all extension introductions and resizes before appends
       to avoid resize overhead as much as possible */
        mail_transaction_log_append_ext_intros(&ctx);

    if (t->pre_hdr_changed) {
        log_append_buffer(&ctx,
                  log_get_hdr_update_buffer(t, TRUE),
                  NULL, MAIL_TRANSACTION_HEADER_UPDATE);
    }
    if (array_is_created(&t->appends)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_APPEND;
        log_append_buffer(&ctx, t->appends.arr.buffer, NULL,
                  MAIL_TRANSACTION_APPEND);
    }
    if (array_is_created(&t->updates)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS;
        log_append_buffer(&ctx, t->updates.arr.buffer, NULL,
                  MAIL_TRANSACTION_FLAG_UPDATE);
    }

    if (array_is_created(&t->ext_rec_updates))
        log_append_ext_rec_updates(&ctx);

    /* keyword resets before updates */
    if (array_is_created(&t->keyword_resets)) {
        change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
        log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
                  NULL, MAIL_TRANSACTION_KEYWORD_RESET);
    }
    if (array_is_created(&t->keyword_updates))
        change_mask |= log_append_keyword_updates(&ctx);

    if (array_is_created(&t->expunges)) {
        /* non-external expunges are only requests, ignore them when
           checking fsync_mask */
        if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
            change_mask |= MAIL_INDEX_SYNC_TYPE_EXPUNGE;
        log_append_buffer(&ctx, t->expunges.arr.buffer, NULL,
                  MAIL_TRANSACTION_EXPUNGE);
    }

    if (t->post_hdr_changed) {
        log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
                  NULL, MAIL_TRANSACTION_HEADER_UPDATE);
    }

    /* NOTE: mailbox sync offset update must be the last change.
       it may update the sync offset to include this transaction, so it
       needs to know this transaction's size */
    if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
        log_append_sync_offset_if_needed(&ctx);

    if (file->sync_offset < file->last_size) {
        /* there is some garbage at the end of the transaction log
           (eg. previous write failed). remove it so reader doesn't
           break because of it. */
        buffer_set_used_size(file->buffer,
                     file->sync_offset - file->buffer_offset);
        if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
            if (ftruncate(file->fd, file->sync_offset) < 0) {
                mail_index_file_set_syscall_error(index,
                    file->filepath, "ftruncate()");
            }
        }
    }

    want_fsync = (view->index->fsync_mask & change_mask) != 0;
    append_offset = file->sync_offset;
    if (log_buffer_write(&ctx, want_fsync) < 0) {
        buffer_free(&ctx.output);
        return -1;
    }
    file->sync_highest_modseq = ctx.modseq;
    buffer_free(&ctx.output);

    if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0) {
        /* mark the area covered by this transaction hidden */
        mail_index_view_add_hidden_transaction(view, file->hdr.file_seq,
            append_offset, file->sync_offset - append_offset);
    }

    *log_file_seq_r = file->hdr.file_seq;
    *log_file_offset_r = file->sync_offset;
    return 0;
}

int mail_transaction_log_append(struct mail_index_transaction *t,
                uint32_t *log_file_seq_r,
                uoff_t *log_file_offset_r)
{
    struct mail_index *index;
    int ret;

    *log_file_seq_r = 0;
    *log_file_offset_r = 0;

    if (!TRANSACTION_HAS_CHANGES(t) && !t->reset) {
        /* nothing to append */
        return 0;
    }

    index = mail_index_view_get_index(t->view);
    if (index->log_locked) {
        i_assert((t->flags &
              MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0);
    } else {
        if (mail_transaction_log_lock_head(index->log) < 0)
            return -1;
    }

    ret = mail_transaction_log_append_locked(t, log_file_seq_r,
                         log_file_offset_r);

    if (!index->log_locked)
        mail_transaction_log_file_unlock(index->log->head);
    return ret;
}