mail-transaction-log-append.c revision 5e96e0a8a59aa5d3e2d38a21a211335a023fbbc8
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "lib.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "array.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "buffer.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "write-full.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "mail-index-private.h"
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen#include "mail-index-view-private.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "mail-index-transaction-private.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen#include "mail-transaction-log-private.h"
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainenstruct log_append_context {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen struct mail_transaction_log_file *file;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen struct mail_index_transaction *trans;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen buffer_t *output;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen uint32_t first_append_size;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen bool sync_includes_this;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen};
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainenstatic void log_append_buffer(struct log_append_context *ctx,
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen const buffer_t *buf, const buffer_t *hdr_buf,
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen enum mail_transaction_type type)
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen{
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen struct mail_transaction_header hdr;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen uint32_t hdr_size;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_assert((buf->used % 4) == 0);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_assert(hdr_buf == NULL || (hdr_buf->used % 4) == 0);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (buf->used == 0)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen memset(&hdr, 0, sizeof(hdr));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr.type = type;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (type == MAIL_TRANSACTION_EXPUNGE)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ctx->trans->external)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr.type |= MAIL_TRANSACTION_EXTERNAL;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr_size = mail_index_uint32_to_offset(sizeof(hdr) + buf->used +
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen (hdr_buf == NULL ? 0 :
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr_buf->used));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) &&
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ctx->first_append_size == 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* size will be written later once everything
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen is in disk */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ctx->first_append_size = hdr_size;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen } else {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr.size = hdr_size;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(ctx->output, &hdr, sizeof(hdr));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (hdr_buf != NULL)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(ctx->output, buf->data, buf->used);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic int log_buffer_move_to_memory(struct log_append_context *ctx)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_log_file *file = ctx->file;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* first we need to truncate this latest write so that log syncing
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen doesn't break */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ftruncate(file->fd, file->sync_offset) < 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mail_index_file_set_syscall_error(file->log->index,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->filepath,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen "ftruncate()");
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (mail_index_move_to_memory(file->log->index) < 0)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return -1;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_assert(MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_assert(file->buffer_offset + file->buffer->used ==
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->sync_offset);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append_buf(file->buffer, ctx->output, 0, (size_t)-1);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen file->sync_offset = file->buffer_offset + file->buffer->used;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen return 0;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen}
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainenstatic int log_buffer_write(struct log_append_context *ctx)
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen{
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen struct mail_transaction_log_file *file = ctx->file;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen buffer_append_buf(file->buffer, ctx->output, 0, (size_t)-1);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->sync_offset = file->buffer_offset + file->buffer->used;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return 0;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen i_assert(ctx->first_append_size != 0);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (pwrite_full(file->fd, ctx->output->data, ctx->output->used,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->sync_offset) < 0) {
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen /* write failure, fallback to in-memory indexes. */
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->filepath,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen "pwrite_full()");
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return log_buffer_move_to_memory(ctx);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen i_assert(!ctx->sync_includes_this ||
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->sync_offset + ctx->output->used ==
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->max_tail_offset);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (!file->log->index->fsync_disable && fdatasync(file->fd) < 0) {
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen mail_index_file_set_syscall_error(file->log->index,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->filepath,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen "fsync()");
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return log_buffer_move_to_memory(ctx);
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen }
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen /* now that the whole transaction has been written, rewrite the first
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen record's size so the transaction becomes visible */
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if (pwrite_full(file->fd, &ctx->first_append_size,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen sizeof(uint32_t), file->sync_offset) < 0) {
eab880d75fd73a80c7803289796d13e08e4b52cbTimo Sirainen mail_index_file_set_syscall_error(file->log->index,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->filepath,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen "pwrite_full()");
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return log_buffer_move_to_memory(ctx);
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen }
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen /* FIXME: when we're relying on O_APPEND and someone else wrote a
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen transaction, we'll need to wait for it to commit its transaction.
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen if it crashes before doing that, we'll need to overwrite it with
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen a dummy record */
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen file->sync_offset += ctx->output->used;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen return 0;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen}
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainenstatic const buffer_t *
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainenlog_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen{
bed3fcccce6a15dfd7491e82df10acd5f5c0bc9bTimo Sirainen buffer_t *buf;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen const unsigned char *data, *mask;
e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainen struct mail_transaction_header_update u;
e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainen uint16_t offset;
e06147cec0f68a6a9d08b38785c36d00ef73868aTimo Sirainen int state = 0;
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen memset(&u, 0, sizeof(u));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen data = prepend ? t->pre_hdr_change : t->post_hdr_change;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 256);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (state == 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen u.offset = offset;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen state++;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen } else {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (state > 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen u.size = offset - u.offset;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(buf, &u, sizeof(u));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(buf, data + u.offset, u.size);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen state = 0;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen return buf;
02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen}
02bb8313a711dfe50c7f01e8132e13ca93ecfb42Timo Sirainen
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainenstatic void log_append_ext_intro(struct log_append_context *ctx,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen uint32_t ext_id, uint32_t reset_id)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_index_transaction *t = ctx->trans;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const struct mail_index_registered_ext *rext;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_ext_intro *intro;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_t *buf;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen uint32_t idx;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen unsigned int count;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (t->reset ||
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen !mail_index_map_get_ext_idx(t->view->map, ext_id, &idx)) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* new extension */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen idx = (uint32_t)-1;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen rext = array_idx(&t->view->index->extensions, ext_id);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!array_is_created(&t->ext_resizes)) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro = NULL;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen count = 0;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen } else {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro = array_get_modifiable(&t->ext_resizes, &count);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 128);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ext_id < count && intro[ext_id].name_size != 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* we're resizing it */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro += ext_id;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen i_assert(intro->ext_id == idx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->name_size = idx != (uint32_t)-1 ? 0 :
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen strlen(rext->name);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(buf, intro, sizeof(*intro));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen } else {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* generate a new intro structure */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro = buffer_append_space_unsafe(buf, sizeof(*intro));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->ext_id = idx;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->hdr_size = rext->hdr_size;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->record_size = rext->record_size;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->record_align = rext->record_align;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->name_size = idx != (uint32_t)-1 ? 0 :
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen strlen(rext->name);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (reset_id != 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* we're going to reset this extension in this transaction */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->reset_id = reset_id;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen } else if (idx != (uint32_t)-1) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* use the existing reset_id */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const struct mail_index_ext *map_ext =
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen array_idx(&t->view->map->extensions, idx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->reset_id = map_ext->reset_id;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen } else {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* new extension, reset_id defaults to 0 */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(buf, rext->name, intro->name_size);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if ((buf->used % 4) != 0)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append_zero(buf, 4 - (buf->used % 4));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic void
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenlog_append_ext_hdr_update(struct log_append_context *ctx,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_index_transaction_ext_hdr_update *hdr)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_ext_hdr_update *trans_hdr;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_t *buf;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen unsigned int hdr_size;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen t_push();
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdr_size = sizeof(*trans_hdr) + hdr->size;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buf = buffer_create_static_hard(pool_datastack_create(), hdr_size);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen trans_hdr = buffer_append_space_unsafe(buf, sizeof(*trans_hdr));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen trans_hdr->offset = hdr->offset;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen trans_hdr->size = hdr->size;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(buf, hdr + 1, hdr->size);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_HDR_UPDATE);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen t_pop();
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic void
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenmail_transaction_log_append_ext_intros(struct log_append_context *ctx)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_index_transaction *t = ctx->trans;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const struct mail_transaction_ext_intro *resize;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_index_transaction_ext_hdr_update *const *hdrs;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_ext_reset ext_reset;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen unsigned int update_count, resize_count, ext_count = 0;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen unsigned int hdrs_count, reset_id_count, reset_count;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen uint32_t ext_id, reset_id;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const uint32_t *reset_ids, *reset;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const ARRAY_TYPE(seq_array) *update;
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen buffer_t *buf;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!array_is_created(&t->ext_rec_updates)) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen update = NULL;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen update_count = 0;
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen } else {
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen update = array_get(&t->ext_rec_updates, &update_count);
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen ext_count = update_count;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (!array_is_created(&t->ext_resizes)) {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen resize = NULL;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen resize_count = 0;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen } else {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen resize = array_get(&t->ext_resizes, &resize_count);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (ext_count < resize_count)
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen ext_count = resize_count;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen }
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (!array_is_created(&t->ext_reset_ids)) {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen reset_ids = NULL;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen reset_id_count = 0;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen } else {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen }
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (!array_is_created(&t->ext_resets)) {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen reset = NULL;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen reset_count = 0;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen } else {
3a017aa592823edf0363d77f13458d569637915eTimo Sirainen reset = array_get(&t->ext_resets, &reset_count);
3a017aa592823edf0363d77f13458d569637915eTimo Sirainen if (ext_count < reset_count)
3a017aa592823edf0363d77f13458d569637915eTimo Sirainen ext_count = reset_count;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen }
3a017aa592823edf0363d77f13458d569637915eTimo Sirainen
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if (!array_is_created(&t->ext_hdr_updates)) {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen hdrs = NULL;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen hdrs_count = 0;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen } else {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ext_count < hdrs_count)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ext_count = hdrs_count;
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen memset(&ext_reset, 0, sizeof(ext_reset));
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen buf = buffer_create_data(pool_datastack_create(),
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen &ext_reset, sizeof(ext_reset));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_set_used_size(buf, sizeof(ext_reset));
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen for (ext_id = 0; ext_id < ext_count; ext_id++) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ext_reset.new_reset_id =
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ext_id < reset_count && reset[ext_id] != 0 ?
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen reset[ext_id] : 0;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if ((ext_id < resize_count && resize[ext_id].name_size) ||
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen (ext_id < update_count &&
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen array_is_created(&update[ext_id])) ||
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen ext_reset.new_reset_id != 0 ||
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen (ext_id < hdrs_count && hdrs[ext_id] != NULL)) {
5e88e4624aa6d482b5b195acd2f4e02aeb385f20Timo Sirainen reset_id = ext_id < reset_id_count ?
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen reset_ids[ext_id] : 0;
b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen log_append_ext_intro(ctx, ext_id, reset_id);
b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen }
b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen if (ext_reset.new_reset_id != 0) {
b346610430690398b8c840006004a2df4aa8ce92Timo Sirainen log_append_buffer(ctx, buf, NULL,
54b51a9c2705a19dfb1639647bc7e9378e37f881Timo Sirainen MAIL_TRANSACTION_EXT_RESET);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ext_id < hdrs_count && hdrs[ext_id] != NULL)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen log_append_ext_hdr_update(ctx, hdrs[ext_id]);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen }
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen}
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic void log_append_ext_rec_updates(struct log_append_context *ctx)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen{
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen struct mail_index_transaction *t = ctx->trans;
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen ARRAY_TYPE(seq_array) *updates;
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen const uint32_t *reset_ids;
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen unsigned int ext_id, count, reset_id_count;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen uint32_t reset_id;
if (!array_is_created(&t->ext_rec_updates)) {
updates = NULL;
count = 0;
} else {
updates = array_get_modifiable(&t->ext_rec_updates, &count);
}
if (!array_is_created(&t->ext_reset_ids)) {
reset_ids = NULL;
reset_id_count = 0;
} else {
reset_ids = array_get_modifiable(&t->ext_reset_ids,
&reset_id_count);
}
for (ext_id = 0; ext_id < count; ext_id++) {
if (!array_is_created(&updates[ext_id]))
continue;
reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
log_append_ext_intro(ctx, ext_id, reset_id);
log_append_buffer(ctx, updates[ext_id].arr.buffer, NULL,
MAIL_TRANSACTION_EXT_REC_UPDATE);
}
}
static void
log_append_keyword_update(struct log_append_context *ctx,
buffer_t *hdr_buf, enum modify_type modify_type,
const char *keyword, const buffer_t *buffer)
{
struct mail_transaction_keyword_update kt_hdr;
memset(&kt_hdr, 0, sizeof(kt_hdr));
kt_hdr.modify_type = modify_type;
kt_hdr.name_size = strlen(keyword);
buffer_set_used_size(hdr_buf, 0);
buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
buffer_append(hdr_buf, keyword, kt_hdr.name_size);
if ((hdr_buf->used % 4) != 0)
buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));
log_append_buffer(ctx, buffer, hdr_buf,
MAIL_TRANSACTION_KEYWORD_UPDATE);
}
static void log_append_keyword_updates(struct log_append_context *ctx)
{
const struct mail_index_transaction_keyword_update *updates;
const char *const *keywords;
buffer_t *hdr_buf;
unsigned int i, count, keywords_count;
hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);
keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
&keywords_count);
updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
i_assert(count <= keywords_count);
for (i = 0; i < count; i++) {
if (array_is_created(&updates[i].add_seq)) {
log_append_keyword_update(ctx, hdr_buf,
MODIFY_ADD, keywords[i],
updates[i].add_seq.arr.buffer);
}
if (array_is_created(&updates[i].remove_seq)) {
log_append_keyword_update(ctx, hdr_buf,
MODIFY_REMOVE, keywords[i],
updates[i].remove_seq.arr.buffer);
}
}
}
static void log_append_sync_offset_if_needed(struct log_append_context *ctx)
{
struct mail_transaction_header_update *u;
buffer_t *buf;
uint32_t offset;
if (ctx->file->max_tail_offset == ctx->file->sync_offset) {
/* FIXME: when we remove exclusive log locking, we
can't rely on this. then write non-changed offset + check
real offset + rewrite the new offset if other transactions
weren't written in the middle */
ctx->file->max_tail_offset += ctx->output->used +
sizeof(struct mail_transaction_header) +
sizeof(*u) + sizeof(offset);
ctx->sync_includes_this = TRUE;
}
offset = ctx->file->max_tail_offset;
if (ctx->file->saved_tail_offset == offset)
return;
buf = buffer_create_static_hard(pool_datastack_create(),
sizeof(*u) + sizeof(offset));
u = buffer_append_space_unsafe(buf, sizeof(*u));
u->offset = offsetof(struct mail_index_header, log_file_tail_offset);
u->size = sizeof(offset);
buffer_append(buf, &offset, sizeof(offset));
log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_HEADER_UPDATE);
}
static int
mail_transaction_log_append_locked(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r)
{
struct mail_index_view *view = t->view;
struct mail_index *index;
struct mail_transaction_log *log;
struct mail_transaction_log_file *file;
struct log_append_context ctx;
uoff_t append_offset;
index = mail_index_view_get_index(view);
log = index->log;
if (t->reset) {
/* Reset the whole index, preserving only indexid. Begin by
rotating the log. We don't care if we skip some non-synced
transactions. */
if (mail_transaction_log_rotate(log, TRUE) < 0)
return -1;
}
if (!index->log_locked) {
/* update sync_offset */
if (mail_transaction_log_file_map(log->head,
log->head->sync_offset,
(uoff_t)-1) <= 0)
return -1;
}
file = log->head;
if (file->sync_offset < file->buffer_offset)
file->sync_offset = file->buffer_offset;
memset(&ctx, 0, sizeof(ctx));
ctx.file = file;
ctx.trans = t;
ctx.output = buffer_create_dynamic(default_pool, 1024);
/* send all extension introductions and resizes before appends
to avoid resize overhead as much as possible */
mail_transaction_log_append_ext_intros(&ctx);
if (t->pre_hdr_changed) {
log_append_buffer(&ctx,
log_get_hdr_update_buffer(t, TRUE),
NULL, MAIL_TRANSACTION_HEADER_UPDATE);
}
if (array_is_created(&t->appends)) {
log_append_buffer(&ctx, t->appends.arr.buffer, NULL,
MAIL_TRANSACTION_APPEND);
}
if (array_is_created(&t->updates)) {
log_append_buffer(&ctx, t->updates.arr.buffer, NULL,
MAIL_TRANSACTION_FLAG_UPDATE);
}
if (array_is_created(&t->ext_rec_updates))
log_append_ext_rec_updates(&ctx);
/* keyword resets before updates */
if (array_is_created(&t->keyword_resets)) {
log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
NULL, MAIL_TRANSACTION_KEYWORD_RESET);
}
if (array_is_created(&t->keyword_updates))
log_append_keyword_updates(&ctx);
if (array_is_created(&t->expunges)) {
log_append_buffer(&ctx, t->expunges.arr.buffer, NULL,
MAIL_TRANSACTION_EXPUNGE);
}
if (t->post_hdr_changed) {
log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
NULL, MAIL_TRANSACTION_HEADER_UPDATE);
}
/* NOTE: mailbox sync offset update must be the last change.
it may update the sync offset to include this transaction, so it
needs to know this transaction's size */
if (t->external)
log_append_sync_offset_if_needed(&ctx);
if (file->sync_offset < file->last_size) {
/* there is some garbage at the end of the transaction log
(eg. previous write failed). remove it so reader doesn't
break because of it. */
buffer_set_used_size(file->buffer,
file->sync_offset - file->buffer_offset);
if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
if (ftruncate(file->fd, file->sync_offset) < 0) {
mail_index_file_set_syscall_error(index,
file->filepath, "ftruncate()");
}
}
}
append_offset = file->sync_offset;
if (log_buffer_write(&ctx) < 0) {
buffer_free(ctx.output);
return -1;
}
buffer_free(ctx.output);
if (t->hide_transaction) {
/* mark the area covered by this transaction hidden */
mail_index_view_add_hidden_transaction(view, file->hdr.file_seq,
append_offset, file->sync_offset - append_offset);
}
*log_file_seq_r = file->hdr.file_seq;
*log_file_offset_r = file->sync_offset;
return 0;
}
int mail_transaction_log_append(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r)
{
struct mail_index *index;
int ret;
if (!t->log_updates) {
/* nothing to append */
*log_file_seq_r = 0;
*log_file_offset_r = 0;
return 0;
}
index = mail_index_view_get_index(t->view);
if (index->log_locked) {
i_assert(t->external);
} else {
if (mail_transaction_log_lock_head(index->log) < 0)
return -1;
}
ret = mail_transaction_log_append_locked(t, log_file_seq_r,
log_file_offset_r);
if (!index->log_locked)
mail_transaction_log_file_unlock(index->log->head);
return ret;
}