mail-index-transaction-export.c revision 724b7fcf28c2547eb9c837d0e99241c0501dccf3
0N/A/* Copyright (c) 2003-2012 Dovecot authors, see the included COPYING file */
0N/A
0N/A#include "lib.h"
0N/A#include "array.h"
0N/A#include "mail-index-private.h"
0N/A#include "mail-index-modseq.h"
115N/A#include "mail-transaction-log-private.h"
0N/A#include "mail-index-transaction-private.h"
0N/A
0N/Astruct mail_index_export_context {
0N/A struct mail_index_transaction *trans;
0N/A struct mail_transaction_log_append_ctx *append_ctx;
115N/A};
0N/A
0N/Astatic void
0N/Alog_append_buffer(struct mail_index_export_context *ctx,
0N/A const buffer_t *buf, enum mail_transaction_type type)
0N/A{
115N/A mail_transaction_log_append_add(ctx->append_ctx, type,
115N/A buf->data, buf->used);
115N/A}
13N/A
0N/Astatic void log_append_flag_updates(struct mail_index_export_context *ctx,
0N/A struct mail_index_transaction *t)
0N/A{
0N/A ARRAY(struct mail_transaction_flag_update) log_updates;
0N/A const struct mail_index_flag_update *updates;
0N/A struct mail_transaction_flag_update *log_update;
0N/A unsigned int i, count;
0N/A
0N/A updates = array_get(&t->updates, &count);
0N/A if (count == 0)
0N/A return;
0N/A
115N/A i_array_init(&log_updates, count);
115N/A
115N/A for (i = 0; i < count; i++) {
115N/A log_update = array_append_space(&log_updates);
115N/A log_update->uid1 = updates[i].uid1;
0N/A log_update->uid2 = updates[i].uid2;
0N/A log_update->add_flags = updates[i].add_flags & 0xff;
0N/A log_update->remove_flags = updates[i].remove_flags & 0xff;
0N/A if ((updates[i].add_flags & MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ) != 0)
0N/A log_update->modseq_inc_flag = 1;
0N/A }
0N/A log_append_buffer(ctx, log_updates.arr.buffer,
0N/A MAIL_TRANSACTION_FLAG_UPDATE);
0N/A array_free(&log_updates);
0N/A}
0N/A
0N/Astatic const buffer_t *
0N/Alog_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
0N/A{
0N/A buffer_t *buf;
0N/A const unsigned char *data, *mask;
0N/A struct mail_transaction_header_update u;
0N/A uint16_t offset;
0N/A int state = 0;
0N/A
0N/A memset(&u, 0, sizeof(u));
0N/A
0N/A data = prepend ? t->pre_hdr_change : t->post_hdr_change;
0N/A mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
0N/A
0N/A buf = buffer_create_dynamic(pool_datastack_create(), 256);
0N/A for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
0N/A if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
0N/A if (state == 0) {
0N/A u.offset = offset;
0N/A state++;
0N/A }
0N/A } else {
0N/A if (state > 0) {
0N/A u.size = offset - u.offset;
0N/A buffer_append(buf, &u, sizeof(u));
0N/A buffer_append(buf, data + u.offset, u.size);
0N/A state = 0;
0N/A }
115N/A }
0N/A }
0N/A return buf;
0N/A}
0N/A
0N/Astatic void log_append_ext_intro(struct mail_index_export_context *ctx,
0N/A uint32_t ext_id, uint32_t reset_id)
0N/A{
0N/A struct mail_index_transaction *t = ctx->trans;
0N/A const struct mail_index_registered_ext *rext;
0N/A struct mail_transaction_ext_intro *intro, *resizes;
0N/A buffer_t *buf;
0N/A uint32_t idx;
0N/A unsigned int count;
0N/A
0N/A i_assert(ext_id != (uint32_t)-1);
0N/A
0N/A if (t->reset ||
0N/A !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
0N/A /* new extension */
0N/A idx = (uint32_t)-1;
0N/A }
0N/A
0N/A rext = array_idx(&t->view->index->extensions, ext_id);
0N/A if (!array_is_created(&t->ext_resizes)) {
0N/A resizes = NULL;
0N/A count = 0;
0N/A } else {
0N/A resizes = array_get_modifiable(&t->ext_resizes, &count);
0N/A }
0N/A
0N/A buf = buffer_create_dynamic(pool_datastack_create(), 128);
0N/A if (ext_id < count && resizes[ext_id].name_size != 0) {
0N/A /* we're resizing the extension. use the resize struct. */
0N/A intro = &resizes[ext_id];
0N/A
0N/A i_assert(intro->ext_id == idx);
0N/A intro->name_size = idx != (uint32_t)-1 ? 0 :
0N/A strlen(rext->name);
13N/A buffer_append(buf, intro, sizeof(*intro));
} else {
/* generate a new intro structure */
intro = buffer_append_space_unsafe(buf, sizeof(*intro));
intro->ext_id = idx;
intro->hdr_size = rext->hdr_size;
intro->record_size = rext->record_size;
intro->record_align = rext->record_align;
intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
intro->name_size = idx != (uint32_t)-1 ? 0 :
strlen(rext->name);
}
if (reset_id != 0) {
/* we're going to reset this extension in this transaction */
intro->reset_id = reset_id;
} else if (idx != (uint32_t)-1) {
/* use the existing reset_id */
const struct mail_index_ext *map_ext =
array_idx(&t->view->index->map->extensions, idx);
intro->reset_id = map_ext->reset_id;
} else {
/* new extension, reset_id defaults to 0 */
}
buffer_append(buf, rext->name, intro->name_size);
if ((buf->used % 4) != 0)
buffer_append_zero(buf, 4 - (buf->used % 4));
if (ctx->append_ctx->new_highest_modseq == 0 &&
strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
/* modseq tracking started */
ctx->append_ctx->new_highest_modseq = 1;
}
log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_INTRO);
}
static void
log_append_ext_hdr_update(struct mail_index_export_context *ctx,
const struct mail_index_transaction_ext_hdr_update *hdr)
{
buffer_t *buf;
const unsigned char *data, *mask;
struct mail_transaction_ext_hdr_update u;
struct mail_transaction_ext_hdr_update32 u32;
size_t offset;
bool started = FALSE, use_32 = hdr->alloc_size >= 65536;
memset(&u, 0, sizeof(u));
memset(&u32, 0, sizeof(u32));
data = hdr->data;
mask = hdr->mask;
buf = buffer_create_dynamic(pool_datastack_create(), 256);
for (offset = 0; offset <= hdr->alloc_size; offset++) {
if (offset < hdr->alloc_size && mask[offset] != 0) {
if (!started) {
u32.offset = offset;
started = TRUE;
}
} else {
if (started) {
u32.size = offset - u32.offset;
if (use_32)
buffer_append(buf, &u32, sizeof(u32));
else {
u.offset = u32.offset;
u.size = u32.size;
buffer_append(buf, &u, sizeof(u));
}
buffer_append(buf, data + u32.offset, u32.size);
started = FALSE;
}
}
}
if (buf->used % 4 != 0)
buffer_append_zero(buf, 4 - buf->used % 4);
log_append_buffer(ctx, buf, use_32 ? MAIL_TRANSACTION_EXT_HDR_UPDATE32 :
MAIL_TRANSACTION_EXT_HDR_UPDATE);
}
static void
mail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx)
{
struct mail_index_transaction *t = ctx->trans;
const struct mail_transaction_ext_intro *resize;
const struct mail_index_transaction_ext_hdr_update *hdrs;
struct mail_transaction_ext_reset ext_reset;
unsigned int resize_count, ext_count = 0;
unsigned int hdrs_count, reset_id_count, reset_count;
uint32_t ext_id, reset_id;
const struct mail_transaction_ext_reset *reset;
const uint32_t *reset_ids;
buffer_t reset_buf;
if (!array_is_created(&t->ext_resizes)) {
resize = NULL;
resize_count = 0;
} else {
resize = array_get(&t->ext_resizes, &resize_count);
if (ext_count < resize_count)
ext_count = resize_count;
}
if (!array_is_created(&t->ext_reset_ids)) {
reset_ids = NULL;
reset_id_count = 0;
} else {
reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
}
if (!array_is_created(&t->ext_resets)) {
reset = NULL;
reset_count = 0;
} else {
reset = array_get(&t->ext_resets, &reset_count);
if (ext_count < reset_count)
ext_count = reset_count;
}
if (!array_is_created(&t->ext_hdr_updates)) {
hdrs = NULL;
hdrs_count = 0;
} else {
hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
if (ext_count < hdrs_count)
ext_count = hdrs_count;
}
memset(&ext_reset, 0, sizeof(ext_reset));
buffer_create_from_data(&reset_buf, &ext_reset, sizeof(ext_reset));
buffer_set_used_size(&reset_buf, sizeof(ext_reset));
for (ext_id = 0; ext_id < ext_count; ext_id++) {
if (ext_id < reset_count)
ext_reset = reset[ext_id];
else
ext_reset.new_reset_id = 0;
if ((ext_id < resize_count && resize[ext_id].name_size) ||
ext_reset.new_reset_id != 0 ||
(ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
if (ext_reset.new_reset_id != 0) {
/* we're going to reset this extension
immediately after the intro */
reset_id = 0;
} else {
reset_id = ext_id < reset_id_count ?
reset_ids[ext_id] : 0;
}
log_append_ext_intro(ctx, ext_id, reset_id);
}
if (ext_reset.new_reset_id != 0) {
i_assert(ext_id < reset_id_count &&
ext_reset.new_reset_id == reset_ids[ext_id]);
log_append_buffer(ctx, &reset_buf,
MAIL_TRANSACTION_EXT_RESET);
}
if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
T_BEGIN {
log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
} T_END;
}
}
}
static void log_append_ext_recs(struct mail_index_export_context *ctx,
const ARRAY_TYPE(seq_array_array) *arr,
enum mail_transaction_type type)
{
struct mail_index_transaction *t = ctx->trans;
const ARRAY_TYPE(seq_array) *updates;
const uint32_t *reset_ids;
unsigned int ext_id, count, reset_id_count;
uint32_t reset_id;
if (!array_is_created(&t->ext_reset_ids)) {
reset_ids = NULL;
reset_id_count = 0;
} else {
reset_ids = array_get_modifiable(&t->ext_reset_ids,
&reset_id_count);
}
updates = array_get(arr, &count);
for (ext_id = 0; ext_id < count; ext_id++) {
if (!array_is_created(&updates[ext_id]))
continue;
reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
log_append_ext_intro(ctx, ext_id, reset_id);
log_append_buffer(ctx, updates[ext_id].arr.buffer, type);
}
}
static void
log_append_keyword_update(struct mail_index_export_context *ctx,
buffer_t *tmp_buf, enum modify_type modify_type,
const char *keyword, const buffer_t *uid_buffer)
{
struct mail_transaction_keyword_update kt_hdr;
i_assert(uid_buffer->used > 0);
memset(&kt_hdr, 0, sizeof(kt_hdr));
kt_hdr.modify_type = modify_type;
kt_hdr.name_size = strlen(keyword);
buffer_set_used_size(tmp_buf, 0);
buffer_append(tmp_buf, &kt_hdr, sizeof(kt_hdr));
buffer_append(tmp_buf, keyword, kt_hdr.name_size);
if ((tmp_buf->used % 4) != 0)
buffer_append_zero(tmp_buf, 4 - (tmp_buf->used % 4));
buffer_append(tmp_buf, uid_buffer->data, uid_buffer->used);
log_append_buffer(ctx, tmp_buf, MAIL_TRANSACTION_KEYWORD_UPDATE);
}
static enum mail_index_fsync_mask
log_append_keyword_updates(struct mail_index_export_context *ctx)
{
const struct mail_index_transaction_keyword_update *updates;
const char *const *keywords;
buffer_t *tmp_buf;
enum mail_index_fsync_mask change_mask = 0;
unsigned int i, count, keywords_count;
tmp_buf = buffer_create_dynamic(pool_datastack_create(), 64);
keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
&keywords_count);
updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
i_assert(count <= keywords_count);
for (i = 0; i < count; i++) {
if (array_is_created(&updates[i].add_seq) &&
array_count(&updates[i].add_seq) > 0) {
change_mask |= MAIL_INDEX_FSYNC_MASK_KEYWORDS;
log_append_keyword_update(ctx, tmp_buf,
MODIFY_ADD, keywords[i],
updates[i].add_seq.arr.buffer);
}
if (array_is_created(&updates[i].remove_seq) &&
array_count(&updates[i].remove_seq) > 0) {
change_mask |= MAIL_INDEX_FSYNC_MASK_KEYWORDS;
log_append_keyword_update(ctx, tmp_buf,
MODIFY_REMOVE, keywords[i],
updates[i].remove_seq.arr.buffer);
}
}
return change_mask;
}
void mail_index_transaction_export(struct mail_index_transaction *t,
struct mail_transaction_log_append_ctx *append_ctx)
{
static uint8_t null4[4] = { 0, 0, 0, 0 };
enum mail_index_fsync_mask change_mask = 0;
struct mail_index_export_context ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.trans = t;
ctx.append_ctx = append_ctx;
if (t->index_undeleted) {
i_assert(!t->index_deleted);
mail_transaction_log_append_add(ctx.append_ctx,
MAIL_TRANSACTION_INDEX_UNDELETED, &null4, 4);
}
/* send all extension introductions and resizes before appends
to avoid resize overhead as much as possible */
mail_transaction_log_append_ext_intros(&ctx);
if (t->pre_hdr_changed) {
log_append_buffer(&ctx, log_get_hdr_update_buffer(t, TRUE),
MAIL_TRANSACTION_HEADER_UPDATE);
}
if (array_is_created(&t->appends)) {
change_mask |= MAIL_INDEX_FSYNC_MASK_APPENDS;
log_append_buffer(&ctx, t->appends.arr.buffer,
MAIL_TRANSACTION_APPEND);
}
if (array_is_created(&t->updates)) {
change_mask |= MAIL_INDEX_FSYNC_MASK_FLAGS;
log_append_flag_updates(&ctx, t);
}
if (array_is_created(&t->ext_rec_updates)) {
log_append_ext_recs(&ctx, &t->ext_rec_updates,
MAIL_TRANSACTION_EXT_REC_UPDATE);
}
if (array_is_created(&t->ext_rec_atomics)) {
log_append_ext_recs(&ctx, &t->ext_rec_atomics,
MAIL_TRANSACTION_EXT_ATOMIC_INC);
}
if (array_is_created(&t->keyword_updates))
change_mask |= log_append_keyword_updates(&ctx);
/* keep modseq updates almost last */
if (array_is_created(&t->modseq_updates)) {
log_append_buffer(&ctx, t->modseq_updates.arr.buffer,
MAIL_TRANSACTION_MODSEQ_UPDATE);
}
if (array_is_created(&t->expunges)) {
/* non-external expunges are only requests, ignore them when
checking fsync_mask */
if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
change_mask |= MAIL_INDEX_FSYNC_MASK_EXPUNGES;
log_append_buffer(&ctx, t->expunges.arr.buffer,
MAIL_TRANSACTION_EXPUNGE_GUID);
}
if (t->post_hdr_changed) {
log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
MAIL_TRANSACTION_HEADER_UPDATE);
}
if (t->index_deleted) {
i_assert(!t->index_undeleted);
mail_transaction_log_append_add(ctx.append_ctx,
MAIL_TRANSACTION_INDEX_DELETED,
&null4, 4);
}
/* Update the tail offsets only when committing the sync transaction.
Other transactions may not know the latest tail offset and might
end up shrinking it. (Alternatively the shrinking tail offsets could
just be ignored, which would probably work fine too.) */
append_ctx->append_sync_offset = t->sync_transaction;
append_ctx->want_fsync =
(t->view->index->fsync_mask & change_mask) != 0 ||
(t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;
}