mail-index-transaction-export.c revision fad068d459cc7b04fedade4e0bb343be62e6d310
e59faf65ce864fe95dc00f5d52b8323cdbd0608aTimo Sirainen/* Copyright (c) 2003-2010 Dovecot authors, see the included COPYING file */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_log_append_ctx *append_ctx;
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainenlog_append_buffer(struct mail_index_export_context *ctx,
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen const buffer_t *buf, enum mail_transaction_type type)
40a5aeebf6b4858b93f0ddff0bf12fba769cf903Timo Sirainen mail_transaction_log_append_add(ctx->append_ctx, type,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenlog_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen memset(&u, 0, sizeof(u));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen data = prepend ? t->pre_hdr_change : t->post_hdr_change;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 256);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic void log_append_ext_intro(struct mail_index_export_context *ctx,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_index_transaction *t = ctx->trans;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_ext_intro *intro, *resizes;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen unsigned int count;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* new extension */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen rext = array_idx(&t->view->index->extensions, ext_id);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen resizes = array_get_modifiable(&t->ext_resizes, &count);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 128);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ext_id < count && resizes[ext_id].name_size != 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* we're resizing the extension. use the resize struct. */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* generate a new intro structure */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro = buffer_append_space_unsafe(buf, sizeof(*intro));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* we're going to reset this extension in this transaction */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* use the existing reset_id */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen array_idx(&t->view->index->map->extensions, idx);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* new extension, reset_id defaults to 0 */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append(buf, rext->name, intro->name_size);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ctx->append_ctx->new_highest_modseq == 0 &&
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* modseq tracking started */
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_INTRO);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenlog_append_ext_hdr_update(struct mail_index_export_context *ctx,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const struct mail_index_transaction_ext_hdr_update *hdr)
e76f5e07be5bec4e5ca99c3e093ff7f11edbe1b7Timo Sirainen bool started = FALSE, use_32 = hdr->alloc_size >= 65536;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen memset(&u, 0, sizeof(u));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 256);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen for (offset = 0; offset <= hdr->alloc_size; offset++) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (offset < hdr->alloc_size && mask[offset] != 0) {
e76f5e07be5bec4e5ca99c3e093ff7f11edbe1b7Timo Sirainen buffer_append(buf, data + u32.offset, u32.size);
e76f5e07be5bec4e5ca99c3e093ff7f11edbe1b7Timo Sirainen log_append_buffer(ctx, buf, use_32 ? MAIL_TRANSACTION_EXT_HDR_UPDATE32 :
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenmail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_index_transaction *t = ctx->trans;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const struct mail_transaction_ext_intro *resize;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const struct mail_index_transaction_ext_hdr_update *hdrs;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen unsigned int hdrs_count, reset_id_count, reset_count;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const struct mail_transaction_ext_reset *reset;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen resize = array_get(&t->ext_resizes, &resize_count);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen reset = array_get(&t->ext_resets, &reset_count);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
8e361d2906b0e44f7175a20981f8d2280645b58bTimo Sirainen buffer_create_data(&reset_buf, &ext_reset, sizeof(ext_reset));
8e361d2906b0e44f7175a20981f8d2280645b58bTimo Sirainen buffer_set_used_size(&reset_buf, sizeof(ext_reset));
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen for (ext_id = 0; ext_id < ext_count; ext_id++) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if ((ext_id < resize_count && resize[ext_id].name_size) ||
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* we're going to reset this extension
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen immediately after the intro */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenstatic void log_append_ext_recs(struct mail_index_export_context *ctx,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_index_transaction *t = ctx->trans;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen reset_ids = array_get_modifiable(&t->ext_reset_ids,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen log_append_buffer(ctx, updates[ext_id].arr.buffer, type);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenlog_append_keyword_update(struct mail_index_export_context *ctx,
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen buffer_t *tmp_buf, enum modify_type modify_type,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const char *keyword, const buffer_t *uid_buffer)
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen struct mail_transaction_keyword_update kt_hdr;
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen buffer_append(tmp_buf, &kt_hdr, sizeof(kt_hdr));
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen buffer_append(tmp_buf, keyword, kt_hdr.name_size);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen buffer_append_zero(tmp_buf, 4 - (tmp_buf->used % 4));
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen buffer_append(tmp_buf, uid_buffer->data, uid_buffer->used);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen log_append_buffer(ctx, tmp_buf, MAIL_TRANSACTION_KEYWORD_UPDATE);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainenlog_append_keyword_updates(struct mail_index_export_context *ctx)
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen const struct mail_index_transaction_keyword_update *updates;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen const char *const *keywords;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen tmp_buf = buffer_create_dynamic(pool_datastack_create(), 64);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen for (i = 0; i < count; i++) {
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen if (array_is_created(&updates[i].remove_seq) &&
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainenvoid mail_index_transaction_export(struct mail_index_transaction *t,
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen struct mail_transaction_log_append_ctx *append_ctx)
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen mail_transaction_log_append_add(ctx.append_ctx,
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen /* send all extension introductions and resizes before appends
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen to avoid resize overhead as much as possible */
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen log_append_buffer(&ctx, log_get_hdr_update_buffer(t, TRUE),
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen log_append_buffer(&ctx, t->appends.arr.buffer,
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen log_append_buffer(&ctx, t->updates.arr.buffer,
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen log_append_ext_recs(&ctx, &t->ext_rec_updates,
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen log_append_ext_recs(&ctx, &t->ext_rec_atomics,
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen /* keyword resets before updates */
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen change_mask |= log_append_keyword_updates(&ctx);
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen /* keep modseq updates almost last */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen log_append_buffer(&ctx, t->modseq_updates.arr.buffer,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen /* non-external expunges are only requests, ignore them when
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen checking fsync_mask */
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen log_append_buffer(&ctx, t->expunges.arr.buffer,
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
d1baa8c6f97cdb1b3c2c44a73cc21f9dfc7a963fTimo Sirainen mail_transaction_log_append_add(ctx.append_ctx,
77bc2bda5b781c4ffddc8a74b175cf32e9e2c2ecTimo Sirainen /* Update the tail offsets only when committing the sync transaction.
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen Other transactions may not know the latest tail offset and might
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen end up shrinking it. (Alternatively the shrinking tail offsets could
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen just be ignored, which would probably work fine too.) */
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen append_ctx->append_sync_offset = t->sync_transaction;
1279090ba03f9c176976a69ab7718f0ed77b19afTimo Sirainen (t->view->index->fsync_mask & change_mask) != 0 ||