mail-index-sync-ext.c revision 68b3667c9ee95951d7c3e03b19b2d37abbaa5736
183bea41fa640dc8117f3eb45ff935cd81377a84Timo Sirainen/* Copyright (c) 2004-2008 Dovecot authors, see the included COPYING file */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_sync_init_expunge_handlers(struct mail_index_sync_map_ctx *ctx)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!array_is_created(&ctx->view->map->extensions))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rext = array_get(&ctx->view->index->extensions, &rext_count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext = array_get(&ctx->view->map->extensions, &ext_count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen id_map = array_get(&ctx->view->map->ext_id_map, &id_map_count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen contexts = array_get_modifiable(&ctx->extra_contexts, &context_count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (idx_ext_id = 0; idx_ext_id < rext_count; idx_ext_id++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (rext[idx_ext_id].expunge_handler == NULL ||
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen !rext[idx_ext_id].expunge_handler_call_always))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen eh.handler = rext[idx_ext_id].expunge_handler;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen eh.context = rext[idx_ext_id].expunge_context;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen eh.record_offset = map_ext_id == (uint32_t)-1 ? 0 :
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_sync_deinit_expunge_handlers(struct mail_index_sync_map_ctx *ctx)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int i, count;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!array_is_created(&ctx->expunge_handlers))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen eh = array_get(&ctx->expunge_handlers, &count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_sync_init_handlers(struct mail_index_sync_map_ctx *ctx)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int count;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!array_is_created(&ctx->view->map->extensions))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* set space for extra contexts */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen count = array_count(&ctx->view->index->extensions);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* make sure the extra_contexts contains everything */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen (void)array_idx_modifiable(&ctx->extra_contexts, count - 1);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen /* we need to update the expunge handler list in case they had
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen already been called */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_sync_deinit_handlers(struct mail_index_sync_map_ctx *ctx)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rext = array_get(&ctx->view->index->extensions, &rext_count);
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen array_get_modifiable(&ctx->extra_contexts, &context_count);
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen for (i = 0; i < context_count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rext[i].sync_handler.callback(ctx, 0, NULL, NULL,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenget_ext_header(struct mail_index_map *map, const struct mail_index_ext *ext)
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen /* do some kludgy jumping to get to it. */
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen hdr_base = buffer_get_modifiable_data(map->hdr_copy_buf, NULL);
ef5fb27361cc5e15766e85e28355750ff04b13c9Timo Sirainen ext_hdr = PTR_OFFSET(hdr_base, ext->ext_offset);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_ext_align_cmp(const void *p1, const void *p2)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_ext *const *e1 = p1, *const *e2 = p2;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return (int)(*e2)->record_align - (int)(*e1)->record_align;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void sync_ext_reorder(struct mail_index_map *map, uint32_t ext_map_idx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint16_t *old_offsets, *copy_sizes, min_align, max_align;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int i, count;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const void *src;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) && map->refcount == 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext = array_get_modifiable(&map->extensions, &count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* @UNSAFE */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen sorted = t_new(struct mail_index_ext *, count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen qsort(sorted, count, sizeof(struct mail_index_ext *),
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we are growing the extension record. remember this
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen so we don't write extra data while copying the record */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we simply try to use the extensions with largest alignment
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen requirement first. FIXME: if the extension sizes don't match
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen alignmentation, this may not give the minimal layout. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* all done */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we have to leave space here */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(min_align > 1 && min_align < (uint16_t)-1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* keep record size divisible with maximum alignment */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* copy the records to new buffer */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen new_buffer_size = map->rec_map->records_count * new_record_size;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen new_buffer = buffer_create_dynamic(default_pool, new_buffer_size);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (rec_idx = 0; rec_idx < map->rec_map->records_count; rec_idx++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* write the base record */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen sizeof(struct mail_index_record));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* write extensions */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen buffer_write(new_buffer, offset + ext[i].record_offset,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen src = CONST_PTR_OFFSET(src, map->hdr.record_size);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we didn't fully write the last record */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen size_t space = new_buffer_size - new_buffer->used;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen buffer_get_modifiable_data(map->rec_map->buffer, NULL);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* update record offsets in headers */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext_hdr->record_offset = ext[i].record_offset;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainensync_ext_resize(const struct mail_transaction_ext_intro *u,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t ext_map_idx, struct mail_index_sync_map_ctx *ctx)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext = array_idx_modifiable(&map->extensions, ext_map_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen old_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen new_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* header shrank */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen buffer_delete(map->hdr_copy_buf, ext->hdr_offset + new_size,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* header grown */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert((map->hdr_copy_buf->used % sizeof(uint64_t)) == 0);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen map->hdr.header_size = map->hdr_copy_buf->used;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen map->write_base_header = map->write_ext_header = TRUE;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* move all hdr_offset of all extensions after this one */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int i, count = array_count(&map->extensions);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ssize_t diff = (ssize_t)new_size - (ssize_t)old_size;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext = array_idx_modifiable(&map->extensions, 0);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen sync_ext_reorder(map, ext_map_idx, old_record_size);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* header size changed. recreate index file. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_sync_ext_unknown_complain(struct mail_index_sync_map_ctx *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned char *p;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen buffer_create_dynamic(default_pool, ext_map_idx + 8);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen p = buffer_get_space_unsafe(ctx->unknown_extensions, ext_map_idx, 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (*p != 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we've already complained once */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenint mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* default to ignoring the following extension updates in case this
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen intro is corrupted */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen u->ext_id >= array_count(&map->extensions))) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_sync_ext_unknown_complain(ctx, u->ext_id))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Extension introduction for unknown id %u", u->ext_id);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (u->ext_id == (uint32_t)-1 && u->name_size == 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Extension introduction without id or name");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_map_lookup_ext(map, name, &ext_map_idx))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!ctx->internal_update && strcmp(name, "keywords") == 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* Keyword extension is handled internally by the keyword
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen code. Any attempt to modify them directly could cause
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen assert-crashes later, so prevent them immediately. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Extension introduction for keywords");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* exists already */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* check if we need to resize anything */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* extension was reset and this transaction hadn't
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen yet seen it. ignore this update. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* be sure to get a unique mapping before we modify the extensions,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen otherwise other map users will see the new extension but not the
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen data records that sync_ext_reorder() adds. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(hdr_buf->used == map->hdr.header_size);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) != hdr_buf->used) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we need to add padding between base header and extensions */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_map_ext_hdr_check(&map->hdr, &ext_hdr,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* register record offset initially using zero,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen sync_ext_reorder() will fix it. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext_map_idx = mail_index_map_register_ext(map, name, hdr_buf->used,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext = array_idx(&map->extensions, ext_map_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* <ext_hdr> <name> [padding] [header data] */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen buffer_append(hdr_buf, &ext_hdr, sizeof(ext_hdr));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* header must begin and end in correct alignment */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) - hdr_buf->used +
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext->hdr_offset + MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert((hdr_buf->used % sizeof(uint64_t)) == 0);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenint mail_index_sync_ext_reset(struct mail_index_sync_map_ctx *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Extension reset without intro prefix");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* a new index file will be created, so the old data won't be
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen accidentally used by other processes. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext = array_idx_modifiable(&map->extensions, ctx->cur_ext_map_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memset(buffer_get_space_unsafe(map->hdr_copy_buf, ext->hdr_offset,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < view->map->rec_map->records_count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memset(PTR_OFFSET(rec, ext->record_offset), 0,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen map->rec_map->write_seq_last = view->map->rec_map->records_count;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_sync_ext_hdr_update(struct mail_index_sync_map_ctx *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_transaction_ext_hdr_update *u)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Extension header update without intro prefix");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext = array_idx(&map->extensions, ctx->cur_ext_map_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen buffer_write(map->hdr_copy_buf, ext->hdr_offset + u->offset,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_transaction_ext_rec_update *u)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(ctx->cur_ext_map_idx != (uint32_t)-1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (u->uid == 0 || u->uid >= view->map->hdr.next_uid) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Extension record update for invalid uid=%u", u->uid);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_lookup_seq(view, u->uid, &seq))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen old_data = PTR_OFFSET(rec, ext->record_offset);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rext = array_idx(&view->index->extensions, ext->index_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* call sync handlers only when its registered type matches with
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen current synchronization type (index/view) */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if ((rext->sync_handler.type & ctx->type) != 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = rext->sync_handler.callback(ctx, seq, old_data, u + 1,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_sync_write_seq_update(ctx, seq, seq);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* @UNSAFE */