13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg#include "lib.h"
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg#include "array.h"
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg#include "buffer.h"
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg#include "mail-index-view-private.h"
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg#include "mail-index-sync-private.h"
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg#include "mail-index-modseq.h"
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg#include "mail-transaction-log.h"
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgvoid mail_index_sync_init_expunge_handlers(struct mail_index_sync_map_ctx *ctx)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg{
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg const struct mail_index_ext *ext;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg const struct mail_index_registered_ext *rext;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg const uint32_t *id_map;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg void **contexts;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg struct mail_index_expunge_handler eh;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg unsigned int ext_count, id_map_count;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg unsigned int rext_count, context_count;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg uint32_t idx_ext_id, map_ext_id;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (!array_is_created(&ctx->view->map->extensions))
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg return;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_zero(&eh);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (array_is_created(&ctx->expunge_handlers))
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg array_clear(&ctx->expunge_handlers);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg else
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_array_init(&ctx->expunge_handlers, 64);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg rext = array_get(&ctx->view->index->extensions, &rext_count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ext = array_get(&ctx->view->map->extensions, &ext_count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg id_map = array_get(&ctx->view->map->ext_id_map, &id_map_count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg contexts = array_get_modifiable(&ctx->extra_contexts, &context_count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(context_count >= rext_count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg for (idx_ext_id = 0; idx_ext_id < rext_count; idx_ext_id++) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg map_ext_id = idx_ext_id >= id_map_count ? (uint32_t)-1 :
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg id_map[idx_ext_id];
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (rext[idx_ext_id].expunge_handler == NULL ||
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg (map_ext_id == (uint32_t)-1 &&
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg !rext[idx_ext_id].expunge_handler_call_always))
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg continue;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg eh.handler = rext[idx_ext_id].expunge_handler;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg eh.context = rext[idx_ext_id].expunge_context;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg eh.sync_context = &contexts[idx_ext_id];
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg eh.record_offset = map_ext_id == (uint32_t)-1 ? 0 :
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ext[map_ext_id].record_offset;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg array_append(&ctx->expunge_handlers, &eh, 1);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ctx->expunge_handlers_set = TRUE;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ctx->expunge_handlers_used = TRUE;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg}
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgvoid
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgmail_index_sync_deinit_expunge_handlers(struct mail_index_sync_map_ctx *ctx)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg{
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg const struct mail_index_expunge_handler *eh;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (!array_is_created(&ctx->expunge_handlers))
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg return;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg array_foreach(&ctx->expunge_handlers, eh) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (eh->sync_context != NULL) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg eh->handler(ctx, 0, NULL, eh->sync_context,
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg eh->context);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg array_free(&ctx->expunge_handlers);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg}
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgvoid mail_index_sync_init_handlers(struct mail_index_sync_map_ctx *ctx)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg{
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg unsigned int count;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (!array_is_created(&ctx->view->map->extensions))
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg return;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* set space for extra contexts */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg count = array_count(&ctx->view->index->extensions);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(count > 0);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (!array_is_created(&ctx->extra_contexts))
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_array_init(&ctx->extra_contexts, count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* make sure the extra_contexts contains everything */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg (void)array_idx_get_space(&ctx->extra_contexts, count - 1);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* we need to update the expunge handler list in case they had
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg already been called */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ctx->expunge_handlers_set = FALSE;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg}
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgvoid mail_index_sync_deinit_handlers(struct mail_index_sync_map_ctx *ctx)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg{
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg const struct mail_index_registered_ext *rext;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg void **extra_contexts;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg unsigned int i, rext_count, context_count;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (!array_is_created(&ctx->extra_contexts))
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg return;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg rext = array_get(&ctx->view->index->extensions, &rext_count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg extra_contexts =
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg array_get_modifiable(&ctx->extra_contexts, &context_count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(context_count <= rext_count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg for (i = 0; i < context_count; i++) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (extra_contexts[i] != NULL) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg rext[i].sync_handler.callback(ctx, 0, NULL, NULL,
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg &extra_contexts[i]);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg array_free(&ctx->extra_contexts);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg}
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgstatic struct mail_index_ext_header *
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgget_ext_header(struct mail_index_map *map, const struct mail_index_ext *ext)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg{
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg struct mail_index_ext_header *ext_hdr;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg void *hdr_base;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* do some kludgy jumping to get to it. */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg hdr_base = buffer_get_modifiable_data(map->hdr_copy_buf, NULL);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ext_hdr = PTR_OFFSET(hdr_base, ext->ext_offset);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(memcmp((char *)(ext_hdr + 1),
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ext->name, strlen(ext->name)) == 0);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg return ext_hdr;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg}
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgstatic int mail_index_ext_align_cmp(const void *p1, const void *p2)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg{
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg const struct mail_index_ext *const *e1 = p1, *const *e2 = p2;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg return (int)(*e2)->record_align - (int)(*e1)->record_align;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg}
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgstatic void sync_ext_reorder(struct mail_index_map *map, uint32_t ext_map_idx,
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg uint16_t old_ext_size)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg{
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg struct mail_index_ext *ext, **sorted;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg struct mail_index_ext_header *ext_hdr;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg uint16_t *old_offsets, *copy_sizes, min_align, max_align;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg uint32_t offset, new_record_size, rec_idx;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg unsigned int i, count;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg const void *src;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg buffer_t *new_buffer;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg size_t new_buffer_size;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) && map->refcount == 1);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ext = array_get_modifiable(&map->extensions, &count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(ext_map_idx < count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* @UNSAFE */
fd7c287b8fcccc3e1e7834bf8daf8fe7fd6cd49dfuankg old_offsets = t_new(uint16_t, count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg copy_sizes = t_new(uint16_t, count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg sorted = t_new(struct mail_index_ext *, count);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg for (i = 0; i < count; i++) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg old_offsets[i] = ext[i].record_offset;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg copy_sizes[i] = ext[i].record_size;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ext[i].record_offset = 0;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg sorted[i] = &ext[i];
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg qsort(sorted, count, sizeof(struct mail_index_ext *),
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg mail_index_ext_align_cmp);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (copy_sizes[ext_map_idx] > old_ext_size) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* we are growing the extension record. remember this
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg so we don't write extra data while copying the record */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg copy_sizes[ext_map_idx] = old_ext_size;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* we simply try to use the extensions with largest alignment
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg requirement first. FIXME: if the extension sizes don't match
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg alignment, this may not give the minimal layout. */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg offset = MAIL_INDEX_RECORD_MIN_SIZE;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg max_align = sizeof(uint32_t);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg for (;;) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg min_align = (uint16_t)-1;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg for (i = 0; i < count; i++) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (sorted[i]->record_align > max_align)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg max_align = sorted[i]->record_align;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (sorted[i]->record_offset == 0 &&
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg sorted[i]->record_size > 0) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if ((offset % sorted[i]->record_align) == 0)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg break;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (sorted[i]->record_align < min_align)
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg min_align = sorted[i]->record_align;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (i == count) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (min_align == (uint16_t)-1) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* all done */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg break;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* we have to leave space here */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(min_align > 1 && min_align < (uint16_t)-1);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg offset += min_align - (offset % min_align);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg } else {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg sorted[i]->record_offset = offset;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg offset += sorted[i]->record_size;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(offset < (uint16_t)-1);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if ((offset % max_align) != 0) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* keep record size divisible with maximum alignment */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg offset += max_align - (offset % max_align);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg new_record_size = offset;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(new_record_size >= sizeof(struct mail_index_record));
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* copy the records to new buffer */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg new_buffer_size = map->rec_map->records_count * new_record_size;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg new_buffer = buffer_create_dynamic(default_pool, new_buffer_size);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg src = map->rec_map->records;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg offset = 0;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg for (rec_idx = 0; rec_idx < map->rec_map->records_count; rec_idx++) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* write the base record */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg buffer_write(new_buffer, offset, src,
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg sizeof(struct mail_index_record));
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* write extensions */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg for (i = 0; i < count; i++) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg buffer_write(new_buffer, offset + ext[i].record_offset,
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg CONST_PTR_OFFSET(src, old_offsets[i]),
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg copy_sizes[i]);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg src = CONST_PTR_OFFSET(src, map->hdr.record_size);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg offset += new_record_size;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg if (new_buffer->used != new_buffer_size) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* we didn't fully write the last record */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg size_t space = new_buffer_size - new_buffer->used;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg i_assert(space < new_record_size);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg buffer_append_zero(new_buffer, space);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg buffer_free(&map->rec_map->buffer);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg map->rec_map->buffer = new_buffer;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg map->rec_map->records =
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg buffer_get_modifiable_data(map->rec_map->buffer, NULL);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg map->hdr.record_size = new_record_size;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg /* update record offsets in headers */
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg for (i = 0; i < count; i++) {
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ext_hdr = get_ext_header(map, &ext[i]);
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg ext_hdr->record_offset = ext[i].record_offset;
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg }
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg}
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgstatic void
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankgsync_ext_resize(const struct mail_transaction_ext_intro *u,
13b501825bce68e7e49b4bc775da93e38d9bd9f3fuankg uint32_t ext_map_idx, struct mail_index_sync_map_ctx *ctx,
bool no_shrink)
{
struct mail_index_map *map;
struct mail_index_ext *ext;
struct mail_index_ext_header *ext_hdr;
uint32_t old_padded_hdr_size, new_padded_hdr_size, old_record_size;
bool reorder = FALSE;
ext = array_idx_modifiable(&ctx->view->map->extensions, ext_map_idx);
old_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size);
new_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size);
if (ext->record_align != u->record_align ||
ext->record_size != u->record_size) {
/* record changed */
} else if (new_padded_hdr_size < old_padded_hdr_size) {
/* header is shrunk. do we allow? */
if (no_shrink)
return;
} else if (ext->hdr_size == u->hdr_size) {
/* no changes */
return;
}
/* something changed. get ourself a new map before we start changing
anything in it. */
map = mail_index_sync_get_atomic_map(ctx);
/* ext was duplicated to the new map. */
ext = array_idx_modifiable(&map->extensions, ext_map_idx);
if (new_padded_hdr_size < old_padded_hdr_size) {
/* header shrank */
if (no_shrink)
new_padded_hdr_size = old_padded_hdr_size;
else {
buffer_delete(map->hdr_copy_buf,
ext->hdr_offset + new_padded_hdr_size,
old_padded_hdr_size - new_padded_hdr_size);
ext->hdr_size = u->hdr_size;
}
} else if (new_padded_hdr_size > old_padded_hdr_size) {
/* header grown */
buffer_insert_zero(map->hdr_copy_buf,
ext->hdr_offset + old_padded_hdr_size,
new_padded_hdr_size - old_padded_hdr_size);
ext->hdr_size = u->hdr_size;
} else {
if (ext->hdr_size != u->hdr_size) {
/* aligned sizes were the same, but the actual sizes
had changed */
ext->hdr_size = u->hdr_size;
}
}
if (ext->record_align < u->record_align ||
(ext->record_align > u->record_align && !no_shrink)) {
ext->record_align = u->record_align;
reorder = TRUE;
}
old_record_size = ext->record_size;
if (ext->record_size < u->record_size ||
(ext->record_size > u->record_size && !no_shrink)) {
ext->record_size = u->record_size;
reorder = TRUE;
}
i_assert((map->hdr_copy_buf->used % sizeof(uint64_t)) == 0);
map->hdr_base = map->hdr_copy_buf->data;
map->hdr.header_size = map->hdr_copy_buf->used;
ext_hdr = get_ext_header(map, ext);
ext_hdr->reset_id = ext->reset_id;
ext_hdr->hdr_size = ext->hdr_size;
ext_hdr->record_offset = ext->record_offset;
ext_hdr->record_size = ext->record_size;
ext_hdr->record_align = ext->record_align;
if (new_padded_hdr_size != old_padded_hdr_size) {
/* move all hdr_offset of all extensions after this one */
unsigned int i, count = array_count(&map->extensions);
ssize_t diff = (ssize_t)new_padded_hdr_size -
(ssize_t)old_padded_hdr_size;
ext = array_idx_modifiable(&map->extensions, 0);
for (i = ext_map_idx + 1; i < count; i++) {
ext[i].ext_offset += diff;
ext[i].hdr_offset += diff;
}
}
if (reorder)
sync_ext_reorder(map, ext_map_idx, old_record_size);
}
static bool
mail_index_sync_ext_unknown_complain(struct mail_index_sync_map_ctx *ctx,
uint32_t ext_map_idx)
{
unsigned char *p;
if (ext_map_idx >= 1024) {
/* don't try to track too high values */
return TRUE;
}
if (ctx->unknown_extensions == NULL) {
ctx->unknown_extensions =
buffer_create_dynamic(default_pool, ext_map_idx + 8);
}
p = buffer_get_space_unsafe(ctx->unknown_extensions, ext_map_idx, 1);
if (*p != 0) {
/* we've already complained once */
return FALSE;
}
*p = 1;
return TRUE;
}
static void
mail_index_sync_ext_init_new(struct mail_index_sync_map_ctx *ctx,
const char *name,
const struct mail_index_ext_header *ext_hdr,
uint32_t *ext_map_idx_r)
{
struct mail_index_map *map;
const struct mail_index_ext *ext;
buffer_t *hdr_buf;
uint32_t ext_map_idx;
/* be sure to get a unique mapping before we modify the extensions,
otherwise other map users will see the new extension but not the
data records that sync_ext_reorder() adds. */
map = mail_index_sync_get_atomic_map(ctx);
hdr_buf = map->hdr_copy_buf;
i_assert(hdr_buf->used == map->hdr.header_size);
if (MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) != hdr_buf->used) {
/* we need to add padding between base header and extensions */
buffer_append_zero(hdr_buf,
MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) -
hdr_buf->used);
}
/* register record offset initially using zero,
sync_ext_reorder() will fix it. */
ext_map_idx = mail_index_map_register_ext(map, name, hdr_buf->used,
ext_hdr);
ext = array_idx(&map->extensions, ext_map_idx);
/* <ext_hdr> <name> [padding] [header data] */
i_assert(ext_hdr->name_size == strlen(name));
buffer_append(hdr_buf, ext_hdr, sizeof(*ext_hdr));
buffer_append(hdr_buf, name, ext_hdr->name_size);
/* header must begin and end in correct alignment */
buffer_append_zero(hdr_buf,
MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) - hdr_buf->used +
MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size));
i_assert(hdr_buf->used ==
ext->hdr_offset + MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size));
i_assert((hdr_buf->used % sizeof(uint64_t)) == 0);
map->hdr.header_size = hdr_buf->used;
map->hdr_base = hdr_buf->data;
mail_index_sync_init_handlers(ctx);
sync_ext_reorder(map, ext_map_idx, 0);
i_assert(ext->record_offset != 0 || ext->record_size == 0);
*ext_map_idx_r = ext_map_idx;
}
void mail_index_sync_ext_init(struct mail_index_sync_map_ctx *ctx,
const char *name, bool fix_size,
uint32_t *ext_map_idx_r)
{
struct mail_index_map *map = ctx->view->map;
const struct mail_index_registered_ext *rext;
struct mail_index_ext_header ext_hdr;
struct mail_transaction_ext_intro u;
uint32_t ext_id;
if (!mail_index_ext_lookup(ctx->view->index, name, &ext_id))
i_unreached();
rext = array_idx(&ctx->view->index->extensions, ext_id);
if (mail_index_map_lookup_ext(map, name, ext_map_idx_r)) {
if (!fix_size)
return;
/* make sure it's the expected size */
i_zero(&u);
u.hdr_size = rext->hdr_size;
u.record_size = rext->record_size;
u.record_align = rext->record_align;
sync_ext_resize(&u, *ext_map_idx_r, ctx, FALSE);
} else {
i_zero(&ext_hdr);
ext_hdr.name_size = strlen(name);
ext_hdr.hdr_size = rext->hdr_size;
ext_hdr.record_size = rext->record_size;
ext_hdr.record_align = rext->record_align;
mail_index_sync_ext_init_new(ctx, name, &ext_hdr,
ext_map_idx_r);
}
}
int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_ext_intro *u)
{
struct mail_index_map *map = ctx->view->map;
struct mail_index_ext_header ext_hdr;
const struct mail_index_ext *ext;
const char *name, *error;
uint32_t ext_map_idx;
bool no_shrink;
/* default to ignoring the following extension updates in case this
intro is corrupted */
ctx->cur_ext_map_idx = (uint32_t)-2;
ctx->cur_ext_ignore = TRUE;
ctx->cur_ext_record_size = 0;
if (u->ext_id != (uint32_t)-1 &&
(!array_is_created(&map->extensions) ||
u->ext_id >= array_count(&map->extensions))) {
if (!mail_index_sync_ext_unknown_complain(ctx, u->ext_id))
return -1;
mail_index_sync_set_corrupted(ctx,
"Extension introduction for unknown id %u", u->ext_id);
return -1;
}
if (u->ext_id == (uint32_t)-1 && u->name_size == 0) {
mail_index_sync_set_corrupted(ctx,
"Extension introduction without id or name");
return -1;
}
if (u->ext_id != (uint32_t)-1) {
name = NULL;
ext_map_idx = u->ext_id;
} else {
name = t_strndup(u + 1, u->name_size);
if (!mail_index_map_lookup_ext(map, name, &ext_map_idx))
ext_map_idx = (uint32_t)-1;
}
if (ext_map_idx == (uint32_t)-1)
ext = NULL;
else {
ext = array_idx(&map->extensions, ext_map_idx);
name = ext->name;
}
i_assert(name != NULL);
if (!ctx->internal_update &&
strcmp(name, MAIL_INDEX_EXT_KEYWORDS) == 0) {
/* Keyword extension is handled internally by the keyword
code. Any attempt to modify them directly could cause
assert-crashes later, so prevent them immediately. */
mail_index_sync_set_corrupted(ctx,
"Extension introduction for keywords");
return -1;
}
i_zero(&ext_hdr);
ext_hdr.name_size = strlen(name);
ext_hdr.reset_id = u->reset_id;
ext_hdr.hdr_size = u->hdr_size;
ext_hdr.record_size = u->record_size;
ext_hdr.record_align = u->record_align;
no_shrink = (u->flags & MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK) != 0;
/* make sure the header looks valid before doing anything with it */
if (mail_index_map_ext_hdr_check(&map->hdr, &ext_hdr,
name, &error) < 0) {
mail_index_sync_set_corrupted(ctx,
"Broken extension introduction: %s", error);
return -1;
}
ctx->cur_ext_record_size = u->record_size;
if (ext != NULL) {
/* exists already */
if (u->reset_id == ext->reset_id) {
/* check if we need to resize anything */
sync_ext_resize(u, ext_map_idx, ctx, no_shrink);
ctx->cur_ext_ignore = FALSE;
} else {
/* extension was reset and this transaction hadn't
yet seen it. ignore this update (except for
resets). */
ctx->cur_ext_ignore = TRUE;
}
ctx->cur_ext_map_idx = ext_map_idx;
return 1;
}
mail_index_sync_ext_init_new(ctx, name, &ext_hdr, &ext_map_idx);
ctx->cur_ext_ignore = FALSE;
ctx->cur_ext_map_idx = ctx->internal_update ?
(uint32_t)-1 : ext_map_idx;
return 1;
}
static void mail_index_sync_ext_clear(struct mail_index_view *view,
struct mail_index_map *map,
struct mail_index_ext *ext)
{
struct mail_index_record *rec;
uint32_t seq;
memset(buffer_get_space_unsafe(map->hdr_copy_buf, ext->hdr_offset,
ext->hdr_size), 0, ext->hdr_size);
map->hdr_base = map->hdr_copy_buf->data;
i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
for (seq = 1; seq <= view->map->rec_map->records_count; seq++) {
rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
memset(PTR_OFFSET(rec, ext->record_offset), 0,
ext->record_size);
}
}
int mail_index_sync_ext_reset(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_ext_reset *u)
{
struct mail_index_map *map;
struct mail_index_ext_header *ext_hdr;
struct mail_index_ext *ext;
if (ctx->cur_ext_map_idx == (uint32_t)-1) {
mail_index_sync_set_corrupted(ctx,
"Extension reset without intro prefix");
return -1;
}
if (ctx->cur_ext_map_idx == (uint32_t)-2 && ctx->cur_ext_ignore) {
/* previous extension intro was broken */
return -1;
}
/* since we're resetting the extension, don't check cur_ext_ignore */
/* a new index file will be created, so the old data won't be
accidentally used by other processes. */
map = mail_index_sync_get_atomic_map(ctx);
ext = array_idx_modifiable(&map->extensions, ctx->cur_ext_map_idx);
ext->reset_id = u->new_reset_id;
if (u->preserve_data == 0)
mail_index_sync_ext_clear(ctx->view, map, ext);
ext_hdr = get_ext_header(map, ext);
ext_hdr->reset_id = u->new_reset_id;
return 1;
}
int mail_index_sync_ext_hdr_update(struct mail_index_sync_map_ctx *ctx,
uint32_t offset, uint32_t size,
const void *data)
{
struct mail_index_map *map = ctx->view->map;
const struct mail_index_ext *ext;
if (ctx->cur_ext_map_idx == (uint32_t)-1) {
mail_index_sync_set_corrupted(ctx,
"Extension header update without intro prefix");
return -1;
}
if (ctx->cur_ext_ignore)
return 1;
ext = array_idx(&map->extensions, ctx->cur_ext_map_idx);
if (offset + size > ext->hdr_size) {
#if 1
mail_index_sync_set_corrupted(ctx,
"Extension header update points outside header size");
return -1;
#else
size = offset > ext->hdr_size ? 0 :
ext->hdr_size - offset;
#endif
}
buffer_write(map->hdr_copy_buf, ext->hdr_offset + offset, data, size);
map->hdr_base = map->hdr_copy_buf->data;
i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
if (ext->index_idx == ctx->view->index->modseq_ext_id)
mail_index_modseq_hdr_update(ctx->modseq_ctx);
return 1;
}
int
mail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_ext_rec_update *u)
{
struct mail_index_view *view = ctx->view;
struct mail_index_record *rec;
const struct mail_index_ext *ext;
const struct mail_index_registered_ext *rext;
void *old_data;
uint32_t seq;
int ret;
i_assert(ctx->cur_ext_map_idx != (uint32_t)-1);
i_assert(!ctx->cur_ext_ignore);
if (u->uid == 0 || u->uid >= view->map->hdr.next_uid) {
mail_index_sync_set_corrupted(ctx,
"Extension record update for invalid uid=%u", u->uid);
return -1;
}
if (!mail_index_lookup_seq(view, u->uid, &seq))
return 1;
ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx);
i_assert(ext->record_offset + ctx->cur_ext_record_size <=
view->map->hdr.record_size);
rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
old_data = PTR_OFFSET(rec, ext->record_offset);
rext = array_idx(&view->index->extensions, ext->index_idx);
/* call sync handlers only when its registered type matches with
current synchronization type (index/view) */
if ((rext->sync_handler.type & ctx->type) != 0) {
void **extra_context =
array_idx_modifiable(&ctx->extra_contexts,
ext->index_idx);
ret = rext->sync_handler.callback(ctx, seq, old_data, u + 1,
extra_context);
if (ret <= 0)
return ret;
}
/* @UNSAFE */
memcpy(old_data, u + 1, ctx->cur_ext_record_size);
if (ctx->cur_ext_record_size < ext->record_size) {
memset(PTR_OFFSET(old_data, ctx->cur_ext_record_size), 0,
ext->record_size - ctx->cur_ext_record_size);
}
return 1;
}
int
mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_ext_atomic_inc *u)
{
struct mail_index_view *view = ctx->view;
struct mail_index_record *rec;
const struct mail_index_ext *ext;
void *data;
uint32_t seq;
uint64_t min_value, max_value, orig_num;
i_assert(ctx->cur_ext_map_idx != (uint32_t)-1);
i_assert(!ctx->cur_ext_ignore);
if (u->uid == 0 || u->uid >= view->map->hdr.next_uid) {
mail_index_sync_set_corrupted(ctx,
"Extension record inc for invalid uid=%u", u->uid);
return -1;
}
if (!mail_index_lookup_seq(view, u->uid, &seq))
return 1;
ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx);
i_assert(ext->record_offset + ctx->cur_ext_record_size <=
view->map->hdr.record_size);
rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
data = PTR_OFFSET(rec, ext->record_offset);
min_value = u->diff >= 0 ? 0 : (uint64_t)(-(int64_t)u->diff);
max_value = ctx->cur_ext_record_size == 8 ? (uint64_t)-1 :
((uint64_t)1 << (ctx->cur_ext_record_size*8)) - 1;
if (u->diff <= 0) {
/* skip */
} else if (max_value >= (uint32_t)u->diff) {
max_value -= u->diff;
} else {
mail_index_sync_set_corrupted(ctx,
"Extension record inc diff=%d larger than max value=%u "
"(uid=%u)", u->diff, (unsigned int)max_value, u->uid);
return -1;
}
switch (ctx->cur_ext_record_size) {
case 1: {
uint8_t *num = data;
orig_num = *num;
if (orig_num >= min_value && orig_num <= max_value)
*num += u->diff;
break;
}
case 2: {
uint16_t *num = data;
orig_num = *num;
if (orig_num >= min_value && orig_num <= max_value)
*num += u->diff;
break;
}
case 4: {
uint32_t *num = data;
orig_num = *num;
if (orig_num >= min_value && orig_num <= max_value)
*num += u->diff;
break;
}
case 8: {
uint64_t *num = data;
orig_num = *num;
if (orig_num >= min_value && orig_num <= max_value)
*num += u->diff;
break;
}
default:
mail_index_sync_set_corrupted(ctx,
"Extension record inc with invalid size=%u",
ctx->cur_ext_record_size);
return -1;
}
if (orig_num < min_value) {
mail_index_sync_set_corrupted(ctx,
"Extension record inc drops number below zero "
"(uid=%u, diff=%d, orig=%"PRIu64")",
u->uid, u->diff, orig_num);
return -1;
} else if (orig_num > max_value) {
mail_index_sync_set_corrupted(ctx,
"Extension record inc overflows number "
"(uid=%u, diff=%d, orig=%"PRIu64")",
u->uid, u->diff, orig_num);
return -1;
}
return 1;
}