mail-index-sync-update.c revision 3342badd8c69adff34db589fb0a221ace5996212
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher/* Copyright (C) 2004 Timo Sirainen */
b355dcb54194f498921743ca33304eac35d89718Stephen Gallagher/* If we have less than this many bytes to sync from log file, don't bother
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher reading the main index */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (prev_offset == ctx->ext_intro_end_offset &&
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* previous transaction was an extension introduction.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher we probably came here from
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_ext_reset(). if there are any more
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher views which want to continue syncing it needs the
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher intro. so back up a bit more.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher don't do this in case the last transaction in the
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher log is the extension intro, so we don't keep trying
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher to sync it over and over again. */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_head_offset = prev_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_update_log_offset(ctx, view->map, FALSE);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_map *map = ctx->view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) ||
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_map_move_to_memory(ctx->view->map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_move_to_private_memory(ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_record_map_move_to_private(ctx->view->map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_header_update_counts(struct mail_index_header *hdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* different seen-flag */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (hdr->seen_messages_count >= hdr->messages_count) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (++hdr->seen_messages_count == hdr->messages_count)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher hdr->first_unseen_uid_lowwater = hdr->next_uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* different deleted-flag */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (hdr->deleted_messages_count > hdr->messages_count) {
056302a92862fda16351d7192600746746f38e5dStephen Gallagher hdr->deleted_messages_count > hdr->messages_count) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher hdr->first_deleted_uid_lowwater = hdr->next_uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i, count;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher maps = array_get(&ctx->view->map->rec_map->maps, &count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < count; i++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (mail_index_header_update_counts(&maps[i]->hdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx, "%s", error);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_header_update_counts_all(ctx, uid, old_flags,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(uid < ctx->view->map->hdr.next_uid);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (mail_index_header_update_counts(&ctx->view->map->hdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx, "%s", error);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i, count;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher maps = array_get(&ctx->view->map->rec_map->maps, &count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < count; i++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uid < maps[i]->hdr.first_unseen_uid_lowwater)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher maps[i]->hdr.first_unseen_uid_lowwater = uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uid < maps[i]->hdr.first_deleted_uid_lowwater)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher maps[i]->hdr.first_deleted_uid_lowwater = uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghersync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_index_expunge_handler *eh;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i, count;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* call expunge handlers only when syncing index file */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_init_expunge_handlers(ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (!array_is_created(&ctx->expunge_handlers))
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher eh = array_get(&ctx->expunge_handlers, &count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher rec = MAIL_INDEX_MAP_IDX(ctx->view->map, seq1-1);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghersync_expunge(const struct mail_transaction_expunge *e, unsigned int count,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_map *map = ctx->view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < count; i++, e++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (mail_index_lookup_uid_range(ctx->view, e->uid1, e->uid2,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* everything expunged already */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map = mail_index_sync_get_atomic_map(ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_header_update_counts(ctx, rec->uid,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (sync_expunge_call_handlers(ctx, seq1, seq2) < 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghervoid mail_index_sync_write_seq_update(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_map *map = ctx->view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic int sync_append(const struct mail_index_record *rec,
056302a92862fda16351d7192600746746f38e5dStephen Gallagher "Append with UID %u, but next_uid = %u",
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* move to memory. the mapping is written when unlocking so we don't
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher waste time re-mmap()ing multiple times or waste space growing index
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher file too large */
056302a92862fda16351d7192600746746f38e5dStephen Gallagher mail_index_sync_move_to_private_memory(ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (rec->uid <= map->rec_map->last_appended_uid) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(map->hdr.messages_count < map->rec_map->records_count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* the flags may have changed since it was added to map.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher use the updated flags already, so flag counters won't get
056302a92862fda16351d7192600746746f38e5dStephen Gallagher MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count)->flags;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* don't rely on buffer->used being at the correct position.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher at least expunges can move it */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher append_pos = map->rec_map->records_count * map->hdr.record_size;
056302a92862fda16351d7192600746746f38e5dStephen Gallagher dest = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher buffer_get_modifiable_data(map->rec_map->buffer, NULL);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->rec_map->last_appended_uid = rec->uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_write_seq_update(ctx, map->hdr.messages_count,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
056302a92862fda16351d7192600746746f38e5dStephen Gallagher mail_index_sync_header_update_counts(ctx, rec->uid,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic int sync_flag_update(const struct mail_transaction_flag_update *u,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (mail_index_lookup_uid_range(view, u->uid1, u->uid2,
056302a92862fda16351d7192600746746f38e5dStephen Gallagher mail_index_sync_write_seq_update(ctx, seq1, seq2);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* we're not modifying any counted/lowwatered flags */
056302a92862fda16351d7192600746746f38e5dStephen Gallagher rec->flags = (rec->flags & flag_mask) | u->add_flags;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher rec->flags = (rec->flags & flag_mask) | u->add_flags;
056302a92862fda16351d7192600746746f38e5dStephen Gallagher mail_index_header_update_lowwaters(ctx, rec->uid,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_header_update_counts(ctx, rec->uid,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic int sync_header_update(const struct mail_transaction_header_update *u,
056302a92862fda16351d7192600746746f38e5dStephen Gallagher struct mail_index_map *map = ctx->view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (u->offset >= map->hdr.base_header_size ||
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher u->offset + u->size > map->hdr.base_header_size) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "Header update outside range: %u + %u > %u",
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher u->offset, u->size, map->hdr.base_header_size);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
056302a92862fda16351d7192600746746f38e5dStephen Gallagher } else if (u->offset < sizeof(map->hdr)) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* the tail offset updates are intended for internal transaction
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher log handling. we'll update the offset in the header only when
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher the sync is finished. */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherint mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_header *hdr,
056302a92862fda16351d7192600746746f38e5dStephen Gallagher switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_index_record *rec, *end;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
056302a92862fda16351d7192600746746f38e5dStephen Gallagher const struct mail_transaction_expunge *rec = data, *end;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* this is simply a request for expunge */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_flag_update *rec, *end;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_header_update *rec;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((i % 4) != 0)
056302a92862fda16351d7192600746746f38e5dStephen Gallagher const struct mail_transaction_ext_intro *rec = data;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i;
056302a92862fda16351d7192600746746f38e5dStephen Gallagher mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* should be just extra padding */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (i + sizeof(*rec) + rec->name_size > hdr->size) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "ext intro: name_size too large");
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = mail_index_sync_ext_intro(ctx, rec);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((i % 4) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_ext_reset *rec = data;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "ext reset: invalid record size");
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = mail_index_sync_ext_reset(ctx, rec);
056302a92862fda16351d7192600746746f38e5dStephen Gallagher const struct mail_transaction_ext_hdr_update *rec = data;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i;
b355dcb54194f498921743ca33304eac35d89718Stephen Gallagher i + sizeof(*rec) + rec->size > hdr->size) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "ext hdr update: invalid record size");
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = mail_index_sync_ext_hdr_update(ctx, rec);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((i % 4) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_ext_rec_update *rec;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i, record_size;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "Extension record updated "
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "without intro prefix");
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ext = array_idx(&ctx->view->map->extensions, ctx->cur_ext_id);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* the record is padded to 32bits in the transaction log */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher record_size = (sizeof(*rec) + ext->record_size + 3) & ~3;
056302a92862fda16351d7192600746746f38e5dStephen Gallagher for (i = 0; i < hdr->size; i += record_size) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "ext rec update: invalid record size");
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = mail_index_sync_ext_rec_update(ctx, rec);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_keyword_update *rec = data;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = mail_index_sync_keywords(ctx, hdr, rec);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_keyword_reset *rec = data;
056302a92862fda16351d7192600746746f38e5dStephen Gallagher ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghervoid mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* make sure we re-read it in case it has changed */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher sync_map_ctx->view->map->keywords_read = FALSE;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_init_handlers(sync_map_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghervoid mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_deinit_handlers(sync_map_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* do we have dirty flags anymore? */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < map->rec_map->records_count; i++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
056302a92862fda16351d7192600746746f38e5dStephen Gallagher map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghervoid mail_index_map_check(struct mail_index_map *map)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_index_header *hdr = &map->hdr;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(hdr->messages_count <= map->rec_map->records_count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < hdr->messages_count; i++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
056302a92862fda16351d7192600746746f38e5dStephen Gallagher i_assert(del == hdr->deleted_messages_count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(seen == hdr->seen_messages_count);
056302a92862fda16351d7192600746746f38e5dStephen Gallagherint mail_index_sync_map(struct mail_index_map **_map,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher enum mail_index_sync_handler_type type, bool force)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_sync_map_ctx sync_map_ctx;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_header *thdr;
056302a92862fda16351d7192600746746f38e5dStephen Gallagher i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* see if we'd prefer to reopen the index file instead of
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher syncing the current map from the transaction log */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher index->log->head->hdr.prev_file_seq != 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* we don't know the index's size, so use the
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher smallest index size we're willing to read */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* this isn't necessary correct currently, but it should be
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher close enough */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (log_size > map->hdr.log_file_tail_offset &&
056302a92862fda16351d7192600746746f38e5dStephen Gallagher log_size - map->hdr.log_file_tail_offset > index_size)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
056302a92862fda16351d7192600746746f38e5dStephen Gallagher map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher view = mail_index_view_open_with_map(index, map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = mail_transaction_log_view_set(view->log_view,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* the seq/offset is probably broken */
056302a92862fda16351d7192600746746f38e5dStephen Gallagher /* can't use it. sync by re-reading index. */
056302a92862fda16351d7192600746746f38e5dStephen Gallagher /* view referenced the map. avoid unnecessary map cloning by
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unreferencing the map while view exists. */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
056302a92862fda16351d7192600746746f38e5dStephen Gallagher if (map->hdr_base != map->hdr_copy_buf->data) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* if syncing updates the header, it updates hdr_copy_buf
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher and updates hdr_base to hdr_copy_buf. so the buffer must
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher initially contain a valid header or we'll break it when
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher writing it. */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher buffer_append(map->hdr_copy_buf, map->hdr_base,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_map_init(&sync_map_ctx, view, type);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* Reset the entire index. Leave only indexid and
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher log_file_seq. */
056302a92862fda16351d7192600746746f38e5dStephen Gallagher mail_transaction_log_view_get_prev_pos(view->log_view,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_replace_map(&sync_map_ctx, map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* FIXME: when transaction sync lock is removed, we'll need to handle
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher the case when a transaction is committed while mailbox is being
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher synced ([synced transactions][new transaction][ext transaction]).
056302a92862fda16351d7192600746746f38e5dStephen Gallagher this means int_offset contains [synced] and ext_offset contains
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_transaction_log_view_get_prev_pos(view->log_view,
056302a92862fda16351d7192600746746f38e5dStephen Gallagher /* this has been synced already. we're here only to call
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher expunge handlers and extension update handlers. */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* we'll just skip over broken entries */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher (void)mail_index_sync_record(&sync_map_ctx, thdr, tdata);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_update_hdr_dirty_flag(map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(map->hdr.indexid == index->indexid);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* transaction log tracks internally the current tail offset.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher besides using header updates, it also updates the offset to skip
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher over following external transactions to avoid extra unneeded log
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_tail_offset = index->log->head->max_tail_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
056302a92862fda16351d7192600746746f38e5dStephen Gallagher memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
056302a92862fda16351d7192600746746f38e5dStephen Gallagher /* avoid the same syncing errors the next time */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* restore refcount before closing the view. this is necessary also
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if map got cloned, because view closing would otherwise destroy it */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_map_deinit(&sync_map_ctx);