mail-index-sync-update.c revision 3a78329166819e06f2929ce44e360514c6a80a8e
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen/* Copyright (c) 2004-2017 Dovecot authors, see the included COPYING file */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "lib.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "ioloop.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "array.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mmap-util.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mail-index-modseq.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mail-index-view-private.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mail-index-sync-private.h"
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen#include "mail-transaction-log.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mail-transaction-log-private.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen/* If we have less than this many bytes to sync from log file, don't bother
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen reading the main index */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *map, bool eol)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t prev_seq;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uoff_t prev_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &prev_seq, &prev_offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (prev_seq == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* handling lost changes in view syncing */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!eol) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (prev_offset == ctx->ext_intro_end_offset &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen prev_seq == ctx->ext_intro_seq) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* previous transaction was an extension introduction.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen we probably came here from
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_ext_reset(). if there are any more
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen views which want to continue syncing it needs the
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen intro. so back up a bit more.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen don't do this in case the last transaction in the
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen log is the extension intro, so we don't keep trying
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen to sync it over and over again. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen prev_offset = ctx->ext_intro_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr.log_file_seq = prev_seq;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (map->hdr.log_file_seq != prev_seq) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr.log_file_seq = prev_seq;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr.log_file_tail_offset = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr.log_file_head_offset = prev_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_view *view = ctx->view;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(view->map != map);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_update_log_offset(ctx, view->map, FALSE);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_unmap(&view->map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen view->map = map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen view->index->map = map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic struct mail_index_map *
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *map = ctx->view->map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (map->refcount > 1) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map = mail_index_map_clone(map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_replace_map(ctx, map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_map_move_to_memory(ctx->view->map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return map;
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstruct mail_index_map *
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (void)mail_index_sync_move_to_private_memory(ctx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_record_map_move_to_private(ctx->view->map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return ctx->view->map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmail_index_header_update_counts(struct mail_index_header *hdr,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint8_t old_flags, uint8_t new_flags,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const char **error_r)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* different seen-flag */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((old_flags & MAIL_SEEN) != 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->seen_messages_count == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *error_r = "Seen counter wrong";
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->seen_messages_count--;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen if (hdr->seen_messages_count >= hdr->messages_count) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *error_r = "Seen counter wrong";
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (++hdr->seen_messages_count == hdr->messages_count)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->first_unseen_uid_lowwater = hdr->next_uid;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* different deleted-flag */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((old_flags & MAIL_DELETED) == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->deleted_messages_count++;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->deleted_messages_count > hdr->messages_count) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *error_r = "Deleted counter wrong";
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->deleted_messages_count == 0 ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->deleted_messages_count > hdr->messages_count) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *error_r = "Deleted counter wrong";
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (--hdr->deleted_messages_count == 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen hdr->first_deleted_uid_lowwater = hdr->next_uid;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t uid,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint8_t old_flags, uint8_t new_flags)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *const *maps;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const char *error;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int i, count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen maps = array_get(&ctx->view->map->rec_map->maps, &count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (i = 0; i < count; i++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (uid >= maps[i]->hdr.next_uid)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen continue;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mail_index_header_update_counts(&maps[i]->hdr,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen old_flags, new_flags,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &error) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_set_corrupted(ctx, "%s", error);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t uid, uint8_t old_flags,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint8_t new_flags)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const char *error;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (uid >= ctx->view->map->hdr.next_uid) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_set_corrupted(ctx, "uid %u >= next_uid %u",
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen uid, ctx->view->map->hdr.next_uid);
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen } else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mail_index_header_update_counts(&ctx->view->map->hdr,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen old_flags, new_flags,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen &error) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_set_corrupted(ctx, "%s", error);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t uid, enum mail_flags flags)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *const *maps;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int i, count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen maps = array_get(&ctx->view->map->rec_map->maps, &count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (i = 0; i < count; i++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((flags & MAIL_SEEN) == 0 &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uid < maps[i]->hdr.first_unseen_uid_lowwater)
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen maps[i]->hdr.first_unseen_uid_lowwater = uid;
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen if ((flags & MAIL_DELETED) != 0 &&
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen uid < maps[i]->hdr.first_deleted_uid_lowwater)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen maps[i]->hdr.first_deleted_uid_lowwater = uid;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainensync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t seq1, uint32_t seq2)
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_expunge_handler *eh;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_record *rec;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t seq;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen array_foreach(&ctx->expunge_handlers, eh) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (seq = seq1; seq <= seq2; seq++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->map, seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* FIXME: does expunge handler's return value matter?
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen we probably shouldn't disallow expunges if the
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen handler returns failure.. should it be just changed
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen to return void? */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (void)eh->handler(ctx, seq,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen PTR_OFFSET(rec, eh->record_offset),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen eh->sync_context, eh->context);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic bool
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainensync_expunge_handlers_init(struct mail_index_sync_map_ctx *ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* call expunge handlers only when syncing index file */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!ctx->expunge_handlers_set)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_init_expunge_handlers(ctx);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!array_is_created(&ctx->expunge_handlers))
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainensync_expunge_range(struct mail_index_sync_map_ctx *ctx, const ARRAY_TYPE(seq_range) *seqs)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct seq_range *range;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int i, count;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t dest_seq1, prev_seq2, orig_rec_count;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen range = array_get(seqs, &count);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (count == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map = mail_index_sync_get_atomic_map(ctx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* call the expunge handlers first */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (sync_expunge_handlers_init(ctx)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0; i < count; i++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_expunge_call_handlers(ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen range[i].seq1, range[i].seq2);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen prev_seq2 = 0;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen dest_seq1 = 1;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen orig_rec_count = map->rec_map->records_count;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen for (i = 0; i < count; i++) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen uint32_t seq1 = range[i].seq1;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen uint32_t seq2 = range[i].seq2;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen struct mail_index_record *rec;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen uint32_t seq_count, seq;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(seq1 > prev_seq2);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (seq = seq1; seq <= seq2; seq++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags, 0);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen if (prev_seq2+1 <= seq1-1) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* @UNSAFE: move (prev_seq2+1) .. (seq1-1) to its
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen final location in the map if necessary */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t move_count = (seq1-1) - (prev_seq2+1) + 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (prev_seq2+1-1 != dest_seq1-1)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memmove(MAIL_INDEX_REC_AT_SEQ(map, dest_seq1),
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen MAIL_INDEX_REC_AT_SEQ(map, prev_seq2+1),
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen move_count * map->hdr.record_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen dest_seq1 += move_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen seq_count = seq2 - seq1 + 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->rec_map->records_count -= seq_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr.messages_count -= seq_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen prev_seq2 = seq2;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* Final stragglers */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (orig_rec_count > prev_seq2) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t final_move_count = orig_rec_count - prev_seq2;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memmove(MAIL_INDEX_REC_AT_SEQ(map, dest_seq1),
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen MAIL_INDEX_REC_AT_SEQ(map, prev_seq2+1),
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen final_move_count * map->hdr.record_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void *sync_append_record(struct mail_index_map *map)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size_t append_pos;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen void *ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen append_pos = map->rec_map->records_count * map->hdr.record_size;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen ret = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen map->hdr.record_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->rec_map->records =
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen buffer_get_modifiable_data(map->rec_map->buffer, NULL);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic bool sync_update_ignored_change(struct mail_index_sync_map_ctx *ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_transaction_commit_result *result =
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ctx->view->index->sync_commit_result;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen uint32_t prev_log_seq;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uoff_t prev_log_offset, trans_start_offset, trans_end_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (result == NULL)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* we'll return TRUE if this modseq change was written within the
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen transaction that was just committed */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &prev_log_seq, &prev_log_offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (prev_log_seq != result->log_file_seq)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen trans_end_offset = result->log_file_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen trans_start_offset = trans_end_offset - result->commit_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (prev_log_offset < trans_start_offset ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen prev_log_offset >= trans_end_offset)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainensync_modseq_update(struct mail_index_sync_map_ctx *ctx,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_transaction_modseq_update *u,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_view *view = ctx->view;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_transaction_modseq_update *end;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen uint32_t seq;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen uint64_t min_modseq;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen int ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen end = CONST_PTR_OFFSET(u, size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (; u < end; u++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (u->uid == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen seq = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen else if (!mail_index_lookup_seq(view, u->uid, &seq))
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen continue;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen min_modseq = ((uint64_t)u->modseq_high32 << 32) |
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen u->modseq_low32;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen ret = seq == 0 ? 1 :
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_modseq_set(view, seq, min_modseq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_set_corrupted(ctx,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen "modseqs updated before they were enabled");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (ret == 0 && sync_update_ignored_change(ctx))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen view->index->sync_commit_result->ignored_modseq_changes++;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int sync_append(const struct mail_index_record *rec,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_sync_map_ctx *ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_view *view = ctx->view;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_map *map = view->map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_record *old_rec;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen enum mail_flags new_flags;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen void *dest;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (rec->uid < map->hdr.next_uid) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_set_corrupted(ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "Append with UID %u, but next_uid = %u",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec->uid, map->hdr.next_uid);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* move to memory. the mapping is written when unlocking so we don't
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen waste time re-mmap()ing multiple times or waste space growing index
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen file too large */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map = mail_index_sync_move_to_private_memory(ctx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (rec->uid <= map->rec_map->last_appended_uid) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(map->hdr.messages_count < map->rec_map->records_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* the flags may have changed since it was added to map.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen use the updated flags already, so flag counters won't get
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen broken. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen old_rec = MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(old_rec->uid == rec->uid);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen new_flags = old_rec->flags;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen } else {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* don't rely on buffer->used being at the correct position.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen at least expunges can move it */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen dest = sync_append_record(map);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memcpy(dest, rec, sizeof(*rec));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.record_size - sizeof(*rec));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->rec_map->records_count++;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->rec_map->last_appended_uid = rec->uid;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen new_flags = rec->flags;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_modseq_append(ctx->modseq_ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->rec_map->records_count);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.messages_count++;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.next_uid = rec->uid+1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0 &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen (view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_header_update_counts(ctx, rec->uid, 0, new_flags);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int sync_flag_update(const struct mail_transaction_flag_update *u,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_sync_map_ctx *ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mail_index_view *view = ctx->view;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_record *rec;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint8_t flag_mask, old_flags;
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen uint32_t seq, seq1, seq2;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2))
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_modseq_update_flags(ctx->modseq_ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen u->add_flags | u->remove_flags,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen seq1, seq2);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0 &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen (view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen flag_mask = ~u->remove_flags;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (((u->add_flags | u->remove_flags) &
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (MAIL_SEEN | MAIL_DELETED)) == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* we're not modifying any counted/lowwatered flags */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (seq = seq1; seq <= seq2; seq++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (seq = seq1; seq <= seq2; seq++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen old_flags = rec->flags;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_header_update_lowwaters(ctx, rec->uid,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec->flags);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_header_update_counts_all(ctx, rec->uid,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen old_flags,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec->flags);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int sync_header_update(const struct mail_transaction_header_update *u,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_sync_map_ctx *ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, field) \
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ((u)->offset <= offsetof(struct mail_index_header, field) && \
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (u)->offset + (u)->size > offsetof(struct mail_index_header, field))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *map = ctx->view->map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t orig_next_uid = map->hdr.next_uid;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (u->offset >= map->hdr.base_header_size ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen u->offset + u->size > map->hdr.base_header_size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_set_corrupted(ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "Header update outside range: %u + %u > %u",
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen u->offset, u->size, map->hdr.base_header_size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr_base = map->hdr_copy_buf->data;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* @UNSAFE */
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen memcpy(PTR_OFFSET(&map->hdr, u->offset),
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen u + 1, u->size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else if (u->offset < sizeof(map->hdr)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memcpy(PTR_OFFSET(&map->hdr, u->offset),
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen u + 1, sizeof(map->hdr) - u->offset);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (map->hdr.next_uid < orig_next_uid) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* next_uid update tried to shrink its value. this can happen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen in some race conditions with e.g. with dsync, so just
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen silently ignore it. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.next_uid = orig_next_uid;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* the tail offset updates are intended for internal transaction
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen log handling. we'll update the offset in the header only when
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen the sync is finished. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_transaction_header *hdr,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const void *data)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen int ret = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen case MAIL_TRANSACTION_APPEND: {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_record *rec, *end;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen end = CONST_PTR_OFFSET(data, hdr->size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (rec = data; rec < end; rec++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ret = sync_append(rec, ctx);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret <= 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen case MAIL_TRANSACTION_EXPUNGE:
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_transaction_expunge *rec = data, *end;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ARRAY_TYPE(seq_range) seqs;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t seq1, seq2;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* this is simply a request for expunge */
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen t_array_init(&seqs, 64);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen end = CONST_PTR_OFFSET(data, hdr->size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (; rec != end; rec++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mail_index_lookup_seq_range(ctx->view,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec->uid1, rec->uid2, &seq1, &seq2))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen seq_range_array_add_range(&seqs, seq1, seq2);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_expunge_range(ctx, &seqs);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen case MAIL_TRANSACTION_EXPUNGE_GUID:
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_transaction_expunge_guid *rec = data, *end;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ARRAY_TYPE(seq_range) seqs;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t seq;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen /* this is simply a request for expunge */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen t_array_init(&seqs, 64);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen end = CONST_PTR_OFFSET(data, hdr->size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (; rec != end; rec++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(rec->uid != 0);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mail_index_lookup_seq(ctx->view, rec->uid, &seq))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen seq_range_array_add(&seqs, seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_expunge_range(ctx, &seqs);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen case MAIL_TRANSACTION_FLAG_UPDATE: {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_transaction_flag_update *rec, *end;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen end = CONST_PTR_OFFSET(data, hdr->size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (rec = data; rec < end; rec++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = sync_flag_update(rec, ctx);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret <= 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case MAIL_TRANSACTION_HEADER_UPDATE: {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_transaction_header_update *rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int i;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0; i < hdr->size; ) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec = CONST_PTR_OFFSET(data, i);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ret = sync_header_update(rec, ctx);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret <= 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i += sizeof(*rec) + rec->size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((i % 4) != 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i += 4 - (i % 4);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case MAIL_TRANSACTION_EXT_INTRO: {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_transaction_ext_intro *rec = data;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int i;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t prev_seq;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uoff_t prev_offset;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen &prev_seq, &prev_offset);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ctx->ext_intro_seq = prev_seq;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ctx->ext_intro_offset = prev_offset;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ctx->ext_intro_end_offset =
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen prev_offset + hdr->size + sizeof(*hdr);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0; i < hdr->size; ) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (i + sizeof(*rec) > hdr->size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* should be just extra padding */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen rec = CONST_PTR_OFFSET(data, i);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen /* name_size checked by _log_view_next() */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(i + sizeof(*rec) + rec->name_size <= hdr->size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_index_sync_ext_intro(ctx, rec);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret <= 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i += sizeof(*rec) + rec->name_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((i % 4) != 0)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen i += 4 - (i % 4);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen case MAIL_TRANSACTION_EXT_RESET: {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_transaction_ext_reset rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* old versions have only new_reset_id */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (hdr->size < sizeof(uint32_t)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_set_corrupted(ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "ext reset: invalid record size");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_zero(&rec);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen memcpy(&rec, data, I_MIN(hdr->size, sizeof(rec)));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_index_sync_ext_reset(ctx, &rec);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen const struct mail_transaction_ext_hdr_update *rec;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen unsigned int i;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen for (i = 0; i < hdr->size; ) {
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen rec = CONST_PTR_OFFSET(data, i);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (i + sizeof(*rec) > hdr->size ||
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i + sizeof(*rec) + rec->size > hdr->size) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_set_corrupted(ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "ext hdr update: invalid record size");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = -1;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen rec->size, rec + 1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret <= 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i += sizeof(*rec) + rec->size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((i % 4) != 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i += 4 - (i % 4);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen case MAIL_TRANSACTION_EXT_HDR_UPDATE32: {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_transaction_ext_hdr_update32 *rec;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen unsigned int i;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen for (i = 0; i < hdr->size; ) {
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen rec = CONST_PTR_OFFSET(data, i);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (i + sizeof(*rec) > hdr->size ||
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i + sizeof(*rec) + rec->size > hdr->size) {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen mail_index_sync_set_corrupted(ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "ext hdr update: invalid record size");
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen ret = -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen rec->size, rec + 1);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret <= 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i += sizeof(*rec) + rec->size;
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen if ((i % 4) != 0)
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen i += 4 - (i % 4);
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen }
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen break;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen }
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen case MAIL_TRANSACTION_EXT_REC_UPDATE: {
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen const struct mail_transaction_ext_rec_update *rec;
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen unsigned int i, record_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ctx->cur_ext_map_idx == (uint32_t)-1) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_set_corrupted(ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "Extension record updated "
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen "without intro prefix");
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen ret = -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen if (ctx->cur_ext_ignore) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ret = 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen /* the record is padded to 32bits in the transaction log */
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen record_size = (sizeof(*rec) + ctx->cur_ext_record_size + 3) & ~3;
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0; i < hdr->size; i += record_size) {
38efb29cb9453917c1b642095a37d0e901f56eeeTimo Sirainen rec = CONST_PTR_OFFSET(data, i);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (i + record_size > hdr->size) {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen mail_index_sync_set_corrupted(ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "ext rec update: invalid record size");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mail_index_sync_ext_rec_update(ctx, rec);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret <= 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen case MAIL_TRANSACTION_EXT_ATOMIC_INC: {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen const struct mail_transaction_ext_atomic_inc *rec, *end;
171f1d1a6e3205447e83cd3984e76b1e8628de43Timo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen if (ctx->cur_ext_map_idx == (uint32_t)-1) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen mail_index_sync_set_corrupted(ctx,
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen "Extension record updated "
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen "without intro prefix");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ret = -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ctx->cur_ext_ignore) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = 1;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen end = CONST_PTR_OFFSET(data, hdr->size);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen for (rec = data; rec < end; rec++) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen ret = mail_index_sync_ext_atomic_inc(ctx, rec);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (ret <= 0)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen case MAIL_TRANSACTION_KEYWORD_UPDATE: {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_transaction_keyword_update *rec = data;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen ret = mail_index_sync_keywords(ctx, hdr, rec);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen case MAIL_TRANSACTION_KEYWORD_RESET: {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_transaction_keyword_reset *rec = data;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen case MAIL_TRANSACTION_MODSEQ_UPDATE: {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_transaction_modseq_update *rec = data;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen ret = sync_modseq_update(ctx, rec, hdr->size);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen case MAIL_TRANSACTION_INDEX_DELETED:
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* next sync finishes the deletion */
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen ctx->view->index->index_delete_requested = TRUE;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen } else {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen /* transaction log reading handles this */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case MAIL_TRANSACTION_INDEX_UNDELETED:
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ctx->view->index->index_delete_requested = FALSE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case MAIL_TRANSACTION_BOUNDARY:
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen default:
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen mail_index_sync_set_corrupted(ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "Unknown transaction record type 0x%x",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen (hdr->type & MAIL_TRANSACTION_TYPE_MASK));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainenint mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen const struct mail_transaction_header *hdr,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const void *data)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen{
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen int ret;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen T_BEGIN {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen ret = mail_index_sync_record_real(ctx, hdr, data);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen } T_END;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen return ret;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainenvoid mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen struct mail_index_view *view,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen enum mail_index_sync_handler_type type)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_zero(sync_map_ctx);
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen sync_map_ctx->view = view;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen sync_map_ctx->cur_ext_map_idx = (uint32_t)-1;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen sync_map_ctx->type = type;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen sync_map_ctx->modseq_ctx = mail_index_modseq_sync_begin(sync_map_ctx);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_sync_init_handlers(sync_map_ctx);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenvoid mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen i_assert(sync_map_ctx->modseq_ctx == NULL);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen buffer_free(&sync_map_ctx->unknown_extensions);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (sync_map_ctx->expunge_handlers_used)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_deinit_handlers(sync_map_ctx);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenstatic void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const struct mail_index_record *rec;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen uint32_t seq;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 ||
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen (map->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) != 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* do we have dirty flags anymore? */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (seq = 1; seq <= map->rec_map->records_count; seq++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen break;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen#ifdef DEBUG
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenvoid mail_index_map_check(struct mail_index_map *map)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen const struct mail_index_header *hdr = &map->hdr;
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen unsigned int del = 0, seen = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t seq, prev_uid = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(hdr->messages_count <= map->rec_map->records_count);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (seq = 1; seq <= hdr->messages_count; seq++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_index_record *rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen i_assert(rec->uid > prev_uid);
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen prev_uid = rec->uid;
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen if (rec->flags & MAIL_DELETED) {
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen del++;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (rec->flags & MAIL_SEEN)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen seen++;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen else
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(del == hdr->deleted_messages_count);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(seen == hdr->seen_messages_count);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen#endif
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenint mail_index_sync_map(struct mail_index_map **_map,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen enum mail_index_sync_handler_type type, bool force,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const char *sync_reason)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_map *map = *_map;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index *index = map->index;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_view *view;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_transaction_header *thdr;
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen const void *tdata;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t prev_seq;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen uoff_t start_offset, prev_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const char *reason, *error;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen int ret;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen bool had_dirty, reset;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (index->log->head == NULL) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(!force);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!force && (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* see if we'd prefer to reopen the index file instead of
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen syncing the current map from the transaction log.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen don't check this if mmap is disabled, because reopening
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index causes sync to get lost. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uoff_t log_size, index_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (index->fd == -1 &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen index->log->head->hdr.prev_file_seq != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* we don't know the index's size, so use the
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen smallest index size we're willing to read */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index_size = map->hdr.header_size +
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->rec_map->records_count *
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen map->hdr.record_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* this isn't necessary correct currently, but it should be
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen close enough */
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen log_size = index->log->head->last_size;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (log_size > start_offset &&
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen log_size - start_offset > index_size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen view = mail_index_view_open_with_map(index, map);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen ret = mail_transaction_log_view_set(view->log_view,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->hdr.log_file_seq, start_offset,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen (uint32_t)-1, (uoff_t)-1,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen &reset, &reason);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (ret <= 0) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_view_close(&view);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (force && ret < 0) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* if we failed because of a syscall error, make sure
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen we return a failure. */
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen return -1;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (force && ret == 0) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen /* the seq/offset is probably broken */
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_set_error(index, "Index %s: Lost log for "
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen "seq=%u offset=%"PRIuUOFF_T": %s "
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen "(initial_mapped=%d, reason=%s)", index->filepath,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.log_file_seq, start_offset, reason,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen index->initial_mapped ? 1 : 0, sync_reason);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (void)mail_index_fsck(index);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* can't use it. sync by re-reading index. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_transaction_log_get_head(index->log, &prev_seq, &prev_offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (prev_seq != map->hdr.log_file_seq ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen prev_offset - map->hdr.log_file_tail_offset >
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index->optimization_set.index.rewrite_min_log_bytes) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* we're reading more from log than we would have preferred.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen remember that we probably want to rewrite index soon. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen index->index_min_write = TRUE;
}
/* view referenced the map. avoid unnecessary map cloning by
unreferencing the map while view exists. */
map->refcount--;
had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
if (had_dirty)
map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
if (map->hdr_base != map->hdr_copy_buf->data) {
/* if syncing updates the header, it updates hdr_copy_buf
and updates hdr_base to hdr_copy_buf. so the buffer must
initially contain a valid header or we'll break it when
writing it. */
buffer_set_used_size(map->hdr_copy_buf, 0);
buffer_append(map->hdr_copy_buf, map->hdr_base,
map->hdr.header_size);
map->hdr_base = map->hdr_copy_buf->data;
}
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
mail_index_sync_map_init(&sync_map_ctx, view, type);
if (reset) {
/* Reset the entire index. Leave only indexid and
log_file_seq. */
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
map = mail_index_map_alloc(index);
if ((index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0)
map->hdr.flags |= MAIL_INDEX_HDR_FLAG_FSCKD;
map->hdr.log_file_seq = prev_seq;
map->hdr.log_file_tail_offset = 0;
mail_index_sync_replace_map(&sync_map_ctx, map);
}
map = NULL;
/* FIXME: when transaction sync lock is removed, we'll need to handle
the case when a transaction is committed while mailbox is being
synced ([synced transactions][new transaction][ext transaction]).
this means int_offset contains [synced] and ext_offset contains
all */
while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
&tdata)) > 0) {
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
if (LOG_IS_BEFORE(prev_seq, prev_offset,
view->map->hdr.log_file_seq,
view->map->hdr.log_file_head_offset)) {
/* this has been synced already. we're here only to call
expunge handlers and extension update handlers. */
i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
continue;
if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
continue;
}
/* we'll just skip over broken entries */
(void)mail_index_sync_record(&sync_map_ctx, thdr, tdata);
}
map = view->map;
if (had_dirty)
mail_index_sync_update_hdr_dirty_flag(map);
mail_index_modseq_sync_end(&sync_map_ctx.modseq_ctx);
mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);
#ifdef DEBUG
mail_index_map_check(map);
#endif
i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0);
/* transaction log tracks internally the current tail offset.
besides using header updates, it also updates the offset to skip
over following external transactions to avoid extra unneeded log
reading. */
i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq);
if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) {
map->hdr.log_file_tail_offset =
index->log->head->max_tail_offset;
}
buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
map->hdr_copy_buf->used);
}
/* restore refcount before closing the view. this is necessary also
if map got cloned, because view closing would otherwise destroy it */
map->refcount++;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(&view);
i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
if (mail_index_map_check_header(map, &error) <= 0) {
mail_index_set_error(index,
"Synchronization corrupted index header %s: %s",
index->filepath, error);
(void)mail_index_fsck(index);
map = index->map;
} else if (sync_map_ctx.errors) {
/* make sure the index looks valid now */
(void)mail_index_fsck(index);
map = index->map;
}
*_map = map;
return ret < 0 ? -1 : 1;
}