mail-index-sync-update.c revision 02d6628c1fea2990c67c60b111c8e68867160885
294N/A/* Copyright (c) 2004-2016 Dovecot authors, see the included COPYING file */
294N/A
787N/A#include "lib.h"
789N/A#include "ioloop.h"
789N/A#include "array.h"
789N/A#include "mmap-util.h"
1336N/A#include "mail-index-modseq.h"
789N/A#include "mail-index-view-private.h"
789N/A#include "mail-index-sync-private.h"
873N/A#include "mail-transaction-log.h"
789N/A#include "mail-transaction-log-private.h"
877N/A
789N/A/* If we have less than this many bytes to sync from log file, don't bother
294N/A reading the main index */
873N/A#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048
873N/A
789N/Astatic void
789N/Amail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
789N/A struct mail_index_map *map, bool eol)
873N/A{
789N/A uint32_t prev_seq;
789N/A uoff_t prev_offset;
789N/A
789N/A mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
1342N/A &prev_seq, &prev_offset);
1342N/A if (prev_seq == 0) {
789N/A /* handling lost changes in view syncing */
789N/A return;
789N/A }
789N/A
789N/A if (!eol) {
1336N/A if (prev_offset == ctx->ext_intro_end_offset &&
1336N/A prev_seq == ctx->ext_intro_seq) {
789N/A /* previous transaction was an extension introduction.
873N/A we probably came here from
1336N/A mail_index_sync_ext_reset(). if there are any more
1336N/A views which want to continue syncing it needs the
1336N/A intro. so back up a bit more.
1336N/A
1336N/A don't do this in case the last transaction in the
1336N/A log is the extension intro, so we don't keep trying
1336N/A to sync it over and over again. */
1336N/A prev_offset = ctx->ext_intro_offset;
1336N/A }
1336N/A map->hdr.log_file_seq = prev_seq;
1336N/A } else {
1336N/A i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
1336N/A if (map->hdr.log_file_seq != prev_seq) {
1336N/A map->hdr.log_file_seq = prev_seq;
1336N/A map->hdr.log_file_tail_offset = 0;
1336N/A }
1336N/A }
1336N/A map->hdr.log_file_head_offset = prev_offset;
873N/A}
873N/A
873N/Astatic void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
789N/A struct mail_index_map *map)
787N/A{
294N/A struct mail_index_view *view = ctx->view;
1336N/A
1336N/A i_assert(view->map != map);
1336N/A
1336N/A mail_index_sync_update_log_offset(ctx, view->map, FALSE);
1337N/A mail_index_unmap(&view->map);
294N/A view->map = map;
294N/A
294N/A if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
1336N/A view->index->map = map;
1336N/A
1337N/A mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
1337N/A}
1337N/A
1337N/Astatic struct mail_index_map *
1337N/Amail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
1337N/A{
1337N/A struct mail_index_map *map = ctx->view->map;
1337N/A
1337N/A if (map->refcount > 1) {
1336N/A map = mail_index_map_clone(map);
1337N/A mail_index_sync_replace_map(ctx, map);
1337N/A }
1337N/A
294N/A if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
294N/A mail_index_map_move_to_memory(ctx->view->map);
294N/A mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
294N/A return map;
789N/A}
294N/A
1109N/Astruct mail_index_map *
1109N/Amail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
1337N/A{
1337N/A (void)mail_index_sync_move_to_private_memory(ctx);
1109N/A mail_index_record_map_move_to_private(ctx->view->map);
1337N/A mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
1337N/A return ctx->view->map;
1109N/A}
787N/A
869N/Astatic int
869N/Amail_index_header_update_counts(struct mail_index_header *hdr,
869N/A uint8_t old_flags, uint8_t new_flags,
869N/A const char **error_r)
789N/A{
789N/A if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
789N/A /* different seen-flag */
789N/A if ((old_flags & MAIL_SEEN) != 0) {
789N/A if (hdr->seen_messages_count == 0) {
789N/A *error_r = "Seen counter wrong";
294N/A return -1;
789N/A }
294N/A hdr->seen_messages_count--;
294N/A } else {
789N/A if (hdr->seen_messages_count >= hdr->messages_count) {
294N/A *error_r = "Seen counter wrong";
294N/A return -1;
294N/A }
294N/A
294N/A if (++hdr->seen_messages_count == hdr->messages_count)
294N/A hdr->first_unseen_uid_lowwater = hdr->next_uid;
294N/A }
789N/A }
789N/A
789N/A if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
787N/A /* different deleted-flag */
789N/A if ((old_flags & MAIL_DELETED) == 0) {
294N/A hdr->deleted_messages_count++;
869N/A if (hdr->deleted_messages_count > hdr->messages_count) {
868N/A *error_r = "Deleted counter wrong";
789N/A return -1;
789N/A }
869N/A } else {
869N/A if (hdr->deleted_messages_count == 0 ||
869N/A hdr->deleted_messages_count > hdr->messages_count) {
869N/A *error_r = "Deleted counter wrong";
789N/A return -1;
789N/A }
869N/A
949N/A if (--hdr->deleted_messages_count == 0)
789N/A hdr->first_deleted_uid_lowwater = hdr->next_uid;
789N/A }
979N/A }
1340N/A return 0;
1340N/A}
789N/A
787N/Astatic void
787N/Amail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
294N/A uint32_t uid,
294N/A uint8_t old_flags, uint8_t new_flags)
789N/A{
294N/A struct mail_index_map *const *maps;
294N/A const char *error;
294N/A unsigned int i, count;
294N/A
789N/A maps = array_get(&ctx->view->map->rec_map->maps, &count);
294N/A for (i = 0; i < count; i++) {
294N/A if (uid >= maps[i]->hdr.next_uid)
294N/A continue;
789N/A
294N/A if (mail_index_header_update_counts(&maps[i]->hdr,
789N/A old_flags, new_flags,
294N/A &error) < 0)
789N/A mail_index_sync_set_corrupted(ctx, "%s", error);
789N/A }
911N/A}
911N/A
911N/Astatic void
915N/Amail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
911N/A uint32_t uid, uint8_t old_flags,
911N/A uint8_t new_flags)
911N/A{
869N/A const char *error;
869N/A
869N/A if (uid >= ctx->view->map->hdr.next_uid) {
869N/A mail_index_sync_set_corrupted(ctx, "uid %u >= next_uid %u",
789N/A uid, ctx->view->map->hdr.next_uid);
857N/A } else {
789N/A if (mail_index_header_update_counts(&ctx->view->map->hdr,
789N/A old_flags, new_flags,
789N/A &error) < 0)
789N/A mail_index_sync_set_corrupted(ctx, "%s", error);
789N/A }
294N/A}
294N/A
789N/Astatic void
294N/Amail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
294N/A uint32_t uid, enum mail_flags flags)
789N/A{
294N/A struct mail_index_map *const *maps;
294N/A unsigned int i, count;
294N/A
789N/A maps = array_get(&ctx->view->map->rec_map->maps, &count);
294N/A for (i = 0; i < count; i++) {
1339N/A if ((flags & MAIL_SEEN) == 0 &&
1339N/A uid < maps[i]->hdr.first_unseen_uid_lowwater)
1340N/A maps[i]->hdr.first_unseen_uid_lowwater = uid;
1340N/A if ((flags & MAIL_DELETED) != 0 &&
877N/A uid < maps[i]->hdr.first_deleted_uid_lowwater)
869N/A maps[i]->hdr.first_deleted_uid_lowwater = uid;
869N/A }
868N/A}
869N/A
869N/Astatic void
869N/Async_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
868N/A uint32_t seq1, uint32_t seq2)
789N/A{
941N/A const struct mail_index_expunge_handler *eh;
1086N/A struct mail_index_record *rec;
911N/A uint32_t seq;
941N/A
911N/A array_foreach(&ctx->expunge_handlers, eh) {
294N/A for (seq = seq1; seq <= seq2; seq++) {
869N/A rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->map, seq);
869N/A /* FIXME: does expunge handler's return value matter?
941N/A we probably shouldn't disallow expunges if the
1086N/A handler returns failure.. should it be just changed
872N/A to return void? */
941N/A (void)eh->handler(ctx, seq,
872N/A PTR_OFFSET(rec, eh->record_offset),
869N/A eh->sync_context, eh->context);
994N/A }
1086N/A }
994N/A}
994N/A
994N/Astatic bool
994N/Async_expunge_handlers_init(struct mail_index_sync_map_ctx *ctx)
869N/A{
1109N/A /* call expunge handlers only when syncing index file */
1109N/A if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE)
1109N/A return FALSE;
1109N/A
1109N/A if (!ctx->expunge_handlers_set)
1109N/A mail_index_sync_init_expunge_handlers(ctx);
868N/A
869N/A if (!array_is_created(&ctx->expunge_handlers))
869N/A return FALSE;
869N/A return TRUE;
869N/A}
868N/A
870N/Astatic void
870N/Async_expunge_range(struct mail_index_sync_map_ctx *ctx, const ARRAY_TYPE(seq_range) *seqs)
870N/A{
870N/A struct mail_index_map *map;
870N/A const struct seq_range *range;
869N/A unsigned int i, count;
869N/A uint32_t dest_seq1, prev_seq2, orig_rec_count;
869N/A
869N/A range = array_get(seqs, &count);
1336N/A if (count == 0)
1336N/A return;
1336N/A
1336N/A map = mail_index_sync_get_atomic_map(ctx);
1336N/A
1336N/A /* call the expunge handlers first */
1336N/A if (sync_expunge_handlers_init(ctx)) {
1336N/A for (i = 0; i < count; i++) {
1336N/A sync_expunge_call_handlers(ctx,
1336N/A range[i].seq1, range[i].seq2);
1336N/A }
1336N/A }
1336N/A
869N/A prev_seq2 = 0;
869N/A dest_seq1 = 1;
1342N/A orig_rec_count = map->rec_map->records_count;
868N/A for (i = 0; i < count; i++) {
789N/A uint32_t seq1 = range[i].seq1;
789N/A uint32_t seq2 = range[i].seq2;
789N/A struct mail_index_record *rec;
789N/A uint32_t seq_count, seq;
789N/A
877N/A i_assert(seq1 > prev_seq2);
877N/A
877N/A for (seq = seq1; seq <= seq2; seq++) {
877N/A rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
877N/A mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags, 0);
787N/A }
294N/A
789N/A if (prev_seq2+1 <= seq1-1) {
789N/A /* @UNSAFE: move (prev_seq2+1) .. (seq1-1) to its
789N/A final location in the map if necessary */
789N/A uint32_t move_count = (seq1-1) - (prev_seq2+1) + 1;
789N/A if (prev_seq2+1-1 != dest_seq1-1)
789N/A memmove(MAIL_INDEX_REC_AT_SEQ(map, dest_seq1),
877N/A MAIL_INDEX_REC_AT_SEQ(map, prev_seq2+1),
877N/A move_count * map->hdr.record_size);
877N/A dest_seq1 += move_count;
877N/A }
877N/A seq_count = seq2 - seq1 + 1;
789N/A map->rec_map->records_count -= seq_count;
294N/A map->hdr.messages_count -= seq_count;
869N/A mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2);
869N/A prev_seq2 = seq2;
1091N/A }
869N/A /* Final stragglers */
869N/A if (orig_rec_count > prev_seq2) {
869N/A uint32_t final_move_count = orig_rec_count - prev_seq2;
869N/A memmove(MAIL_INDEX_REC_AT_SEQ(map, dest_seq1),
1086N/A MAIL_INDEX_REC_AT_SEQ(map, prev_seq2+1),
1086N/A final_move_count * map->hdr.record_size);
1086N/A }
1086N/A}
294N/A
294N/Astatic void *sync_append_record(struct mail_index_map *map)
294N/A{
294N/A size_t append_pos;
294N/A void *ret;
294N/A
294N/A append_pos = map->rec_map->records_count * map->hdr.record_size;
294N/A ret = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
294N/A map->hdr.record_size);
294N/A map->rec_map->records =
869N/A buffer_get_modifiable_data(map->rec_map->buffer, NULL);
294N/A return ret;
294N/A}
869N/A
873N/Astatic bool sync_update_ignored_change(struct mail_index_sync_map_ctx *ctx)
1086N/A{
1091N/A struct mail_index_transaction_commit_result *result =
873N/A ctx->view->index->sync_commit_result;
873N/A uint32_t prev_log_seq;
872N/A uoff_t prev_log_offset, trans_start_offset, trans_end_offset;
1155N/A
1114N/A if (result == NULL)
1114N/A return FALSE;
1114N/A
1114N/A /* we'll return TRUE if this modseq change was written within the
789N/A transaction that was just committed */
789N/A mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
789N/A &prev_log_seq, &prev_log_offset);
789N/A if (prev_log_seq != result->log_file_seq)
789N/A return FALSE;
789N/A
789N/A trans_end_offset = result->log_file_offset;
789N/A trans_start_offset = trans_end_offset - result->commit_size;
789N/A if (prev_log_offset < trans_start_offset ||
789N/A prev_log_offset >= trans_end_offset)
789N/A return FALSE;
789N/A
789N/A return TRUE;
789N/A}
789N/A
789N/Astatic int
789N/Async_modseq_update(struct mail_index_sync_map_ctx *ctx,
789N/A const struct mail_transaction_modseq_update *u,
789N/A unsigned int size)
789N/A{
789N/A struct mail_index_view *view = ctx->view;
789N/A const struct mail_transaction_modseq_update *end;
789N/A uint32_t seq;
789N/A uint64_t min_modseq, highest_modseq = 0;
789N/A int ret;
789N/A
789N/A end = CONST_PTR_OFFSET(u, size);
789N/A for (; u < end; u++) {
789N/A if (u->uid == 0)
789N/A seq = 0;
789N/A else if (!mail_index_lookup_seq(view, u->uid, &seq))
789N/A continue;
789N/A
789N/A min_modseq = ((uint64_t)u->modseq_high32 << 32) |
1340N/A u->modseq_low32;
1340N/A if (highest_modseq < min_modseq)
1340N/A highest_modseq = min_modseq;
1340N/A
1340N/A ret = seq == 0 ? 1 :
789N/A mail_index_modseq_set(view, seq, min_modseq);
789N/A if (ret < 0) {
789N/A mail_index_sync_set_corrupted(ctx,
789N/A "modseqs updated before they were enabled");
789N/A return -1;
789N/A }
789N/A if (ret == 0 && sync_update_ignored_change(ctx))
789N/A view->index->sync_commit_result->ignored_modseq_changes++;
789N/A }
789N/A
789N/A mail_index_modseq_update_highest(ctx->modseq_ctx, highest_modseq);
789N/A return 1;
789N/A}
789N/A
789N/Astatic int sync_append(const struct mail_index_record *rec,
789N/A struct mail_index_sync_map_ctx *ctx)
294N/A{
1155N/A struct mail_index_view *view = ctx->view;
1155N/A struct mail_index_map *map = view->map;
1155N/A const struct mail_index_record *old_rec;
1339N/A enum mail_flags new_flags;
1155N/A void *dest;
1155N/A
1155N/A if (rec->uid < map->hdr.next_uid) {
1155N/A mail_index_sync_set_corrupted(ctx,
1155N/A "Append with UID %u, but next_uid = %u",
1155N/A rec->uid, map->hdr.next_uid);
1155N/A return -1;
1155N/A }
1155N/A
1155N/A /* move to memory. the mapping is written when unlocking so we don't
1155N/A waste time re-mmap()ing multiple times or waste space growing index
789N/A file too large */
789N/A map = mail_index_sync_move_to_private_memory(ctx);
1155N/A
1155N/A if (rec->uid <= map->rec_map->last_appended_uid) {
789N/A i_assert(map->hdr.messages_count < map->rec_map->records_count);
789N/A /* the flags may have changed since it was added to map.
789N/A use the updated flags already, so flag counters won't get
789N/A broken. */
789N/A old_rec = MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count);
789N/A i_assert(old_rec->uid == rec->uid);
1155N/A new_flags = old_rec->flags;
1155N/A } else {
1155N/A /* don't rely on buffer->used being at the correct position.
1155N/A at least expunges can move it */
1155N/A dest = sync_append_record(map);
1155N/A memcpy(dest, rec, sizeof(*rec));
1155N/A memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
1155N/A map->hdr.record_size - sizeof(*rec));
1155N/A map->rec_map->records_count++;
1155N/A map->rec_map->last_appended_uid = rec->uid;
789N/A new_flags = rec->flags;
789N/A
789N/A mail_index_modseq_append(ctx->modseq_ctx,
789N/A map->rec_map->records_count);
1091N/A }
789N/A
789N/A map->hdr.messages_count++;
789N/A map->hdr.next_uid = rec->uid+1;
1091N/A
1091N/A if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
1091N/A map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
1091N/A
1091N/A mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
1091N/A mail_index_sync_header_update_counts(ctx, rec->uid, 0, new_flags);
1091N/A return 1;
1091N/A}
1091N/A
1091N/Astatic int sync_flag_update(const struct mail_transaction_flag_update *u,
1091N/A struct mail_index_sync_map_ctx *ctx)
1091N/A{
1091N/A struct mail_index_view *view = ctx->view;
1091N/A struct mail_index_record *rec;
1091N/A uint8_t flag_mask, old_flags;
1091N/A uint32_t seq, seq1, seq2;
1091N/A
1091N/A if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2))
1091N/A return 1;
789N/A
1091N/A if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u)) {
789N/A mail_index_modseq_update_flags(ctx->modseq_ctx,
789N/A u->add_flags | u->remove_flags,
789N/A seq1, seq2);
789N/A }
789N/A
789N/A if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
789N/A view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
789N/A
789N/A flag_mask = ~u->remove_flags;
789N/A
789N/A if (((u->add_flags | u->remove_flags) &
789N/A (MAIL_SEEN | MAIL_DELETED)) == 0) {
789N/A /* we're not modifying any counted/lowwatered flags */
789N/A for (seq = seq1; seq <= seq2; seq++) {
789N/A rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
789N/A rec->flags = (rec->flags & flag_mask) | u->add_flags;
789N/A }
789N/A } else {
789N/A for (seq = seq1; seq <= seq2; seq++) {
789N/A rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
789N/A
789N/A old_flags = rec->flags;
789N/A rec->flags = (rec->flags & flag_mask) | u->add_flags;
789N/A
789N/A mail_index_header_update_lowwaters(ctx, rec->uid,
789N/A rec->flags);
787N/A mail_index_sync_header_update_counts_all(ctx, rec->uid,
789N/A old_flags,
789N/A rec->flags);
789N/A }
789N/A }
789N/A return 1;
789N/A}
789N/A
789N/Astatic int sync_header_update(const struct mail_transaction_header_update *u,
789N/A struct mail_index_sync_map_ctx *ctx)
789N/A{
789N/A#define MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, field) \
789N/A ((u)->offset <= offsetof(struct mail_index_header, field) && \
789N/A (u)->offset + (u)->size > offsetof(struct mail_index_header, field))
1341N/A struct mail_index_map *map = ctx->view->map;
1341N/A uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
1341N/A uint32_t orig_next_uid = map->hdr.next_uid;
1341N/A
789N/A if (u->offset >= map->hdr.base_header_size ||
789N/A u->offset + u->size > map->hdr.base_header_size) {
789N/A mail_index_sync_set_corrupted(ctx,
789N/A "Header update outside range: %u + %u > %u",
789N/A u->offset, u->size, map->hdr.base_header_size);
789N/A return -1;
789N/A }
789N/A
789N/A buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
789N/A map->hdr_base = map->hdr_copy_buf->data;
789N/A i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
789N/A
789N/A /* @UNSAFE */
789N/A if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
789N/A memcpy(PTR_OFFSET(&map->hdr, u->offset),
789N/A u + 1, u->size);
789N/A } else if (u->offset < sizeof(map->hdr)) {
789N/A memcpy(PTR_OFFSET(&map->hdr, u->offset),
789N/A u + 1, sizeof(map->hdr) - u->offset);
789N/A }
789N/A
789N/A if (map->hdr.next_uid < orig_next_uid) {
789N/A /* next_uid update tried to shrink its value. this can happen
789N/A in some race conditions with e.g. with dsync, so just
789N/A silently ignore it. */
789N/A map->hdr.next_uid = orig_next_uid;
789N/A }
789N/A
789N/A /* the tail offset updates are intended for internal transaction
789N/A log handling. we'll update the offset in the header only when
789N/A the sync is finished. */
789N/A map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
1155N/A return 1;
1341N/A}
789N/A
789N/Astatic int
789N/Amail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx,
789N/A const struct mail_transaction_header *hdr,
789N/A const void *data)
789N/A{
789N/A uint64_t modseq;
789N/A int ret = 0;
787N/A
789N/A switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
789N/A case MAIL_TRANSACTION_APPEND: {
789N/A const struct mail_index_record *rec, *end;
789N/A
789N/A end = CONST_PTR_OFFSET(data, hdr->size);
789N/A for (rec = data; rec < end; rec++) {
789N/A ret = sync_append(rec, ctx);
789N/A if (ret <= 0)
789N/A break;
789N/A }
789N/A break;
789N/A }
789N/A case MAIL_TRANSACTION_EXPUNGE:
789N/A case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
789N/A const struct mail_transaction_expunge *rec = data, *end;
1341N/A ARRAY_TYPE(seq_range) seqs;
789N/A uint32_t seq1, seq2;
789N/A
789N/A if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
789N/A /* this is simply a request for expunge */
789N/A break;
789N/A }
789N/A t_array_init(&seqs, 64);
789N/A end = CONST_PTR_OFFSET(data, hdr->size);
789N/A for (; rec != end; rec++) {
789N/A if (mail_index_lookup_seq_range(ctx->view,
789N/A rec->uid1, rec->uid2, &seq1, &seq2))
789N/A seq_range_array_add_range(&seqs, seq1, seq2);
789N/A }
789N/A sync_expunge_range(ctx, &seqs);
789N/A break;
789N/A }
789N/A case MAIL_TRANSACTION_EXPUNGE_GUID:
789N/A case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
789N/A const struct mail_transaction_expunge_guid *rec = data, *end;
789N/A ARRAY_TYPE(seq_range) seqs;
789N/A uint32_t seq;
789N/A
1339N/A if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
1339N/A /* this is simply a request for expunge */
789N/A break;
789N/A }
789N/A t_array_init(&seqs, 64);
789N/A end = CONST_PTR_OFFSET(data, hdr->size);
789N/A for (; rec != end; rec++) {
789N/A i_assert(rec->uid != 0);
789N/A
789N/A if (mail_index_lookup_seq(ctx->view, rec->uid, &seq))
789N/A seq_range_array_add(&seqs, seq);
789N/A }
789N/A
1339N/A sync_expunge_range(ctx, &seqs);
1339N/A break;
789N/A }
789N/A case MAIL_TRANSACTION_FLAG_UPDATE: {
789N/A const struct mail_transaction_flag_update *rec, *end;
789N/A
787N/A end = CONST_PTR_OFFSET(data, hdr->size);
294N/A for (rec = data; rec < end; rec++) {
294N/A ret = sync_flag_update(rec, ctx);
294N/A if (ret <= 0)
789N/A break;
789N/A }
294N/A break;
789N/A }
789N/A case MAIL_TRANSACTION_HEADER_UPDATE: {
789N/A const struct mail_transaction_header_update *rec;
789N/A unsigned int i;
1339N/A
1339N/A for (i = 0; i < hdr->size; ) {
789N/A rec = CONST_PTR_OFFSET(data, i);
789N/A ret = sync_header_update(rec, ctx);
789N/A if (ret <= 0)
789N/A break;
789N/A
789N/A i += sizeof(*rec) + rec->size;
789N/A if ((i % 4) != 0)
789N/A i += 4 - (i % 4);
1338N/A }
1338N/A break;
1338N/A }
789N/A case MAIL_TRANSACTION_EXT_INTRO: {
789N/A const struct mail_transaction_ext_intro *rec = data;
789N/A unsigned int i;
789N/A uint32_t prev_seq;
789N/A uoff_t prev_offset;
789N/A
789N/A mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
789N/A &prev_seq, &prev_offset);
789N/A ctx->ext_intro_seq = prev_seq;
1338N/A ctx->ext_intro_offset = prev_offset;
1338N/A ctx->ext_intro_end_offset =
789N/A prev_offset + hdr->size + sizeof(*hdr);
789N/A
789N/A for (i = 0; i < hdr->size; ) {
789N/A if (i + sizeof(*rec) > hdr->size) {
789N/A /* should be just extra padding */
789N/A break;
789N/A }
789N/A
789N/A rec = CONST_PTR_OFFSET(data, i);
789N/A /* name_size checked by _log_view_next() */
789N/A i_assert(i + sizeof(*rec) + rec->name_size <= hdr->size);
789N/A
789N/A ret = mail_index_sync_ext_intro(ctx, rec);
789N/A if (ret <= 0)
789N/A break;
789N/A
789N/A i += sizeof(*rec) + rec->name_size;
789N/A if ((i % 4) != 0)
789N/A i += 4 - (i % 4);
873N/A }
873N/A break;
873N/A }
873N/A case MAIL_TRANSACTION_EXT_RESET: {
873N/A struct mail_transaction_ext_reset rec;
873N/A
873N/A /* old versions have only new_reset_id */
1086N/A if (hdr->size < sizeof(uint32_t)) {
1086N/A mail_index_sync_set_corrupted(ctx,
1086N/A "ext reset: invalid record size");
1339N/A ret = -1;
1339N/A break;
1086N/A }
789N/A memset(&rec, 0, sizeof(rec));
789N/A memcpy(&rec, data, I_MIN(hdr->size, sizeof(rec)));
789N/A ret = mail_index_sync_ext_reset(ctx, &rec);
789N/A break;
789N/A }
789N/A case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
1086N/A const struct mail_transaction_ext_hdr_update *rec;
789N/A unsigned int i;
789N/A
789N/A for (i = 0; i < hdr->size; ) {
857N/A rec = CONST_PTR_OFFSET(data, i);
857N/A
1086N/A if (i + sizeof(*rec) > hdr->size ||
857N/A i + sizeof(*rec) + rec->size > hdr->size) {
857N/A mail_index_sync_set_corrupted(ctx,
857N/A "ext hdr update: invalid record size");
857N/A ret = -1;
1339N/A break;
1339N/A }
857N/A
857N/A ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
857N/A rec->size, rec + 1);
857N/A if (ret <= 0)
857N/A break;
1339N/A
1339N/A i += sizeof(*rec) + rec->size;
857N/A if ((i % 4) != 0)
857N/A i += 4 - (i % 4);
869N/A }
868N/A break;
869N/A }
869N/A case MAIL_TRANSACTION_EXT_HDR_UPDATE32: {
869N/A const struct mail_transaction_ext_hdr_update32 *rec;
911N/A unsigned int i;
911N/A
869N/A for (i = 0; i < hdr->size; ) {
917N/A rec = CONST_PTR_OFFSET(data, i);
1339N/A
1339N/A if (i + sizeof(*rec) > hdr->size ||
1339N/A i + sizeof(*rec) + rec->size > hdr->size) {
1339N/A mail_index_sync_set_corrupted(ctx,
1339N/A "ext hdr update: invalid record size");
1339N/A ret = -1;
1339N/A break;
1339N/A }
1339N/A
1339N/A ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
1339N/A rec->size, rec + 1);
869N/A if (ret <= 0)
868N/A break;
1086N/A
1086N/A i += sizeof(*rec) + rec->size;
1086N/A if ((i % 4) != 0)
1339N/A i += 4 - (i % 4);
1339N/A }
1339N/A break;
1339N/A }
1339N/A case MAIL_TRANSACTION_EXT_REC_UPDATE: {
1339N/A const struct mail_transaction_ext_rec_update *rec;
1339N/A unsigned int i, record_size;
1086N/A
789N/A if (ctx->cur_ext_map_idx == (uint32_t)-1) {
294N/A mail_index_sync_set_corrupted(ctx,
294N/A "Extension record updated "
294N/A "without intro prefix");
789N/A ret = -1;
869N/A break;
877N/A }
789N/A
869N/A if (ctx->cur_ext_ignore) {
869N/A ret = 1;
869N/A break;
994N/A }
868N/A
1114N/A /* the record is padded to 32bits in the transaction log */
789N/A record_size = (sizeof(*rec) + ctx->cur_ext_record_size + 3) & ~3;
869N/A
911N/A for (i = 0; i < hdr->size; i += record_size) {
872N/A rec = CONST_PTR_OFFSET(data, i);
1336N/A
789N/A if (i + record_size > hdr->size) {
789N/A mail_index_sync_set_corrupted(ctx,
294N/A "ext rec update: invalid record size");
294N/A ret = -1;
294N/A break;
294N/A }
294N/A
294N/A ret = mail_index_sync_ext_rec_update(ctx, rec);
294N/A if (ret <= 0)
294N/A break;
294N/A }
294N/A break;
789N/A }
868N/A case MAIL_TRANSACTION_EXT_ATOMIC_INC: {
294N/A const struct mail_transaction_ext_atomic_inc *rec, *end;
789N/A
789N/A if (ctx->cur_ext_map_idx == (uint32_t)-1) {
294N/A mail_index_sync_set_corrupted(ctx,
294N/A "Extension record updated "
294N/A "without intro prefix");
294N/A ret = -1;
294N/A break;
869N/A }
294N/A
294N/A if (ctx->cur_ext_ignore) {
294N/A ret = 1;
787N/A break;
787N/A }
789N/A
789N/A end = CONST_PTR_OFFSET(data, hdr->size);
789N/A for (rec = data; rec < end; rec++) {
789N/A ret = mail_index_sync_ext_atomic_inc(ctx, rec);
789N/A if (ret <= 0)
789N/A break;
787N/A }
789N/A break;
789N/A }
789N/A case MAIL_TRANSACTION_KEYWORD_UPDATE: {
787N/A const struct mail_transaction_keyword_update *rec = data;
789N/A
789N/A ret = mail_index_sync_keywords(ctx, hdr, rec);
1339N/A break;
1339N/A }
789N/A case MAIL_TRANSACTION_KEYWORD_RESET: {
787N/A const struct mail_transaction_keyword_reset *rec = data;
789N/A
789N/A ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
789N/A break;
789N/A }
789N/A case MAIL_TRANSACTION_MODSEQ_UPDATE: {
789N/A const struct mail_transaction_modseq_update *rec = data;
789N/A
789N/A ret = sync_modseq_update(ctx, rec, hdr->size);
789N/A break;
787N/A }
789N/A case MAIL_TRANSACTION_INDEX_DELETED:
1308N/A if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
789N/A /* next sync finishes the deletion */
789N/A ctx->view->index->index_delete_requested = TRUE;
789N/A } else {
789N/A /* transaction log reading handles this */
787N/A }
789N/A break;
1339N/A case MAIL_TRANSACTION_INDEX_UNDELETED:
1339N/A ctx->view->index->index_delete_requested = FALSE;
1339N/A break;
1339N/A case MAIL_TRANSACTION_BOUNDARY:
789N/A break;
789N/A case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
789N/A modseq = mail_transaction_log_view_get_prev_modseq(ctx->view->log_view);
789N/A mail_index_modseq_update_highest(ctx->modseq_ctx, modseq);
787N/A break;
787N/A default:
1086N/A mail_index_sync_set_corrupted(ctx,
1086N/A "Unknown transaction record type 0x%x",
1086N/A (hdr->type & MAIL_TRANSACTION_TYPE_MASK));
1086N/A ret = -1;
1091N/A break;
1278N/A }
1086N/A return ret;
1091N/A}
1086N/A
1339N/Aint mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
1339N/A const struct mail_transaction_header *hdr,
1086N/A const void *data)
1086N/A{
1086N/A int ret;
1086N/A
1086N/A T_BEGIN {
294N/A ret = mail_index_sync_record_real(ctx, hdr, data);
294N/A } T_END;
294N/A return ret;
294N/A}
868N/A
294N/Avoid mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
294N/A struct mail_index_view *view,
294N/A enum mail_index_sync_handler_type type)
294N/A{
789N/A memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
789N/A sync_map_ctx->view = view;
789N/A sync_map_ctx->cur_ext_map_idx = (uint32_t)-1;
789N/A sync_map_ctx->type = type;
294N/A sync_map_ctx->modseq_ctx = mail_index_modseq_sync_begin(sync_map_ctx);
787N/A
787N/A mail_index_sync_init_handlers(sync_map_ctx);
787N/A}
1340N/A
1086N/Avoid mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
1086N/A{
1086N/A i_assert(sync_map_ctx->modseq_ctx == NULL);
1086N/A
1086N/A if (sync_map_ctx->unknown_extensions != NULL)
1340N/A buffer_free(&sync_map_ctx->unknown_extensions);
294N/A if (sync_map_ctx->expunge_handlers_used)
868N/A mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
868N/A mail_index_sync_deinit_handlers(sync_map_ctx);
789N/A}
789N/A
789N/Astatic void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
294N/A{
294N/A const struct mail_index_record *rec;
868N/A uint32_t seq;
868N/A
789N/A if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0)
789N/A return;
789N/A
789N/A /* do we have dirty flags anymore? */
789N/A for (seq = 1; seq <= map->rec_map->records_count; seq++) {
294N/A rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
868N/A if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
869N/A map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
869N/A break;
869N/A }
868N/A }
868N/A}
868N/A
868N/A#ifdef DEBUG
868N/Avoid mail_index_map_check(struct mail_index_map *map)
868N/A{
868N/A const struct mail_index_header *hdr = &map->hdr;
294N/A unsigned int del = 0, seen = 0;
294N/A uint32_t seq, prev_uid = 0;
868N/A
789N/A i_assert(hdr->messages_count <= map->rec_map->records_count);
294N/A for (seq = 1; seq <= hdr->messages_count; seq++) {
294N/A const struct mail_index_record *rec;
294N/A
789N/A rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
294N/A i_assert(rec->uid > prev_uid);
prev_uid = rec->uid;
if (rec->flags & MAIL_DELETED) {
i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
del++;
}
if (rec->flags & MAIL_SEEN)
seen++;
else
i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
}
i_assert(del == hdr->deleted_messages_count);
i_assert(seen == hdr->seen_messages_count);
}
#endif
int mail_index_sync_map(struct mail_index_map **_map,
enum mail_index_sync_handler_type type, bool force,
const char *sync_reason)
{
struct mail_index_map *map = *_map;
struct mail_index *index = map->index;
struct mail_index_view *view;
struct mail_index_sync_map_ctx sync_map_ctx;
const struct mail_transaction_header *thdr;
const void *tdata;
uint32_t prev_seq;
uoff_t start_offset, prev_offset;
const char *reason, *error;
int ret;
bool had_dirty, reset;
i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
if (index->log->head == NULL) {
i_assert(!force);
return 0;
}
start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
if (!force && (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0) {
/* see if we'd prefer to reopen the index file instead of
syncing the current map from the transaction log.
don't check this if mmap is disabled, because reopening
index causes sync to get lost. */
uoff_t log_size, index_size;
if (index->fd == -1 &&
index->log->head->hdr.prev_file_seq != 0) {
/* we don't know the index's size, so use the
smallest index size we're willing to read */
index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
} else {
index_size = map->hdr.header_size +
map->rec_map->records_count *
map->hdr.record_size;
}
/* this isn't necessary correct currently, but it should be
close enough */
log_size = index->log->head->last_size;
if (log_size > start_offset &&
log_size - start_offset > index_size)
return 0;
}
view = mail_index_view_open_with_map(index, map);
ret = mail_transaction_log_view_set(view->log_view,
map->hdr.log_file_seq, start_offset,
(uint32_t)-1, (uoff_t)-1,
&reset, &reason);
if (ret <= 0) {
mail_index_view_close(&view);
if (force && ret < 0) {
/* if we failed because of a syscall error, make sure
we return a failure. */
return -1;
}
if (force && ret == 0) {
/* the seq/offset is probably broken */
mail_index_set_error(index, "Index %s: Lost log for "
"seq=%u offset=%"PRIuUOFF_T": %s "
"(initial_mapped=%d, reason=%s)", index->filepath,
map->hdr.log_file_seq, start_offset, reason,
index->initial_mapped ? 1 : 0, sync_reason);
(void)mail_index_fsck(index);
}
/* can't use it. sync by re-reading index. */
return 0;
}
mail_transaction_log_get_head(index->log, &prev_seq, &prev_offset);
if (prev_seq != map->hdr.log_file_seq ||
prev_offset - map->hdr.log_file_tail_offset >
MAIL_INDEX_MIN_WRITE_BYTES) {
/* we're reading more from log than we would have preferred.
remember that we probably want to rewrite index soon. */
index->index_min_write = TRUE;
}
/* view referenced the map. avoid unnecessary map cloning by
unreferencing the map while view exists. */
map->refcount--;
had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
if (had_dirty)
map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
if (map->hdr_base != map->hdr_copy_buf->data) {
/* if syncing updates the header, it updates hdr_copy_buf
and updates hdr_base to hdr_copy_buf. so the buffer must
initially contain a valid header or we'll break it when
writing it. */
buffer_set_used_size(map->hdr_copy_buf, 0);
buffer_append(map->hdr_copy_buf, map->hdr_base,
map->hdr.header_size);
map->hdr_base = map->hdr_copy_buf->data;
}
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
mail_index_sync_map_init(&sync_map_ctx, view, type);
if (reset) {
/* Reset the entire index. Leave only indexid and
log_file_seq. */
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
map = mail_index_map_alloc(index);
if ((index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0)
map->hdr.flags |= MAIL_INDEX_HDR_FLAG_FSCKD;
map->hdr.log_file_seq = prev_seq;
map->hdr.log_file_tail_offset = 0;
mail_index_sync_replace_map(&sync_map_ctx, map);
}
map = NULL;
/* FIXME: when transaction sync lock is removed, we'll need to handle
the case when a transaction is committed while mailbox is being
synced ([synced transactions][new transaction][ext transaction]).
this means int_offset contains [synced] and ext_offset contains
all */
while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
&tdata)) > 0) {
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
if (LOG_IS_BEFORE(prev_seq, prev_offset,
view->map->hdr.log_file_seq,
view->map->hdr.log_file_head_offset)) {
/* this has been synced already. we're here only to call
expunge handlers and extension update handlers. */
i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
continue;
if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
continue;
}
/* we'll just skip over broken entries */
(void)mail_index_sync_record(&sync_map_ctx, thdr, tdata);
}
map = view->map;
if (had_dirty)
mail_index_sync_update_hdr_dirty_flag(map);
mail_index_modseq_sync_end(&sync_map_ctx.modseq_ctx);
mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);
#ifdef DEBUG
mail_index_map_check(map);
#endif
i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0);
/* transaction log tracks internally the current tail offset.
besides using header updates, it also updates the offset to skip
over following external transactions to avoid extra unneeded log
reading. */
i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq);
if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) {
map->hdr.log_file_tail_offset =
index->log->head->max_tail_offset;
}
buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
map->hdr_copy_buf->used);
}
/* restore refcount before closing the view. this is necessary also
if map got cloned, because view closing would otherwise destroy it */
map->refcount++;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(&view);
i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
if (mail_index_map_check_header(map, &error) <= 0) {
mail_index_set_error(index,
"Synchronization corrupted index header %s: %s",
index->filepath, error);
(void)mail_index_fsck(index);
map = index->map;
} else if (sync_map_ctx.errors) {
/* make sure the index looks valid now */
(void)mail_index_fsck(index);
map = index->map;
}
*_map = map;
return ret < 0 ? -1 : 1;
}