mail-index-sync-update.c revision f6d5c9fbdac9af5c4d3f467f828dc6f056309d5e
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher/* Copyright (c) 2004-2016 Dovecot authors, see the included COPYING file */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "lib.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "ioloop.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "array.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "mmap-util.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "mail-index-modseq.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "mail-index-view-private.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "mail-index-sync-private.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "mail-transaction-log.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#include "mail-transaction-log-private.h"
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher/* If we have less than this many bytes to sync from log file, don't bother
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher reading the main index */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic void
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghermail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_map *map, bool eol)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t prev_seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uoff_t prev_offset;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher &prev_seq, &prev_offset);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (prev_seq == 0) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* handling lost changes in view syncing */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (!eol) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (prev_offset == ctx->ext_intro_end_offset &&
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher prev_seq == ctx->ext_intro_seq) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* previous transaction was an extension introduction.
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher we probably came here from
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_ext_reset(). if there are any more
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher views which want to continue syncing it needs the
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher intro. so back up a bit more.
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher don't do this in case the last transaction in the
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher log is the extension intro, so we don't keep trying
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher to sync it over and over again. */
681742138b2afbbefa7f14de937beb438409208eSimo Sorce prev_offset = ctx->ext_intro_offset;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.log_file_seq = prev_seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher } else {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (map->hdr.log_file_seq != prev_seq) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.log_file_seq = prev_seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.log_file_tail_offset = 0;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce map->hdr.log_file_head_offset = prev_offset;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce}
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
fd555d130dc733509347fa096a2cb858b014a196Simo Sorcestatic void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_map *map)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_view *view = ctx->view;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i_assert(view->map != map);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_update_log_offset(ctx, view->map, FALSE);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_unmap(&view->map);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher view->map = map;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher view->index->map = map;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic struct mail_index_map *
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghermail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce{
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce struct mail_index_map *map = ctx->view->map;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce if (map->refcount > 1) {
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce map = mail_index_map_clone(map);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_replace_map(ctx, map);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce mail_index_map_move_to_memory(ctx->view->map);
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce return map;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce}
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
fd555d130dc733509347fa096a2cb858b014a196Simo Sorcestruct mail_index_map *
fd555d130dc733509347fa096a2cb858b014a196Simo Sorcemail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce{
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce (void)mail_index_sync_move_to_private_memory(ctx);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_record_map_move_to_private(ctx->view->map);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce return ctx->view->map;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic int
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghermail_index_header_update_counts(struct mail_index_header *hdr,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint8_t old_flags, uint8_t new_flags,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const char **error_r)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
681742138b2afbbefa7f14de937beb438409208eSimo Sorce /* different seen-flag */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((old_flags & MAIL_SEEN) != 0) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (hdr->seen_messages_count == 0) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher *error_r = "Seen counter wrong";
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return -1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher hdr->seen_messages_count--;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher } else {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (hdr->seen_messages_count >= hdr->messages_count) {
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce *error_r = "Seen counter wrong";
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce return -1;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce }
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (++hdr->seen_messages_count == hdr->messages_count)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher hdr->first_unseen_uid_lowwater = hdr->next_uid;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* different deleted-flag */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((old_flags & MAIL_DELETED) == 0) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher hdr->deleted_messages_count++;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (hdr->deleted_messages_count > hdr->messages_count) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher *error_r = "Deleted counter wrong";
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return -1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher } else {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (hdr->deleted_messages_count == 0 ||
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher hdr->deleted_messages_count > hdr->messages_count) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher *error_r = "Deleted counter wrong";
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce return -1;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce }
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce if (--hdr->deleted_messages_count == 0)
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce hdr->first_deleted_uid_lowwater = hdr->next_uid;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return 0;
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek}
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
fd555d130dc733509347fa096a2cb858b014a196Simo Sorcestatic void
fd555d130dc733509347fa096a2cb858b014a196Simo Sorcemail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce uint32_t uid,
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce uint8_t old_flags, uint8_t new_flags)
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce{
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce struct mail_index_map *const *maps;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce const char *error;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce unsigned int i, count;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher maps = array_get(&ctx->view->map->rec_map->maps, &count);
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce for (i = 0; i < count; i++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (uid >= maps[i]->hdr.next_uid)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher continue;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (mail_index_header_update_counts(&maps[i]->hdr,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher old_flags, new_flags,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher &error) < 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_set_corrupted(ctx, "%s", error);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic void
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghermail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t uid, uint8_t old_flags,
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher uint8_t new_flags)
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const char *error;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (uid >= ctx->view->map->hdr.next_uid) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_set_corrupted(ctx, "uid %u >= next_uid %u",
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uid, ctx->view->map->hdr.next_uid);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher } else {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (mail_index_header_update_counts(&ctx->view->map->hdr,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher old_flags, new_flags,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher &error) < 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_set_corrupted(ctx, "%s", error);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic void
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghermail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t uid, enum mail_flags flags)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek struct mail_index_map *const *maps;
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek unsigned int i, count;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek maps = array_get(&ctx->view->map->rec_map->maps, &count);
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek for (i = 0; i < count; i++) {
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek if ((flags & MAIL_SEEN) == 0 &&
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek uid < maps[i]->hdr.first_unseen_uid_lowwater)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher maps[i]->hdr.first_unseen_uid_lowwater = uid;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((flags & MAIL_DELETED) != 0 &&
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uid < maps[i]->hdr.first_deleted_uid_lowwater)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher maps[i]->hdr.first_deleted_uid_lowwater = uid;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic void
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghersync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq1, uint32_t seq2)
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_index_expunge_handler *eh;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_record *rec;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher array_foreach(&ctx->expunge_handlers, eh) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (seq = seq1; seq <= seq2; seq++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->map, seq);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* FIXME: does expunge handler's return value matter?
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher we probably shouldn't disallow expunges if the
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher handler returns failure.. should it be just changed
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher to return void? */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher (void)eh->handler(ctx, seq,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher PTR_OFFSET(rec, eh->record_offset),
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher eh->sync_context, eh->context);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic bool
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghersync_expunge_handlers_init(struct mail_index_sync_map_ctx *ctx)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* call expunge handlers only when syncing index file */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return FALSE;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (!ctx->expunge_handlers_set)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_init_expunge_handlers(ctx);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (!array_is_created(&ctx->expunge_handlers))
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return FALSE;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return TRUE;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic void
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghersync_expunge_range(struct mail_index_sync_map_ctx *ctx, const ARRAY_TYPE(seq_range) *seqs)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_map *map;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct seq_range *range;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher unsigned int i, count;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t dest_seq1, prev_seq2, orig_rec_count;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher range = array_get(seqs, &count);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (count == 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map = mail_index_sync_get_atomic_map(ctx);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* call the expunge handlers first */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (sync_expunge_handlers_init(ctx)) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (i = 0; i < count; i++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher sync_expunge_call_handlers(ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher range[i].seq1, range[i].seq2);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher prev_seq2 = 0;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher dest_seq1 = 1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher orig_rec_count = map->rec_map->records_count;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (i = 0; i < count; i++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq1 = range[i].seq1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq2 = range[i].seq2;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_record *rec;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq_count, seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i_assert(seq1 > prev_seq2);
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (seq = seq1; seq <= seq2; seq++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags, 0);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (prev_seq2+1 <= seq1-1) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* @UNSAFE: move (prev_seq2+1) .. (seq1-1) to its
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher final location in the map if necessary */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t move_count = (seq1-1) - (prev_seq2+1) + 1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (prev_seq2+1-1 != dest_seq1-1)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher memmove(MAIL_INDEX_REC_AT_SEQ(map, dest_seq1),
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher MAIL_INDEX_REC_AT_SEQ(map, prev_seq2+1),
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher move_count * map->hdr.record_size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher dest_seq1 += move_count;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher seq_count = seq2 - seq1 + 1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->rec_map->records_count -= seq_count;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.messages_count -= seq_count;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher prev_seq2 = seq2;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* Final stragglers */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (orig_rec_count > prev_seq2) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t final_move_count = orig_rec_count - prev_seq2;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher memmove(MAIL_INDEX_REC_AT_SEQ(map, dest_seq1),
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher MAIL_INDEX_REC_AT_SEQ(map, prev_seq2+1),
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher final_move_count * map->hdr.record_size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic void *sync_append_record(struct mail_index_map *map)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher size_t append_pos;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher void *ret;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher append_pos = map->rec_map->records_count * map->hdr.record_size;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.record_size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->rec_map->records =
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher buffer_get_modifiable_data(map->rec_map->buffer, NULL);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return ret;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic bool sync_update_ignored_change(struct mail_index_sync_map_ctx *ctx)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_transaction_commit_result *result =
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ctx->view->index->sync_commit_result;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t prev_log_seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uoff_t prev_log_offset, trans_start_offset, trans_end_offset;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (result == NULL)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return FALSE;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* we'll return TRUE if this modseq change was written within the
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher transaction that was just committed */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher &prev_log_seq, &prev_log_offset);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (prev_log_seq != result->log_file_seq)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return FALSE;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher trans_end_offset = result->log_file_offset;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher trans_start_offset = trans_end_offset - result->commit_size;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (prev_log_offset < trans_start_offset ||
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher prev_log_offset >= trans_end_offset)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return FALSE;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return TRUE;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic int
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghersync_modseq_update(struct mail_index_sync_map_ctx *ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_modseq_update *u,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher unsigned int size)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_view *view = ctx->view;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_modseq_update *end;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint64_t min_modseq, highest_modseq = 0;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher int ret;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher end = CONST_PTR_OFFSET(u, size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (; u < end; u++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (u->uid == 0)
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek seq = 0;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher else if (!mail_index_lookup_seq(view, u->uid, &seq))
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher continue;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher min_modseq = ((uint64_t)u->modseq_high32 << 32) |
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher u->modseq_low32;
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher if (highest_modseq < min_modseq)
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher highest_modseq = min_modseq;
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher ret = seq == 0 ? 1 :
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher mail_index_modseq_set(view, seq, min_modseq);
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher if (ret < 0) {
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher mail_index_sync_set_corrupted(ctx,
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher "modseqs updated before they were enabled");
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return -1;
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ret == 0 && sync_update_ignored_change(ctx))
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher view->index->sync_commit_result->ignored_modseq_changes++;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_modseq_update_highest(ctx->modseq_ctx, highest_modseq);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return 1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic int sync_append(const struct mail_index_record *rec,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_sync_map_ctx *ctx)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher struct mail_index_view *view = ctx->view;
2ca23577d3a25aead24ba759a1f6f67ffc24decfSimo Sorce struct mail_index_map *map = view->map;
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher const struct mail_index_record *old_rec;
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher enum mail_flags new_flags;
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher void *dest;
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher if (rec->uid < map->hdr.next_uid) {
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher mail_index_sync_set_corrupted(ctx,
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher "Append with UID %u, but next_uid = %u",
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher rec->uid, map->hdr.next_uid);
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher return -1;
e299638926171e0e92a36122aeff6611cd52418dStephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek /* move to memory. the mapping is written when unlocking so we don't
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek waste time re-mmap()ing multiple times or waste space growing index
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek file too large */
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek map = mail_index_sync_move_to_private_memory(ctx);
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (rec->uid <= map->rec_map->last_appended_uid) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i_assert(map->hdr.messages_count < map->rec_map->records_count);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* the flags may have changed since it was added to map.
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher use the updated flags already, so flag counters won't get
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher broken. */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher old_rec = MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i_assert(old_rec->uid == rec->uid);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher new_flags = old_rec->flags;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher } else {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* don't rely on buffer->used being at the correct position.
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher at least expunges can move it */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher dest = sync_append_record(map);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher memcpy(dest, rec, sizeof(*rec));
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.record_size - sizeof(*rec));
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->rec_map->records_count++;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->rec_map->last_appended_uid = rec->uid;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher new_flags = rec->flags;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_modseq_append(ctx->modseq_ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->rec_map->records_count);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.messages_count++;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.next_uid = rec->uid+1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_header_update_counts(ctx, rec->uid, 0, new_flags);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return 1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
681742138b2afbbefa7f14de937beb438409208eSimo Sorcestatic int sync_flag_update(const struct mail_transaction_flag_update *u,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_sync_map_ctx *ctx)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_view *view = ctx->view;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_record *rec;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint8_t flag_mask, old_flags;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq, seq1, seq2;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2))
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return 1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u)) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_modseq_update_flags(ctx->modseq_ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher u->add_flags | u->remove_flags,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher seq1, seq2);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher flag_mask = ~u->remove_flags;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (((u->add_flags | u->remove_flags) &
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek (MAIL_SEEN | MAIL_DELETED)) == 0) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* we're not modifying any counted/lowwatered flags */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (seq = seq1; seq <= seq2; seq++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec->flags = (rec->flags & flag_mask) | u->add_flags;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher } else {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (seq = seq1; seq <= seq2; seq++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher old_flags = rec->flags;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec->flags = (rec->flags & flag_mask) | u->add_flags;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_header_update_lowwaters(ctx, rec->uid,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec->flags);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_header_update_counts_all(ctx, rec->uid,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher old_flags,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec->flags);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return 1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic int sync_header_update(const struct mail_transaction_header_update *u,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_sync_map_ctx *ctx)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher#define MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, field) \
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ((u)->offset <= offsetof(struct mail_index_header, field) && \
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher (u)->offset + (u)->size > offsetof(struct mail_index_header, field))
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_index_map *map = ctx->view->map;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t orig_next_uid = map->hdr.next_uid;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (u->offset >= map->hdr.base_header_size ||
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher u->offset + u->size > map->hdr.base_header_size) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_set_corrupted(ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher "Header update outside range: %u + %u > %u",
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher u->offset, u->size, map->hdr.base_header_size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return -1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr_base = map->hdr_copy_buf->data;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* @UNSAFE */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher memcpy(PTR_OFFSET(&map->hdr, u->offset),
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher u + 1, u->size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher } else if (u->offset < sizeof(map->hdr)) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher memcpy(PTR_OFFSET(&map->hdr, u->offset),
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher u + 1, sizeof(map->hdr) - u->offset);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (map->hdr.next_uid < orig_next_uid) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* next_uid update tried to shrink its value. this can happen
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher in some race conditions with e.g. with dsync, so just
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher silently ignore it. */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.next_uid = orig_next_uid;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* the tail offset updates are intended for internal transaction
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher log handling. we'll update the offset in the header only when
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher the sync is finished. */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher return 1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher}
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagherstatic int
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallaghermail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_header *hdr,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const void *data)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher{
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint64_t modseq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher int ret = 0;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_APPEND: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_index_record *rec, *end;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher end = CONST_PTR_OFFSET(data, hdr->size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (rec = data; rec < end; rec++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = sync_append(rec, ctx);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ret <= 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXPUNGE:
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_expunge *rec = data, *end;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ARRAY_TYPE(seq_range) seqs;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq1, seq2;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* this is simply a request for expunge */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher t_array_init(&seqs, 64);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher end = CONST_PTR_OFFSET(data, hdr->size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (; rec != end; rec++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (mail_index_lookup_seq_range(ctx->view,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec->uid1, rec->uid2, &seq1, &seq2))
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher seq_range_array_add_range(&seqs, seq1, seq2);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher sync_expunge_range(ctx, &seqs);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXPUNGE_GUID:
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_expunge_guid *rec = data, *end;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ARRAY_TYPE(seq_range) seqs;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
d3d297c62e0340151da1d4ce1e082dcfcb45b431Jakub Hrozek if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* this is simply a request for expunge */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher t_array_init(&seqs, 64);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher end = CONST_PTR_OFFSET(data, hdr->size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (; rec != end; rec++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i_assert(rec->uid != 0);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (mail_index_lookup_seq(ctx->view, rec->uid, &seq))
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher seq_range_array_add(&seqs, seq);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher sync_expunge_range(ctx, &seqs);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
d3d297c62e0340151da1d4ce1e082dcfcb45b431Jakub Hrozek case MAIL_TRANSACTION_FLAG_UPDATE: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_flag_update *rec, *end;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher end = CONST_PTR_OFFSET(data, hdr->size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (rec = data; rec < end; rec++) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = sync_flag_update(rec, ctx);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ret <= 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_HEADER_UPDATE: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_header_update *rec;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher unsigned int i;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (i = 0; i < hdr->size; ) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec = CONST_PTR_OFFSET(data, i);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = sync_header_update(rec, ctx);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ret <= 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i += sizeof(*rec) + rec->size;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((i % 4) != 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i += 4 - (i % 4);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXT_INTRO: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_ext_intro *rec = data;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher unsigned int i;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uint32_t prev_seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher uoff_t prev_offset;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher &prev_seq, &prev_offset);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ctx->ext_intro_seq = prev_seq;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ctx->ext_intro_offset = prev_offset;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ctx->ext_intro_end_offset =
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher prev_offset + hdr->size + sizeof(*hdr);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (i = 0; i < hdr->size; ) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (i + sizeof(*rec) > hdr->size) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* should be just extra padding */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec = CONST_PTR_OFFSET(data, i);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher /* name_size checked by _log_view_next() */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i_assert(i + sizeof(*rec) + rec->name_size <= hdr->size);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = mail_index_sync_ext_intro(ctx, rec);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ret <= 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i += sizeof(*rec) + rec->name_size;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((i % 4) != 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i += 4 - (i % 4);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXT_RESET: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher struct mail_transaction_ext_reset rec;
ae765daed93b64b9b4177466a2191561b6734e63Stephen Gallagher
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek /* old versions have only new_reset_id */
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (hdr->size < sizeof(uint32_t)) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_set_corrupted(ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher "ext reset: invalid record size");
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = -1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher memset(&rec, 0, sizeof(rec));
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek memcpy(&rec, data, I_MIN(hdr->size, sizeof(rec)));
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek ret = mail_index_sync_ext_reset(ctx, &rec);
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek break;
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_ext_hdr_update *rec;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher unsigned int i;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek for (i = 0; i < hdr->size; ) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec = CONST_PTR_OFFSET(data, i);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (i + sizeof(*rec) > hdr->size ||
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i + sizeof(*rec) + rec->size > hdr->size) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_set_corrupted(ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher "ext hdr update: invalid record size");
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = -1;
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec->size, rec + 1);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ret <= 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i += sizeof(*rec) + rec->size;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((i % 4) != 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i += 4 - (i % 4);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXT_HDR_UPDATE32: {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher const struct mail_transaction_ext_hdr_update32 *rec;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher unsigned int i;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher for (i = 0; i < hdr->size; ) {
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek rec = CONST_PTR_OFFSET(data, i);
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek if (i + sizeof(*rec) > hdr->size ||
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek i + sizeof(*rec) + rec->size > hdr->size) {
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher mail_index_sync_set_corrupted(ctx,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher "ext hdr update: invalid record size");
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = -1;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher rec->size, rec + 1);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if (ret <= 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
3a59cbd0b7b9c5dd3c62ac1679876070c264d80fMichal Zidek
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i += sizeof(*rec) + rec->size;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher if ((i % 4) != 0)
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher i += 4 - (i % 4);
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher break;
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher }
4c1bf6607060cea867fccf667063c028dfd51e96Stephen Gallagher case MAIL_TRANSACTION_EXT_REC_UPDATE: {
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher const struct mail_transaction_ext_rec_update *rec;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher const struct mail_index_ext *ext;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher unsigned int i, record_size;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher
681742138b2afbbefa7f14de937beb438409208eSimo Sorce if (ctx->cur_ext_map_idx == (uint32_t)-1) {
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher mail_index_sync_set_corrupted(ctx,
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher "Extension record updated "
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher "without intro prefix");
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher ret = -1;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher break;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce }
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce if (ctx->cur_ext_ignore) {
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher ret = 1;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher break;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher }
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher ext = array_idx(&ctx->view->map->extensions,
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher ctx->cur_ext_map_idx);
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek /* the record is padded to 32bits in the transaction log */
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce record_size = (sizeof(*rec) + ext->record_size + 3) & ~3;
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce for (i = 0; i < hdr->size; i += record_size) {
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce rec = CONST_PTR_OFFSET(data, i);
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce if (i + record_size > hdr->size) {
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce mail_index_sync_set_corrupted(ctx,
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce "ext rec update: invalid record size");
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce ret = -1;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher break;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher }
fd555d130dc733509347fa096a2cb858b014a196Simo Sorce
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher ret = mail_index_sync_ext_rec_update(ctx, rec);
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher if (ret <= 0)
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher break;
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher }
1e7c355a2d36eb7b942b5111c96eb2a2285d49ccStephen Gallagher break;
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek }
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek case MAIL_TRANSACTION_EXT_ATOMIC_INC: {
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek const struct mail_transaction_ext_atomic_inc *rec, *end;
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek if (ctx->cur_ext_map_idx == (uint32_t)-1) {
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek mail_index_sync_set_corrupted(ctx,
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek "Extension record updated "
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek "without intro prefix");
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek ret = -1;
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek break;
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek }
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek if (ctx->cur_ext_ignore) {
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek ret = 1;
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek break;
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek }
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek end = CONST_PTR_OFFSET(data, hdr->size);
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek for (rec = data; rec < end; rec++) {
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek ret = mail_index_sync_ext_atomic_inc(ctx, rec);
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek if (ret <= 0)
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek break;
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek }
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek break;
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek }
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek case MAIL_TRANSACTION_KEYWORD_UPDATE: {
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek const struct mail_transaction_keyword_update *rec = data;
eb29ae58117ca88868491fe2240e27393c7a9068Jakub Hrozek
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce ret = mail_index_sync_keywords(ctx, hdr, rec);
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce break;
681742138b2afbbefa7f14de937beb438409208eSimo Sorce }
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce case MAIL_TRANSACTION_KEYWORD_RESET: {
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce const struct mail_transaction_keyword_reset *rec = data;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce break;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce }
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce case MAIL_TRANSACTION_MODSEQ_UPDATE: {
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce const struct mail_transaction_modseq_update *rec = data;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce ret = sync_modseq_update(ctx, rec, hdr->size);
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce break;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce }
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce case MAIL_TRANSACTION_INDEX_DELETED:
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce /* next sync finishes the deletion */
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek ctx->view->index->index_delete_requested = TRUE;
681742138b2afbbefa7f14de937beb438409208eSimo Sorce } else {
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce /* transaction log reading handles this */
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce }
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce break;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce case MAIL_TRANSACTION_INDEX_UNDELETED:
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce ctx->view->index->index_delete_requested = FALSE;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce break;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce case MAIL_TRANSACTION_BOUNDARY:
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce break;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce modseq = mail_transaction_log_view_get_prev_modseq(ctx->view->log_view);
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce mail_index_modseq_update_highest(ctx->modseq_ctx, modseq);
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce break;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce default:
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce mail_index_sync_set_corrupted(ctx,
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce "Unknown transaction record type 0x%x",
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce (hdr->type & MAIL_TRANSACTION_TYPE_MASK));
4fcc50e133f90cd4c5931a3ac48c84cb628b16fcMichal Zidek ret = -1;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce break;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce }
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce return ret;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce}
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorceint mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce const struct mail_transaction_header *hdr,
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce const void *data)
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce{
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce int ret;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce T_BEGIN {
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce ret = mail_index_sync_record_real(ctx, hdr, data);
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce } T_END;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce return ret;
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce}
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorcevoid mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
d0483eefc41ac295ed4c56e08ad76ca7b5fb3b2cSimo Sorce struct mail_index_view *view,
enum mail_index_sync_handler_type type)
{
memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
sync_map_ctx->view = view;
sync_map_ctx->cur_ext_map_idx = (uint32_t)-1;
sync_map_ctx->type = type;
sync_map_ctx->modseq_ctx = mail_index_modseq_sync_begin(sync_map_ctx);
mail_index_sync_init_handlers(sync_map_ctx);
}
void mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
{
i_assert(sync_map_ctx->modseq_ctx == NULL);
if (sync_map_ctx->unknown_extensions != NULL)
buffer_free(&sync_map_ctx->unknown_extensions);
if (sync_map_ctx->expunge_handlers_used)
mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
mail_index_sync_deinit_handlers(sync_map_ctx);
}
static void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
{
const struct mail_index_record *rec;
uint32_t seq;
if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0)
return;
/* do we have dirty flags anymore? */
for (seq = 1; seq <= map->rec_map->records_count; seq++) {
rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
break;
}
}
}
#ifdef DEBUG
void mail_index_map_check(struct mail_index_map *map)
{
const struct mail_index_header *hdr = &map->hdr;
unsigned int del = 0, seen = 0;
uint32_t seq, prev_uid = 0;
i_assert(hdr->messages_count <= map->rec_map->records_count);
for (seq = 1; seq <= hdr->messages_count; seq++) {
const struct mail_index_record *rec;
rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
i_assert(rec->uid > prev_uid);
prev_uid = rec->uid;
if (rec->flags & MAIL_DELETED) {
i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
del++;
}
if (rec->flags & MAIL_SEEN)
seen++;
else
i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
}
i_assert(del == hdr->deleted_messages_count);
i_assert(seen == hdr->seen_messages_count);
}
#endif
int mail_index_sync_map(struct mail_index_map **_map,
enum mail_index_sync_handler_type type, bool force)
{
struct mail_index_map *map = *_map;
struct mail_index *index = map->index;
struct mail_index_view *view;
struct mail_index_sync_map_ctx sync_map_ctx;
const struct mail_transaction_header *thdr;
const void *tdata;
uint32_t prev_seq;
uoff_t start_offset, prev_offset;
const char *reason, *error;
int ret;
bool had_dirty, reset;
i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
if (index->log->head == NULL) {
i_assert(!force);
return 0;
}
start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
if (!force && (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0) {
/* see if we'd prefer to reopen the index file instead of
syncing the current map from the transaction log.
don't check this if mmap is disabled, because reopening
index causes sync to get lost. */
uoff_t log_size, index_size;
if (index->fd == -1 &&
index->log->head->hdr.prev_file_seq != 0) {
/* we don't know the index's size, so use the
smallest index size we're willing to read */
index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
} else {
index_size = map->hdr.header_size +
map->rec_map->records_count *
map->hdr.record_size;
}
/* this isn't necessary correct currently, but it should be
close enough */
log_size = index->log->head->last_size;
if (log_size > start_offset &&
log_size - start_offset > index_size)
return 0;
}
view = mail_index_view_open_with_map(index, map);
ret = mail_transaction_log_view_set(view->log_view,
map->hdr.log_file_seq, start_offset,
(uint32_t)-1, (uoff_t)-1,
&reset, &reason);
if (ret <= 0) {
mail_index_view_close(&view);
if (force && ret == 0) {
/* the seq/offset is probably broken */
mail_index_set_error(index, "Index %s: Lost log for "
"seq=%u offset=%"PRIuUOFF_T": %s", index->filepath,
map->hdr.log_file_seq, start_offset, reason);
(void)mail_index_fsck(index);
}
/* can't use it. sync by re-reading index. */
return 0;
}
mail_transaction_log_get_head(index->log, &prev_seq, &prev_offset);
if (prev_seq != map->hdr.log_file_seq ||
prev_offset - map->hdr.log_file_tail_offset >
MAIL_INDEX_MIN_WRITE_BYTES) {
/* we're reading more from log than we would have preferred.
remember that we probably want to rewrite index soon. */
index->index_min_write = TRUE;
}
/* view referenced the map. avoid unnecessary map cloning by
unreferencing the map while view exists. */
map->refcount--;
had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
if (had_dirty)
map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
if (map->hdr_base != map->hdr_copy_buf->data) {
/* if syncing updates the header, it updates hdr_copy_buf
and updates hdr_base to hdr_copy_buf. so the buffer must
initially contain a valid header or we'll break it when
writing it. */
buffer_set_used_size(map->hdr_copy_buf, 0);
buffer_append(map->hdr_copy_buf, map->hdr_base,
map->hdr.header_size);
map->hdr_base = map->hdr_copy_buf->data;
}
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
mail_index_sync_map_init(&sync_map_ctx, view, type);
if (reset) {
/* Reset the entire index. Leave only indexid and
log_file_seq. */
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
map = mail_index_map_alloc(index);
map->hdr.log_file_seq = prev_seq;
map->hdr.log_file_tail_offset = 0;
mail_index_sync_replace_map(&sync_map_ctx, map);
}
map = NULL;
/* FIXME: when transaction sync lock is removed, we'll need to handle
the case when a transaction is committed while mailbox is being
synced ([synced transactions][new transaction][ext transaction]).
this means int_offset contains [synced] and ext_offset contains
all */
while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
&tdata)) > 0) {
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
if (LOG_IS_BEFORE(prev_seq, prev_offset,
view->map->hdr.log_file_seq,
view->map->hdr.log_file_head_offset)) {
/* this has been synced already. we're here only to call
expunge handlers and extension update handlers. */
i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
continue;
if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
continue;
}
/* we'll just skip over broken entries */
(void)mail_index_sync_record(&sync_map_ctx, thdr, tdata);
}
map = view->map;
if (had_dirty)
mail_index_sync_update_hdr_dirty_flag(map);
mail_index_modseq_sync_end(&sync_map_ctx.modseq_ctx);
mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);
#ifdef DEBUG
mail_index_map_check(map);
#endif
i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0);
/* transaction log tracks internally the current tail offset.
besides using header updates, it also updates the offset to skip
over following external transactions to avoid extra unneeded log
reading. */
i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq);
if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) {
map->hdr.log_file_tail_offset =
index->log->head->max_tail_offset;
}
buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
map->hdr_copy_buf->used);
}
/* restore refcount before closing the view. this is necessary also
if map got cloned, because view closing would otherwise destroy it */
map->refcount++;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(&view);
i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
if (mail_index_map_check_header(map, &error) <= 0) {
mail_index_set_error(index,
"Synchronization corrupted index header %s: %s",
index->filepath, error);
(void)mail_index_fsck(index);
map = index->map;
} else if (sync_map_ctx.errors) {
/* make sure the index looks valid now */
(void)mail_index_fsck(index);
map = index->map;
}
*_map = map;
return ret < 0 ? -1 : 1;
}