mail-index-sync-update.c revision 23079bf0a6e7489c5f542b0b897a71bdfd884a51
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher/* Copyright (c) 2004-2013 Dovecot authors, see the included COPYING file */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher#include "lib.h"
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher#include "ioloop.h"
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher#include "array.h"
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek#include "mmap-util.h"
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek#include "mail-index-modseq.h"
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek#include "mail-index-view-private.h"
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek#include "mail-index-sync-private.h"
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek#include "mail-transaction-log.h"
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher#include "mail-transaction-log-private.h"
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher/* If we have less than this many bytes to sync from log file, don't bother
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher reading the main index */
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozekstatic void
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozekmail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek struct mail_index_map *map, bool eol)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t prev_seq;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uoff_t prev_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
7a14e8f66c0e932fe2954d792614a3b61d444bd1Jakub Hrozek mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher &prev_seq, &prev_offset);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (prev_seq == 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* handling lost changes in view syncing */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return;
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek }
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek if (!eol) {
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek if (prev_offset == ctx->ext_intro_end_offset &&
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek prev_seq == ctx->ext_intro_seq) {
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher /* previous transaction was an extension introduction.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher we probably came here from
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek mail_index_sync_ext_reset(). if there are any more
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher views which want to continue syncing it needs the
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher intro. so back up a bit more.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher don't do this in case the last transaction in the
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher log is the extension intro, so we don't keep trying
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher to sync it over and over again. */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher prev_offset = ctx->ext_intro_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_seq = prev_seq;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher } else {
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek if (map->hdr.log_file_seq != prev_seq) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_seq = prev_seq;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_tail_offset = 0;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_head_offset = prev_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_map *map)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_view *view = ctx->view;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(view->map != map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_update_log_offset(ctx, view->map, FALSE);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_unmap(&view->map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher view->map = map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher view->index->map = map;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek}
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozekstatic struct mail_index_map *
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozekmail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek{
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher struct mail_index_map *map = ctx->view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (map->refcount > 1) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map = mail_index_map_clone(map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_replace_map(ctx, map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_map_move_to_memory(ctx->view->map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return map;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstruct mail_index_map *
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozekmail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher (void)mail_index_sync_move_to_private_memory(ctx);
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek mail_index_record_map_move_to_private(ctx->view->map);
ea929f1b022fc2cb77dec89b0e12accef983ec85Jakub Hrozek mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return ctx->view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic int
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_header_update_counts(struct mail_index_header *hdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint8_t old_flags, uint8_t new_flags,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const char **error_r)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* different seen-flag */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((old_flags & MAIL_SEEN) != 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (hdr->seen_messages_count == 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher *error_r = "Seen counter wrong";
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher hdr->seen_messages_count--;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher } else {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (hdr->seen_messages_count >= hdr->messages_count) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher *error_r = "Seen counter wrong";
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (++hdr->seen_messages_count == hdr->messages_count)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher hdr->first_unseen_uid_lowwater = hdr->next_uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* different deleted-flag */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((old_flags & MAIL_DELETED) == 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher hdr->deleted_messages_count++;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (hdr->deleted_messages_count > hdr->messages_count) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher *error_r = "Deleted counter wrong";
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher } else {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (hdr->deleted_messages_count == 0 ||
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher hdr->deleted_messages_count > hdr->messages_count) {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek *error_r = "Deleted counter wrong";
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (--hdr->deleted_messages_count == 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher hdr->first_deleted_uid_lowwater = hdr->next_uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return 0;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozekstatic void
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t uid,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint8_t old_flags, uint8_t new_flags)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_map *const *maps;
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek const char *error;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek unsigned int i, count;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher maps = array_get(&ctx->view->map->rec_map->maps, &count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < count; i++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (uid >= maps[i]->hdr.next_uid)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek continue;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if (mail_index_header_update_counts(&maps[i]->hdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher old_flags, new_flags,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher &error) < 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx, "%s", error);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic void
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghermail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t uid, uint8_t old_flags,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint8_t new_flags, bool all)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const char *error;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (all) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_header_update_counts_all(ctx, uid, old_flags,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher new_flags);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek } else if (uid >= ctx->view->map->hdr.next_uid) {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek mail_index_sync_set_corrupted(ctx, "uid %u >= next_uid %u",
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uid, ctx->view->map->hdr.next_uid);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher } else {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (mail_index_header_update_counts(&ctx->view->map->hdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher old_flags, new_flags,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher &error) < 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx, "%s", error);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic void
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozekmail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek uint32_t uid, enum mail_flags flags)
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek{
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek struct mail_index_map *const *maps;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek unsigned int i, count;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher maps = array_get(&ctx->view->map->rec_map->maps, &count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < count; i++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((flags & MAIL_SEEN) == 0 &&
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uid < maps[i]->hdr.first_unseen_uid_lowwater)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher maps[i]->hdr.first_unseen_uid_lowwater = uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((flags & MAIL_DELETED) != 0 &&
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uid < maps[i]->hdr.first_deleted_uid_lowwater)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher maps[i]->hdr.first_deleted_uid_lowwater = uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic void
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghersync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t seq1, uint32_t seq2)
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_index_expunge_handler *eh;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek struct mail_index_record *rec;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek uint32_t seq;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* call expunge handlers only when syncing index file */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE)
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher return;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (!ctx->expunge_handlers_set)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_init_expunge_handlers(ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek if (!array_is_created(&ctx->expunge_handlers))
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek return;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher array_foreach(&ctx->expunge_handlers, eh) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (seq = seq1; seq <= seq2; seq++) {
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher rec = MAIL_INDEX_MAP_IDX(ctx->view->map, seq-1);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* FIXME: does expunge handler's return value matter?
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher we probably shouldn't disallow expunges if the
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher handler returns failure.. should it be just changed
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher to return void? */
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek (void)eh->handler(ctx, seq,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek PTR_OFFSET(rec, eh->record_offset),
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher eh->sync_context, eh->context);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher }
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek}
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozekstatic void
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozeksync_expunge(struct mail_index_sync_map_ctx *ctx, uint32_t uid1, uint32_t uid2)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek{
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek struct mail_index_map *map;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek struct mail_index_record *rec;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek uint32_t seq_count, seq, seq1, seq2;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (!mail_index_lookup_seq_range(ctx->view, uid1, uid2, &seq1, &seq2)) {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek /* everything expunged already */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek return;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek }
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek sync_expunge_call_handlers(ctx, seq1, seq2);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map = mail_index_sync_get_atomic_map(ctx);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek for (seq = seq1; seq <= seq2; seq++) {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek rec = MAIL_INDEX_MAP_IDX(map, seq-1);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek 0, FALSE);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek }
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek /* @UNSAFE */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek memmove(MAIL_INDEX_MAP_IDX(map, seq1-1),
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek MAIL_INDEX_MAP_IDX(map, seq2),
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek (map->rec_map->records_count - seq2) * map->hdr.record_size);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek seq_count = seq2 - seq1 + 1;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map->rec_map->records_count -= seq_count;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map->hdr.messages_count -= seq_count;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek}
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozekstatic void *sync_append_record(struct mail_index_map *map)
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek{
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek size_t append_pos;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek void *ret;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek append_pos = map->rec_map->records_count * map->hdr.record_size;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek ret = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map->hdr.record_size);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map->rec_map->records =
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek buffer_get_modifiable_data(map->rec_map->buffer, NULL);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek return ret;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek}
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozekstatic bool sync_update_ignored_change(struct mail_index_sync_map_ctx *ctx)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek{
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek struct mail_index_transaction_commit_result *result =
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek ctx->view->index->sync_commit_result;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek uint32_t prev_log_seq;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek uoff_t prev_log_offset, trans_start_offset, trans_end_offset;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (result == NULL)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek return FALSE;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek /* we'll return TRUE if this modseq change was written within the
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek transaction that was just committed */
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek &prev_log_seq, &prev_log_offset);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (prev_log_seq != result->log_file_seq)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek return FALSE;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek trans_end_offset = result->log_file_offset;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek trans_start_offset = trans_end_offset - result->commit_size;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek if (prev_log_offset < trans_start_offset ||
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek prev_log_offset >= trans_end_offset)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek return FALSE;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek return TRUE;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek}
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozekstatic int
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozeksync_modseq_update(struct mail_index_sync_map_ctx *ctx,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek const struct mail_transaction_modseq_update *u,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek unsigned int size)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek{
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek struct mail_index_view *view = ctx->view;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek const struct mail_transaction_modseq_update *end;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek uint32_t seq;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek uint64_t min_modseq, highest_modseq = 0;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek int ret;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek end = CONST_PTR_OFFSET(u, size);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek for (; u < end; u++) {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (u->uid == 0)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek seq = 0;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek else if (!mail_index_lookup_seq(view, u->uid, &seq))
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher continue;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher min_modseq = ((uint64_t)u->modseq_high32 >> 32) |
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher u->modseq_low32;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (highest_modseq < min_modseq)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher highest_modseq = min_modseq;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = seq == 0 ? 1 :
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_modseq_set(view, seq, min_modseq);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek if (ret < 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek "modseqs updated before they were enabled");
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (ret == 0 && sync_update_ignored_change(ctx))
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher view->index->sync_commit_result->ignored_modseq_changes++;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_modseq_update_highest(ctx->modseq_ctx, highest_modseq);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek return 1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic int sync_append(const struct mail_index_record *rec,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_sync_map_ctx *ctx)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek struct mail_index_view *view = ctx->view;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_map *map = view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_index_record *old_rec;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek enum mail_flags new_flags;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher void *dest;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (rec->uid < map->hdr.next_uid) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek "Append with UID %u, but next_uid = %u",
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher rec->uid, map->hdr.next_uid);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek /* move to memory. the mapping is written when unlocking so we don't
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher waste time re-mmap()ing multiple times or waste space growing index
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher file too large */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map = mail_index_sync_move_to_private_memory(ctx);
056302a92862fda16351d7192600746746f38e5dStephen Gallagher
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher if (rec->uid <= map->rec_map->last_appended_uid) {
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher i_assert(map->hdr.messages_count < map->rec_map->records_count);
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher /* the flags may have changed since it was added to map.
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek use the updated flags already, so flag counters won't get
056302a92862fda16351d7192600746746f38e5dStephen Gallagher broken. */
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek old_rec = MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek i_assert(old_rec->uid == rec->uid);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek new_flags = old_rec->flags;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek } else {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* don't rely on buffer->used being at the correct position.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher at least expunges can move it */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek dest = sync_append_record(map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher memcpy(dest, rec, sizeof(*rec));
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.record_size - sizeof(*rec));
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->rec_map->records_count++;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map->rec_map->last_appended_uid = rec->uid;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher new_flags = rec->flags;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_modseq_append(ctx->modseq_ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->rec_map->records_count);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek map->hdr.messages_count++;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.next_uid = rec->uid+1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_header_update_counts(ctx, rec->uid,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek 0, new_flags, FALSE);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return 1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic int sync_flag_update(const struct mail_transaction_flag_update *u,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek struct mail_index_sync_map_ctx *ctx)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_view *view = ctx->view;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_record *rec;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint8_t flag_mask, old_flags;
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek uint32_t idx, seq1, seq2;
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2))
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return 1;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u)) {
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek mail_index_modseq_update_flags(ctx->modseq_ctx,
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek u->add_flags | u->remove_flags,
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek seq1, seq2);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher flag_mask = ~u->remove_flags;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (((u->add_flags | u->remove_flags) &
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher (MAIL_SEEN | MAIL_DELETED)) == 0) {
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher /* we're not modifying any counted/lowwatered flags */
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher for (idx = seq1-1; idx < seq2; idx++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher rec = MAIL_INDEX_MAP_IDX(view->map, idx);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek rec->flags = (rec->flags & flag_mask) | u->add_flags;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher } else {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (idx = seq1-1; idx < seq2; idx++) {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek rec = MAIL_INDEX_MAP_IDX(view->map, idx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek old_flags = rec->flags;
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek rec->flags = (rec->flags & flag_mask) | u->add_flags;
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_header_update_lowwaters(ctx, rec->uid,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek rec->flags);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek mail_index_sync_header_update_counts(ctx, rec->uid,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek old_flags,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek rec->flags, TRUE);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher }
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek return 1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic int sync_header_update(const struct mail_transaction_header_update *u,
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek struct mail_index_sync_map_ctx *ctx)
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher#define MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, field) \
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek ((u)->offset <= offsetof(struct mail_index_header, field) && \
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek (u)->offset + (u)->size > offsetof(struct mail_index_header, field))
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek struct mail_index_map *map = ctx->view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek uint32_t orig_next_uid = map->hdr.next_uid;
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek if (u->offset >= map->hdr.base_header_size ||
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek u->offset + u->size > map->hdr.base_header_size) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx,
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek "Header update outside range: %u + %u > %u",
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek u->offset, u->size, map->hdr.base_header_size);
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek return -1;
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek }
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek map->hdr_base = map->hdr_copy_buf->data;
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek /* @UNSAFE */
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek memcpy(PTR_OFFSET(&map->hdr, u->offset),
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek u + 1, u->size);
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek } else if (u->offset < sizeof(map->hdr)) {
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek memcpy(PTR_OFFSET(&map->hdr, u->offset),
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek u + 1, sizeof(map->hdr) - u->offset);
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek }
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek if (map->hdr.next_uid < orig_next_uid) {
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek /* next_uid update tried to shrink its value. this can happen
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek in some race conditions with e.g. with dsync, so just
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek silently ignore it. */
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek map->hdr.next_uid = orig_next_uid;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek }
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek /* the tail offset updates are intended for internal transaction
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek log handling. we'll update the offset in the header only when
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek the sync is finished. */
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek return 1;
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek}
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozekstatic int
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozekmail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx,
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek const struct mail_transaction_header *hdr,
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek const void *data)
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek{
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek uint64_t modseq;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek int ret = 0;
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek case MAIL_TRANSACTION_APPEND: {
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek const struct mail_index_record *rec, *end;
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek
0172959f117b545c8a6b1893f5f56818d82dd624Jakub Hrozek end = CONST_PTR_OFFSET(data, hdr->size);
b20208b80e99abb79c00d5ec526caa9465859c52Jakub Hrozek for (rec = data; rec < end; rec++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = sync_append(rec, ctx);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (ret <= 0)
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher break;
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher }
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher break;
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek case MAIL_TRANSACTION_EXPUNGE:
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_expunge *rec = data, *end;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* this is simply a request for expunge */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher end = CONST_PTR_OFFSET(data, hdr->size);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek for (; rec != end; rec++)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher sync_expunge(ctx, rec->uid1, rec->uid2);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher case MAIL_TRANSACTION_EXPUNGE_GUID:
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_expunge_guid *rec = data, *end;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher ARRAY_TYPE(seq_range) uids;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek const struct seq_range *range;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek unsigned int i, count;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* this is simply a request for expunge */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher t_array_init(&uids, 64);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher end = CONST_PTR_OFFSET(data, hdr->size);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (; rec != end; rec++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(rec->uid != 0);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher seq_range_array_add(&uids, rec->uid);
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* do this in reverse so the memmove()s are smaller */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek range = array_get(&uids, &count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = count; i > 0; i--)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher sync_expunge(ctx, range[i-1].seq1, range[i-1].seq2);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek case MAIL_TRANSACTION_FLAG_UPDATE: {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek const struct mail_transaction_flag_update *rec, *end;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher end = CONST_PTR_OFFSET(data, hdr->size);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek for (rec = data; rec < end; rec++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = sync_flag_update(rec, ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (ret <= 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek case MAIL_TRANSACTION_HEADER_UPDATE: {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_header_update *rec;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < hdr->size; ) {
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher rec = CONST_PTR_OFFSET(data, i);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = sync_header_update(rec, ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (ret <= 0)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i += sizeof(*rec) + rec->size;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((i % 4) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i += 4 - (i % 4);
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek }
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher case MAIL_TRANSACTION_EXT_INTRO: {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek const struct mail_transaction_ext_intro *rec = data;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t prev_seq;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uoff_t prev_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek &prev_seq, &prev_offset);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek ctx->ext_intro_seq = prev_seq;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ctx->ext_intro_offset = prev_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ctx->ext_intro_end_offset =
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek prev_offset + hdr->size + sizeof(*hdr);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < hdr->size; ) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (i + sizeof(*rec) > hdr->size) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* should be just extra padding */
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher rec = CONST_PTR_OFFSET(data, i);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek /* name_size checked by _log_view_next() */
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek i_assert(i + sizeof(*rec) + rec->name_size <= hdr->size);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek ret = mail_index_sync_ext_intro(ctx, rec);
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek if (ret <= 0)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek break;
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek i += sizeof(*rec) + rec->name_size;
e5c33e0bd03a2deb8e5011deeb3ae93f960910eeJakub Hrozek if ((i % 4) != 0)
e5c33e0bd03a2deb8e5011deeb3ae93f960910eeJakub Hrozek i += 4 - (i % 4);
e5c33e0bd03a2deb8e5011deeb3ae93f960910eeJakub Hrozek }
e5c33e0bd03a2deb8e5011deeb3ae93f960910eeJakub Hrozek break;
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek }
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek case MAIL_TRANSACTION_EXT_RESET: {
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek struct mail_transaction_ext_reset rec;
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek /* old versions have only new_reset_id */
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if (hdr->size < sizeof(uint32_t)) {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek mail_index_sync_set_corrupted(ctx,
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek "ext reset: invalid record size");
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek ret = -1;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek break;
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek }
e5c33e0bd03a2deb8e5011deeb3ae93f960910eeJakub Hrozek memset(&rec, 0, sizeof(rec));
e5c33e0bd03a2deb8e5011deeb3ae93f960910eeJakub Hrozek memcpy(&rec, data, I_MIN(hdr->size, sizeof(rec)));
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek ret = mail_index_sync_ext_reset(ctx, &rec);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek break;
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek }
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek const struct mail_transaction_ext_hdr_update *rec;
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek unsigned int i;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek for (i = 0; i < hdr->size; ) {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek rec = CONST_PTR_OFFSET(data, i);
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek if (i + sizeof(*rec) > hdr->size ||
524ceecc11f3d458eb3c1cf1489c3ff6ccb22226Jakub Hrozek i + sizeof(*rec) + rec->size > hdr->size) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek "ext hdr update: invalid record size");
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher rec->size, rec + 1);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (ret <= 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek i += sizeof(*rec) + rec->size;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if ((i % 4) != 0)
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek i += 4 - (i % 4);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher case MAIL_TRANSACTION_EXT_HDR_UPDATE32: {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek const struct mail_transaction_ext_hdr_update32 *rec;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher unsigned int i;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < hdr->size; ) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher rec = CONST_PTR_OFFSET(data, i);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (i + sizeof(*rec) > hdr->size ||
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i + sizeof(*rec) + rec->size > hdr->size) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "ext hdr update: invalid record size");
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek rec->size, rec + 1);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (ret <= 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i += sizeof(*rec) + rec->size;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if ((i % 4) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i += 4 - (i % 4);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher case MAIL_TRANSACTION_EXT_REC_UPDATE: {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_ext_rec_update *rec;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_index_ext *ext;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek unsigned int i, record_size;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if (ctx->cur_ext_map_idx == (uint32_t)-1) {
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher mail_index_sync_set_corrupted(ctx,
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher "Extension record updated "
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek "without intro prefix");
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher ret = -1;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher break;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher }
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher if (ctx->cur_ext_ignore) {
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher ret = 1;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek ext = array_idx(&ctx->view->map->extensions,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek ctx->cur_ext_map_idx);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek /* the record is padded to 32bits in the transaction log */
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher record_size = (sizeof(*rec) + ext->record_size + 3) & ~3;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek for (i = 0; i < hdr->size; i += record_size) {
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher rec = CONST_PTR_OFFSET(data, i);
7a14e8f66c0e932fe2954d792614a3b61d444bd1Jakub Hrozek
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher if (i + record_size > hdr->size) {
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher mail_index_sync_set_corrupted(ctx,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek "ext rec update: invalid record size");
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher ret = -1;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek break;
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher }
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek ret = mail_index_sync_ext_rec_update(ctx, rec);
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher if (ret <= 0)
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher break;
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher }
7797e361155f7ce937085fd98e360469d7baf1b6Jakub Hrozek break;
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek case MAIL_TRANSACTION_EXT_ATOMIC_INC: {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek const struct mail_transaction_ext_atomic_inc *rec, *end;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if (ctx->cur_ext_map_idx == (uint32_t)-1) {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek mail_index_sync_set_corrupted(ctx,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek "Extension record updated "
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek "without intro prefix");
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek ret = -1;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if (ctx->cur_ext_ignore) {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek ret = 1;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek end = CONST_PTR_OFFSET(data, hdr->size);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek for (rec = data; rec < end; rec++) {
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek ret = mail_index_sync_ext_atomic_inc(ctx, rec);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if (ret <= 0)
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek case MAIL_TRANSACTION_KEYWORD_UPDATE: {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek const struct mail_transaction_keyword_update *rec = data;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek ret = mail_index_sync_keywords(ctx, hdr, rec);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek break;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek case MAIL_TRANSACTION_KEYWORD_RESET: {
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher const struct mail_transaction_keyword_reset *rec = data;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek break;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek }
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek case MAIL_TRANSACTION_MODSEQ_UPDATE: {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek const struct mail_transaction_modseq_update *rec = data;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek ret = sync_modseq_update(ctx, rec, hdr->size);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek break;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek }
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek case MAIL_TRANSACTION_INDEX_DELETED:
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek /* next sync finishes the deletion */
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek ctx->view->index->index_delete_requested = TRUE;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek } else {
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek /* transaction log reading handles this */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek case MAIL_TRANSACTION_INDEX_UNDELETED:
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ctx->view->index->index_delete_requested = FALSE;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher case MAIL_TRANSACTION_BOUNDARY:
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher modseq = mail_transaction_log_view_get_prev_modseq(ctx->view->log_view);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_modseq_update_highest(ctx->modseq_ctx, modseq);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher default:
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_set_corrupted(ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "Unknown transaction record type 0x%x",
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher (hdr->type & MAIL_TRANSACTION_TYPE_MASK));
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek ret = -1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return ret;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherint mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_transaction_header *hdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const void *data)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek int ret;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher T_BEGIN {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher ret = mail_index_sync_record_real(ctx, hdr, data);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek } T_END;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return ret;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghervoid mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek struct mail_index_view *view,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher enum mail_index_sync_handler_type type)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher sync_map_ctx->view = view;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek sync_map_ctx->cur_ext_map_idx = (uint32_t)-1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher sync_map_ctx->type = type;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher sync_map_ctx->modseq_ctx = mail_index_modseq_sync_begin(sync_map_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_init_handlers(sync_map_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghervoid mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(sync_map_ctx->modseq_ctx == NULL);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek if (sync_map_ctx->unknown_extensions != NULL)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher buffer_free(&sync_map_ctx->unknown_extensions);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (sync_map_ctx->expunge_handlers_used)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_deinit_handlers(sync_map_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek{
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek const struct mail_index_record *rec;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek unsigned int i;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0)
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek return;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* do we have dirty flags anymore? */
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek for (i = 0; i < map->rec_map->records_count; i++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher rec = MAIL_INDEX_MAP_IDX(map, i);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher break;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher#ifdef DEBUG
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallaghervoid mail_index_map_check(struct mail_index_map *map)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct mail_index_header *hdr = &map->hdr;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek unsigned int i, del = 0, seen = 0;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t prev_uid = 0;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(hdr->messages_count <= map->rec_map->records_count);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher for (i = 0; i < hdr->messages_count; i++) {
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek const struct mail_index_record *rec;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher rec = MAIL_INDEX_MAP_IDX(map, i);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(rec->uid > prev_uid);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher prev_uid = rec->uid;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (rec->flags & MAIL_DELETED) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher del++;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (rec->flags & MAIL_SEEN)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher seen++;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher else
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(del == hdr->deleted_messages_count);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek i_assert(seen == hdr->seen_messages_count);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher#endif
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherint mail_index_sync_map(struct mail_index_map **_map,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher enum mail_index_sync_handler_type type, bool force)
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek{
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_map *map = *_map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index *index = map->index;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_view *view;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher struct mail_index_sync_map_ctx sync_map_ctx;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek const struct mail_transaction_header *thdr;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const void *tdata;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uint32_t prev_seq;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uoff_t start_offset, prev_offset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher int ret;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher bool had_dirty, reset;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek if (index->log->head == NULL) {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek i_assert(!force);
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek return 0;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher if (!force && (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* see if we'd prefer to reopen the index file instead of
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher syncing the current map from the transaction log.
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek don't check this if mmap is disabled, because reopening
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher index causes sync to get lost. */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher uoff_t log_size, index_size;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (index->fd == -1 &&
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek index->log->head->hdr.prev_file_seq != 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* we don't know the index's size, so use the
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher smallest index size we're willing to read */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek } else {
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek index_size = map->hdr.header_size +
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->rec_map->records_count *
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek map->hdr.record_size;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek }
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek /* this isn't necessary correct currently, but it should be
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek close enough */
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek log_size = index->log->head->last_size;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek if (log_size > start_offset &&
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek log_size - start_offset > index_size)
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek return 0;
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek }
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek view = mail_index_view_open_with_map(index, map);
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek ret = mail_transaction_log_view_set(view->log_view,
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek map->hdr.log_file_seq, start_offset,
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek (uint32_t)-1, (uoff_t)-1, &reset);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek if (ret <= 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_view_close(&view);
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek if (force && ret == 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* the seq/offset is probably broken */
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek mail_index_set_error(index, "Index %s: Lost log for "
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek "seq=%u offset=%"PRIuUOFF_T, index->filepath,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_seq, start_offset);
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek (void)mail_index_fsck(index);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek }
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek /* can't use it. sync by re-reading index. */
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek return 0;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek }
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek mail_transaction_log_get_head(index->log, &prev_seq, &prev_offset);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek if (prev_seq != map->hdr.log_file_seq ||
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek prev_offset - map->hdr.log_file_tail_offset >
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek MAIL_INDEX_MIN_WRITE_BYTES) {
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek /* we're reading more from log than we would have preferred.
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek remember that we probably want to rewrite index soon. */
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek index->index_min_write = TRUE;
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek }
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek /* view referenced the map. avoid unnecessary map cloning by
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek unreferencing the map while view exists. */
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher map->refcount--;
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher
7a14e8f66c0e932fe2954d792614a3b61d444bd1Jakub Hrozek had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
7a14e8f66c0e932fe2954d792614a3b61d444bd1Jakub Hrozek if (had_dirty)
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek if (map->hdr_base != map->hdr_copy_buf->data) {
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek /* if syncing updates the header, it updates hdr_copy_buf
6463ed1dcdd45416468b3fa178bd856b5a9ed2c3Jakub Hrozek and updates hdr_base to hdr_copy_buf. so the buffer must
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek initially contain a valid header or we'll break it when
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek writing it. */
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek buffer_reset(map->hdr_copy_buf);
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek buffer_append(map->hdr_copy_buf, map->hdr_base,
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map->hdr.header_size);
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek map->hdr_base = map->hdr_copy_buf->data;
a23014d69b56cbdf48ad05229c334648b5309d8fJakub Hrozek }
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek mail_transaction_log_view_get_prev_pos(view->log_view,
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek &prev_seq, &prev_offset);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek mail_index_sync_map_init(&sync_map_ctx, view, type);
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek if (reset) {
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek /* Reset the entire index. Leave only indexid and
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek log_file_seq. */
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek mail_transaction_log_view_get_prev_pos(view->log_view,
7a14e8f66c0e932fe2954d792614a3b61d444bd1Jakub Hrozek &prev_seq, &prev_offset);
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek map = mail_index_map_alloc(index);
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek map->hdr.log_file_seq = prev_seq;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek map->hdr.log_file_tail_offset = 0;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_replace_map(&sync_map_ctx, map);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map = NULL;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek /* FIXME: when transaction sync lock is removed, we'll need to handle
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher the case when a transaction is committed while mailbox is being
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher synced ([synced transactions][new transaction][ext transaction]).
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher this means int_offset contains [synced] and ext_offset contains
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek all */
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher &tdata)) > 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_transaction_log_view_get_prev_pos(view->log_view,
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek &prev_seq, &prev_offset);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (LOG_IS_BEFORE(prev_seq, prev_offset,
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher view->map->hdr.log_file_seq,
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek view->map->hdr.log_file_head_offset)) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* this has been synced already. we're here only to call
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher expunge handlers and extension update handlers. */
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher continue;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher continue;
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek }
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* we'll just skip over broken entries */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher (void)mail_index_sync_record(&sync_map_ctx, thdr, tdata);
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map = view->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (had_dirty)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_update_hdr_dirty_flag(map);
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher mail_index_modseq_sync_end(&sync_map_ctx.modseq_ctx);
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek#ifdef DEBUG
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_map_check(map);
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher#endif
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0);
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher /* transaction log tracks internally the current tail offset.
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher besides using header updates, it also updates the offset to skip
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek over following external transactions to avoid extra unneeded log
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher reading. */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr.log_file_tail_offset =
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher index->log->head->max_tail_offset;
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher }
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->hdr_copy_buf->used);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher /* restore refcount before closing the view. this is necessary also
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher if map got cloned, because view closing would otherwise destroy it */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map->refcount++;
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek mail_index_sync_map_deinit(&sync_map_ctx);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_view_close(&view);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek if (mail_index_map_check_header(map) <= 0) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher mail_index_set_error(index,
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher "Synchronization corrupted index header: %s",
2cb6f28b3a12bb714bf14494d31eb6b6fff64b8bJakub Hrozek index->filepath);
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher (void)mail_index_fsck(index);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map = index->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher } else if (sync_map_ctx.errors) {
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek /* make sure the index looks valid now */
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher (void)mail_index_fsck(index);
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher map = index->map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher }
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher
a7797068c4deb6ce2bdbcda27c45ff1bbb4a8e78Jakub Hrozek *_map = map;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher return ret < 0 ? -1 : 1;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher}
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher