mail-index-sync-update.c revision ccffbed92cb02c24fd717808a84138240bf1885b
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "ioloop.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "array.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "buffer.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "file-set-size.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mmap-util.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-view-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-sync-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-transaction-log.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-transaction-log-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-transaction-util.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic void
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_map *map, bool eol)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t prev_seq;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uoff_t prev_offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &prev_seq, &prev_offset);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (prev_offset == ctx->ext_intro_offset + ctx->ext_intro_size &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen prev_seq == ctx->ext_intro_seq && !eol) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* previous transaction was an extension introduction.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen we probably came here from mail_index_sync_ext_reset().
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if there are any more views which want to continue syncing
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it needs the intro. so back up a bit more.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen don't do this in case the last transaction in the log is
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen the extension intro, so we don't keep trying to sync it
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen over and over again. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen prev_offset = ctx->ext_intro_offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (!ctx->sync_only_external) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(prev_offset >= map->hdr.log_file_int_offset ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen prev_seq > map->hdr.log_file_seq);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr.log_file_int_offset = prev_offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else if (map->hdr.log_file_seq != prev_seq) {
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen /* log sequence changed. update internal offset to
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen beginning of the new file. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(map->hdr.log_file_int_offset ==
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->view->index->log->head->hdr.prev_file_offset);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr.log_file_int_offset =
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->view->index->log->head->hdr.hdr_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* we might be in the middle of syncing internal transactions, with
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen some of the following external transactions already synced. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(prev_seq > map->hdr.log_file_seq ||
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen prev_offset >= map->hdr.log_file_ext_offset ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (!eol && !ctx->sync_only_external));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (map->hdr.log_file_seq != prev_seq ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen prev_offset > map->hdr.log_file_ext_offset) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr.log_file_seq = prev_seq;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr.log_file_ext_offset = prev_offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_index_map_msync(struct mail_index *index, struct mail_index_map *map)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int base_size;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen return 0;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen base_size = I_MIN(map->hdr.base_header_size, sizeof(map->hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen map->mmap_used_size = index->hdr->header_size +
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen map->records_count * map->hdr.record_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen memcpy(map->mmap_base, &map->hdr, base_size);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen memcpy(PTR_OFFSET(map->mmap_base, base_size),
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen CONST_PTR_OFFSET(map->hdr_base, base_size),
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen map->hdr.header_size - base_size);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen mail_index_set_syscall_error(index, "msync()");
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainenvoid mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen struct mail_index_map *map)
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen{
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen struct mail_index_view *view = ctx->view;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen struct mail_index_map *old_map = view->map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen i_assert(view->map != map);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen old_map->refcount++;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen /* if map still exists after this, it's only in views. */
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen view->map->write_to_disk = FALSE;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen /* keywords aren't parsed for the new map yet */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen view->map->keywords_read = FALSE;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen mail_index_unmap(view->index, &view->map);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen view->map = map;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if ((ctx->type & (MAIL_INDEX_SYNC_HANDLER_FILE |
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen MAIL_INDEX_SYNC_HANDLER_HEAD)) != 0 &&
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen view->index->map != map) {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen mail_index_unmap(view->index, &view->index->map);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen view->index->map = map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->index->hdr = &map->hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->refcount++;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->type == MAIL_INDEX_SYNC_HANDLER_FILE) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen map->write_to_disk = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->write_atomic = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen /* some views may still use the same mapping, and since we could have
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen already updated the records, make sure we leave the header in a
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen valid state as well */
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen mail_index_sync_update_log_offset(ctx, old_map, FALSE);
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen (void)mail_index_map_msync(view->index, old_map);
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen mail_index_unmap(view->index, &old_map);
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen i_assert(view->hdr.messages_count == map->hdr.messages_count);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainenstatic int
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainenmail_index_header_update_counts(struct mail_index_header *hdr,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint8_t old_flags, uint8_t new_flags,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char **error_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
37e6cf44d61a81c6839e3ab76234b54309d8d292Timo Sirainen if (((old_flags ^ new_flags) & MAIL_RECENT) != 0) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen /* different recent-flag */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if ((old_flags & MAIL_RECENT) == 0) {
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen hdr->recent_messages_count++;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (hdr->recent_messages_count > hdr->messages_count) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen *error_r = "Recent counter wrong";
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (hdr->recent_messages_count == 0 ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr->recent_messages_count > hdr->messages_count) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen *error_r = "Recent counter wrong";
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen return -1;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (--hdr->recent_messages_count == 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr->first_recent_uid_lowwater = hdr->next_uid;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* different seen-flag */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if ((old_flags & MAIL_SEEN) != 0) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (hdr->seen_messages_count == 0) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen *error_r = "Seen counter wrong";
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen hdr->seen_messages_count--;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen } else {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (hdr->seen_messages_count >= hdr->messages_count) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen *error_r = "Seen counter wrong";
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen return -1;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (++hdr->seen_messages_count == hdr->messages_count)
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen hdr->first_unseen_uid_lowwater = hdr->next_uid;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* different deleted-flag */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if ((old_flags & MAIL_DELETED) == 0) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen hdr->deleted_messages_count++;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (hdr->deleted_messages_count > hdr->messages_count) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen *error_r = "Deleted counter wrong";
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen return -1;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen } else {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (hdr->deleted_messages_count == 0 ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr->deleted_messages_count > hdr->messages_count) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen *error_r = "Deleted counter wrong";
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (--hdr->deleted_messages_count == 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr->first_deleted_uid_lowwater = hdr->next_uid;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return 0;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen}
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainenvoid mail_index_view_recalc_counters(struct mail_index_view *view)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen{
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen struct mail_index_map *map = view->map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mail_index_record *rec;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *error;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int i;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr.recent_messages_count = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr.seen_messages_count = 0;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen map->hdr.deleted_messages_count = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (i = 0; i < view->hdr.messages_count; i++) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen rec = MAIL_INDEX_MAP_IDX(map, i);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_header_update_counts(&map->hdr, 0, rec->flags,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &error) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_panic("mail_index_view_recalc_counters(): %s", error);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->hdr.recent_messages_count = map->hdr.recent_messages_count;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen view->hdr.seen_messages_count = map->hdr.seen_messages_count;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen view->hdr.deleted_messages_count = map->hdr.deleted_messages_count;
d565eaa943f29a49b97230ced57eec40ee65b4f9Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen view->broken_counters = FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic void
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_index_header_update_lowwaters(struct mail_index_header *hdr,
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen const struct mail_index_record *rec)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if ((rec->flags & MAIL_RECENT) != 0 &&
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen rec->uid < hdr->first_recent_uid_lowwater)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr->first_recent_uid_lowwater = rec->uid;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if ((rec->flags & MAIL_SEEN) == 0 &&
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen rec->uid < hdr->first_unseen_uid_lowwater)
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen hdr->first_unseen_uid_lowwater = rec->uid;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if ((rec->flags & MAIL_DELETED) != 0 &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rec->uid < hdr->first_deleted_uid_lowwater)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen hdr->first_deleted_uid_lowwater = rec->uid;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainenstatic int sync_expunge(const struct mail_transaction_expunge *e,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_sync_map_ctx *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_view *view = ctx->view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_map *map = view->map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_record *rec;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen const char *error;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen uint32_t count, seq, seq1, seq2;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen const struct mail_index_expunge_handler *expunge_handlers, *eh;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen unsigned int i, expunge_handlers_count;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (e->uid1 > e->uid2 || e->uid1 == 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_index_sync_set_corrupted(ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "Invalid UID range in expunge (%u .. %u)",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen e->uid1, e->uid2);
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (!view->map->write_to_disk || view->map->refcount != 1) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* expunges have to be atomic. so we'll have to copy
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the mapping, do the changes there and then finally
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen replace the whole index file. to avoid extra disk
befeac661293b8d4206118ac50b8be9751df8424Timo Sirainen I/O we copy the index into memory rather than to
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen temporary file */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen map = mail_index_map_clone(map, map->hdr.record_size);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_index_sync_replace_map(ctx, map);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map));
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* we want atomic rename()ing */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen map->write_atomic = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_lookup_uid_range(view, e->uid1, e->uid2,
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen &seq1, &seq2) < 0)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen return -1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen if (seq1 == 0)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen return 1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen /* call expunge handlers only when syncing index file */
69bd816e46fdee6182d0cb2e4c6be32399a555c8Timo Sirainen if (ctx->type == MAIL_INDEX_SYNC_HANDLER_FILE &&
69bd816e46fdee6182d0cb2e4c6be32399a555c8Timo Sirainen !ctx->expunge_handlers_set)
69bd816e46fdee6182d0cb2e4c6be32399a555c8Timo Sirainen mail_index_sync_init_expunge_handlers(ctx);
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->type == MAIL_INDEX_SYNC_HANDLER_FILE &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen array_is_created(&ctx->expunge_handlers)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen expunge_handlers = array_get(&ctx->expunge_handlers,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &expunge_handlers_count);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen } else {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen expunge_handlers = NULL;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen expunge_handlers_count = 0;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->unreliable_flags || view->broken_counters)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->broken_counters = TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (seq = seq1; seq <= seq2; seq++) {
0cea9b1f4fa0495a48f5f097e40492517d67e1baTimo Sirainen rec = MAIL_INDEX_MAP_IDX(map, seq-1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (mail_index_header_update_counts(&map->hdr,
0cea9b1f4fa0495a48f5f097e40492517d67e1baTimo Sirainen rec->flags, 0,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &error) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_sync_set_corrupted(ctx, "%s", error);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen for (i = 0; i < expunge_handlers_count; i++) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen eh = &expunge_handlers[i];
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen for (seq = seq1; seq <= seq2; seq++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rec = MAIL_INDEX_MAP_IDX(map, seq-1);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (eh->handler(ctx, seq,
ba482d3624ca4f1b3d638e6e8470ba5134f21493Timo Sirainen PTR_OFFSET(rec, eh->record_offset),
ba482d3624ca4f1b3d638e6e8470ba5134f21493Timo Sirainen eh->sync_context, eh->context) < 0)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen return -1;
21455709020274a628faa9b9bd7839cb8efe3c73Timo Sirainen }
21455709020274a628faa9b9bd7839cb8efe3c73Timo Sirainen }
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen /* @UNSAFE */
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen count = seq2 - seq1 + 1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen memmove(MAIL_INDEX_MAP_IDX(map, seq1-1), MAIL_INDEX_MAP_IDX(map, seq2),
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen (map->records_count - seq2) * map->hdr.record_size);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen map->records_count -= count;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen map->hdr.messages_count -= count;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen view->hdr.messages_count -= count;
21455709020274a628faa9b9bd7839cb8efe3c73Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen if (map->buffer != NULL) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen buffer_set_used_size(map->buffer, map->records_count *
21455709020274a628faa9b9bd7839cb8efe3c73Timo Sirainen map->hdr.record_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->records = buffer_get_modifiable_data(map->buffer, NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return 1;
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainen}
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainen
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainenstatic void write_seq_update(struct mail_index_map *map,
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainen uint32_t seq1, uint32_t seq2)
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainen{
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (map->write_seq_first == 0 ||
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen map->write_seq_first > seq1)
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen map->write_seq_first = seq1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (map->write_seq_last < seq2)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen map->write_seq_last = seq2;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen}
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int sync_append(const struct mail_index_record *rec,
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen struct mail_index_sync_map_ctx *ctx)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen{
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen struct mail_index_view *view = ctx->view;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen struct mail_index_map *map = view->map;
21455709020274a628faa9b9bd7839cb8efe3c73Timo Sirainen const char *error;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen void *dest;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (rec->uid < map->hdr.next_uid) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen mail_index_sync_set_corrupted(ctx,
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen "Append with UID %u, but next_uid = %u",
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen rec->uid, map->hdr.next_uid);
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen return -1;
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen }
114a0f74e0f825c6bd8aeadfafb248a030762a1fTimo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen i_assert(map->records_count * map->hdr.record_size ==
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen buffer_get_used_size(map->buffer));
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen dest = buffer_append_space_unsafe(map->buffer,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen map->hdr.record_size);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen map->records = buffer_get_modifiable_data(map->buffer, NULL);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen } else {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen i_assert((map->records_count+1) * map->hdr.record_size <=
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen map->mmap_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen dest = MAIL_INDEX_MAP_IDX(map, map->records_count);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memcpy(dest, rec, sizeof(*rec));
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr.record_size - sizeof(*rec));
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen map->hdr.messages_count++;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen map->hdr.next_uid = rec->uid+1;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen map->records_count++;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen view->hdr.messages_count++;
089756ceca55966f1a70e8591f6086026e51015cTimo Sirainen view->hdr.next_uid = rec->uid+1;
089756ceca55966f1a70e8591f6086026e51015cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen write_seq_update(map, map->hdr.messages_count, map->hdr.messages_count);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_header_update_lowwaters(&map->hdr, rec);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (!view->broken_counters) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (mail_index_header_update_counts(&map->hdr, 0, rec->flags,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen &error) < 0) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen mail_index_sync_set_corrupted(ctx, "%s", error);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenstatic int sync_flag_update(const struct mail_transaction_flag_update *u,
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen struct mail_index_sync_map_ctx *ctx)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen{
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen struct mail_index_view *view = ctx->view;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen struct mail_index_header *hdr;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen struct mail_index_record *rec;
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen const char *error;
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen uint8_t flag_mask, old_flags;
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen uint32_t idx, seq1, seq2;
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen if (u->uid1 > u->uid2 || u->uid1 == 0) {
62cfc346eb7b0a4fd9e1ab6edd63b98711161229Timo Sirainen mail_index_sync_set_corrupted(ctx,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen "Invalid UID range in flag update (%u .. %u)",
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen u->uid1, u->uid2);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen return -1;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (mail_index_lookup_uid_range(view, u->uid1, u->uid2,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen &seq1, &seq2) < 0)
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen return -1;
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainen
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainen if (seq1 == 0)
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainen return 1;
a53cb86b4d733d9c48ee4d285bed477c80825804Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen write_seq_update(view->map, seq1, seq2);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr = &view->map->hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen flag_mask = ~u->remove_flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (((u->add_flags | u->remove_flags) &
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (MAIL_SEEN | MAIL_DELETED | MAIL_RECENT)) == 0) {
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen /* we're not modifying any counted/lowwatered flags */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (idx = seq1-1; idx < seq2; idx++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen rec = MAIL_INDEX_MAP_IDX(view->map, idx);
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else if (view->broken_counters || ctx->unreliable_flags) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->broken_counters = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (idx = seq1-1; idx < seq2; idx++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rec = MAIL_INDEX_MAP_IDX(view->map, idx);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen mail_index_header_update_lowwaters(hdr, rec);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen }
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen } else {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen for (idx = seq1-1; idx < seq2; idx++) {
58febed28f2af78b2d8a281c851d9b67160c4bd3Timo Sirainen rec = MAIL_INDEX_MAP_IDX(view->map, idx);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen old_flags = rec->flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
mail_index_header_update_lowwaters(hdr, rec);
if (mail_index_header_update_counts(hdr, old_flags,
rec->flags,
&error) < 0) {
mail_index_sync_set_corrupted(ctx, "%s", error);
return -1;
}
}
}
return 1;
}
static int sync_header_update(const struct mail_transaction_header_update *u,
struct mail_index_sync_map_ctx *ctx)
{
struct mail_index_map *map = ctx->view->map;
if (u->offset >= map->hdr.base_header_size ||
u->offset + u->size > map->hdr.base_header_size) {
mail_index_sync_set_corrupted(ctx,
"Header update outside range: %u + %u > %u",
u->offset, u->size, map->hdr.base_header_size);
return -1;
}
buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
map->hdr_base = map->hdr_copy_buf->data;
/* @UNSAFE */
if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
memcpy(PTR_OFFSET(&map->hdr, u->offset),
u + 1, u->size);
} else if (u->offset < sizeof(map->hdr)) {
memcpy(PTR_OFFSET(&map->hdr, u->offset),
u + 1, sizeof(map->hdr) - u->offset);
}
return 1;
}
static int mail_index_grow(struct mail_index *index, struct mail_index_map *map,
unsigned int count)
{
void *hdr_copy;
size_t size, hdr_copy_size;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
return 1;
i_assert(map == index->map);
i_assert(!index->mapping); /* mail_index_sync_from_transactions() */
size = map->hdr.header_size +
(map->records_count + count) * map->hdr.record_size;
if (size <= map->mmap_size)
return 1;
/* when we grow fast, do it exponentially */
if (count < index->last_grow_count)
count = index->last_grow_count;
if (count < MAIL_INDEX_MAX_POWER_GROW)
count = nearest_power(count);
index->last_grow_count = count;
size = map->hdr.header_size +
(map->records_count + count) * map->hdr.record_size;
if (file_set_size(index->fd, (off_t)size) < 0) {
mail_index_set_syscall_error(index, "file_set_size()");
return !ENOSPACE(errno) ? -1 :
mail_index_move_to_memory(index);
}
/* we only wish to grow the file, but mail_index_map() updates the
headers as well and may break our modified hdr_copy. so, take
a backup of it and put it back afterwards */
t_push();
i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
hdr_copy_size = map->hdr_copy_buf->used;
hdr_copy = t_malloc(hdr_copy_size);
memcpy(hdr_copy, map->hdr_copy_buf->data, hdr_copy_size);
memcpy(hdr_copy, &map->hdr, sizeof(map->hdr));
if (mail_index_map(index, TRUE) <= 0) {
t_pop();
return -1;
}
map = index->map;
buffer_reset(map->hdr_copy_buf);
buffer_append(map->hdr_copy_buf, hdr_copy, hdr_copy_size);
map->hdr_base = map->hdr_copy_buf->data;
memcpy(&map->hdr, hdr_copy, sizeof(map->hdr));
map->records_count = map->hdr.messages_count;
i_assert(map->mmap_size >= size);
t_pop();
return 1;
}
static void
mail_index_update_day_headers(struct mail_index_header *hdr, uint32_t uid)
{
const int max_days =
sizeof(hdr->day_first_uid) / sizeof(hdr->day_first_uid[0]);
struct tm tm;
time_t stamp;
int i, days;
/* get beginning of today */
tm = *localtime(&ioloop_time);
tm.tm_hour = 0;
tm.tm_min = 0;
tm.tm_sec = 0;
stamp = mktime(&tm);
if (stamp == (time_t)-1)
i_panic("mktime(today) failed");
if ((time_t)hdr->day_stamp >= stamp)
return;
/* get number of days since last message */
days = (stamp - hdr->day_stamp) / (3600*24);
if (days > max_days)
days = max_days;
/* @UNSAFE: move days forward and fill the missing days with old
day_first_uid[0]. */
memcpy(hdr->day_first_uid + days,
hdr->day_first_uid, max_days - days);
for (i = 1; i < days; i++)
hdr->day_first_uid[i] = hdr->day_first_uid[0];
hdr->day_stamp = stamp;
hdr->day_first_uid[0] = uid;
}
int mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_header *hdr,
const void *data)
{
int ret = 0;
t_push();
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_APPEND: {
const struct mail_index_record *rec, *end;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec < end; rec++) {
ret = sync_append(rec, ctx);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_EXPUNGE:
case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
const struct mail_transaction_expunge *rec, *end;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec < end; rec++) {
ret = sync_expunge(rec, ctx);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *rec, *end;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec < end; rec++) {
ret = sync_flag_update(rec, ctx);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_HEADER_UPDATE: {
const struct mail_transaction_header_update *rec;
unsigned int i;
for (i = 0; i < hdr->size; ) {
rec = CONST_PTR_OFFSET(data, i);
ret = sync_header_update(rec, ctx);
if (ret <= 0)
break;
i += sizeof(*rec) + rec->size;
if ((i % 4) != 0)
i += 4 - (i % 4);
}
break;
}
case MAIL_TRANSACTION_EXT_INTRO: {
const struct mail_transaction_ext_intro *rec = data;
unsigned int i;
uint32_t prev_seq;
uoff_t prev_offset;
mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
&prev_seq, &prev_offset);
ctx->ext_intro_seq = prev_seq;
ctx->ext_intro_offset = prev_offset;
ctx->ext_intro_size = hdr->size + sizeof(*hdr);
for (i = 0; i < hdr->size; ) {
if (i + sizeof(*rec) > hdr->size) {
/* should be just extra padding */
break;
}
rec = CONST_PTR_OFFSET(data, i);
ret = mail_index_sync_ext_intro(ctx, rec);
if (ret <= 0)
break;
i += sizeof(*rec) + rec->name_size;
if ((i % 4) != 0)
i += 4 - (i % 4);
}
break;
}
case MAIL_TRANSACTION_EXT_RESET: {
const struct mail_transaction_ext_reset *rec = data;
ret = mail_index_sync_ext_reset(ctx, rec);
break;
}
case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
const struct mail_transaction_ext_hdr_update *rec = data;
unsigned int i;
for (i = 0; i < hdr->size; ) {
rec = CONST_PTR_OFFSET(data, i);
ret = mail_index_sync_ext_hdr_update(ctx, rec);
if (ret <= 0)
break;
i += sizeof(*rec) + rec->size;
if ((i % 4) != 0)
i += 4 - (i % 4);
}
break;
}
case MAIL_TRANSACTION_EXT_REC_UPDATE: {
const struct mail_transaction_ext_rec_update *rec, *end;
const struct mail_index_ext *ext;
unsigned int record_size;
if (ctx->cur_ext_id == (uint32_t)-1) {
mail_index_sync_set_corrupted(ctx,
"Extension record updated "
"without intro prefix");
ret = -1;
break;
}
if (ctx->cur_ext_ignore) {
ret = 1;
break;
}
ext = array_idx(&ctx->view->map->extensions, ctx->cur_ext_id);
/* the record is padded to 32bits in the transaction log */
record_size = (sizeof(*rec) + ext->record_size + 3) & ~3;
rec = data;
end = CONST_PTR_OFFSET(data, hdr->size);
while (rec < end) {
ret = mail_index_sync_ext_rec_update(ctx, rec);
if (ret <= 0)
break;
rec = CONST_PTR_OFFSET(rec, record_size);
}
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *rec = data;
ret = mail_index_sync_keywords(ctx, hdr, rec);
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *rec = data;
ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
break;
}
default:
i_unreached();
}
t_pop();
i_assert(ctx->view->map->records_count ==
ctx->view->map->hdr.messages_count);
i_assert(ctx->view->hdr.messages_count ==
ctx->view->map->hdr.messages_count);
return ret;
}
void mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
struct mail_index_view *view,
enum mail_index_sync_handler_type type)
{
memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
sync_map_ctx->view = view;
sync_map_ctx->cur_ext_id = (uint32_t)-1;
sync_map_ctx->type = type;
/* make sure we re-read it in case it has changed */
sync_map_ctx->view->map->keywords_read = FALSE;
mail_index_sync_init_handlers(sync_map_ctx);
}
void mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
{
if (sync_map_ctx->expunge_handlers_used)
mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
mail_index_sync_deinit_handlers(sync_map_ctx);
}
static void mail_index_sync_remove_recent(struct mail_index_sync_ctx *sync_ctx)
{
struct mail_index_map *map = sync_ctx->view->map;
struct mail_index_record *rec;
unsigned int i;
for (i = 0; i < map->records_count; i++) {
rec = MAIL_INDEX_MAP_IDX(map, i);
if ((rec->flags & MAIL_RECENT) != 0) {
rec->flags &= ~MAIL_RECENT;
write_seq_update(map, i + 1, i + 1);
}
}
map->hdr.recent_messages_count = 0;
map->hdr.first_recent_uid_lowwater = map->hdr.next_uid;
}
static void log_view_seek_back(struct mail_transaction_log_view *log_view)
{
uint32_t prev_seq;
uoff_t prev_offset;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
mail_transaction_log_view_seek(log_view, prev_seq, prev_offset);
}
int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
bool sync_only_external)
{
struct mail_index *index = sync_ctx->index;
struct mail_index_view *view = sync_ctx->view;
struct mail_index_map *map;
struct mail_index_sync_map_ctx sync_map_ctx;
const struct mail_transaction_header *thdr;
const void *data;
unsigned int count, old_lock_id;
uint32_t i, first_append_uid;
int ret;
bool had_dirty, skipped, check_ext_offsets;
mail_index_sync_map_init(&sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_FILE);
sync_map_ctx.sync_only_external = sync_only_external;
/* we'll have to update view->lock_id to avoid mail_index_view_lock()
trying to update the file later. */
old_lock_id = view->lock_id;
if (mail_index_lock_exclusive(index, &view->lock_id) < 0)
return -1;
mail_index_unlock(index, old_lock_id);
/* NOTE: locking may change index->map so make sure the assignment is
after locking */
map = index->map;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
map->write_to_disk = TRUE;
if (map->hdr_base != map->hdr_copy_buf->data) {
buffer_reset(map->hdr_copy_buf);
buffer_append(map->hdr_copy_buf, map->hdr_base,
map->hdr.header_size);
map->hdr_base = map->hdr_copy_buf->data;
}
i_assert(map->hdr.base_header_size >= sizeof(map->hdr));
mail_index_unmap(index, &view->map);
view->map = map;
view->map->refcount++;
i_assert(view->hdr.messages_count == map->hdr.messages_count);
had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
if (had_dirty)
map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
if (sync_ctx->sync_recent) {
/* mark all messages non-recent */
mail_index_sync_remove_recent(sync_ctx);
}
/* make sure we don't go doing fsck while modifying the index */
index->sync_update = TRUE;
first_append_uid = 0;
check_ext_offsets = TRUE;
while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
&data, &skipped)) > 0) {
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
if (sync_only_external) {
/* we're syncing only external changes. */
continue;
}
} else if (check_ext_offsets) {
if (mail_index_is_ext_synced(view->log_view, view->map))
continue;
check_ext_offsets = FALSE;
}
if ((thdr->type & MAIL_TRANSACTION_APPEND) != 0) {
const struct mail_index_record *rec = data;
if (first_append_uid == 0)
first_append_uid = rec->uid;
map = view->map;
count = thdr->size / sizeof(*rec);
if ((ret = mail_index_grow(index, map, count)) < 0)
break;
if (map != index->map) {
index->map->refcount++;
mail_index_sync_replace_map(&sync_map_ctx,
index->map);
}
if (ret == 0) {
/* moved to memory. data pointer is invalid,
seek back and do this append again. */
log_view_seek_back(view->log_view);
continue;
}
}
if (mail_index_sync_record(&sync_map_ctx, thdr, data) < 0) {
ret = -1;
break;
}
}
if (ret == 0) {
mail_index_sync_update_log_offset(&sync_map_ctx, view->map,
TRUE);
}
mail_index_sync_map_deinit(&sync_map_ctx);
index->sync_update = FALSE;
if (ret < 0) {
mail_index_view_unlock(view);
return -1;
}
map = view->map;
i_assert(map->records_count == map->hdr.messages_count);
i_assert(map->hdr_copy_buf->used <= map->hdr.header_size);
if (first_append_uid != 0)
mail_index_update_day_headers(&map->hdr, first_append_uid);
if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 &&
had_dirty) {
/* do we have dirty flags anymore? */
const struct mail_index_record *rec;
for (i = 0; i < map->records_count; i++) {
rec = MAIL_INDEX_MAP_IDX(map, i);
if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
map->hdr.flags |=
MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
break;
}
}
}
if (mail_index_map_msync(index, map) < 0)
ret = -1;
i_assert(view->map == index->map);
view->hdr = map->hdr;
mail_index_view_unlock(view);
return ret;
}