mail-index-sync-update.c revision 76b43e4417bab52e913da39b5f5bc2a130d3f149
76b43e4417bab52e913da39b5f5bc2a130d3f149Timo Sirainen/* Copyright (c) 2004-2008 Dovecot authors, see the included COPYING file */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "ioloop.h"
16f816d3f3c32ae3351834253f52ddd0212bcbf3Timo Sirainen#include "array.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mmap-util.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-view-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-sync-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-transaction-log.h"
463e82bdf0e990f4f2252d2b53ea23a5abe5883cTimo Sirainen#include "mail-transaction-log-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen/* If we have less than this many bytes to sync from log file, don't bother
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen reading the main index */
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainenstatic void
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainenmail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen struct mail_index_map *map, bool eol)
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen{
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen uint32_t prev_seq;
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen uoff_t prev_offset;
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen &prev_seq, &prev_offset);
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen if (!eol) {
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen if (prev_offset == ctx->ext_intro_end_offset &&
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen prev_seq == ctx->ext_intro_seq) {
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen /* previous transaction was an extension introduction.
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen we probably came here from
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen mail_index_sync_ext_reset(). if there are any more
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen views which want to continue syncing it needs the
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen intro. so back up a bit more.
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen don't do this in case the last transaction in the
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen log is the extension intro, so we don't keep trying
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen to sync it over and over again. */
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen prev_offset = ctx->ext_intro_offset;
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen }
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen } else {
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
7bd3f5614e0dd2324dd1015f084de72c0b069a1aTimo Sirainen if (map->hdr.log_file_seq != prev_seq) {
7bd3f5614e0dd2324dd1015f084de72c0b069a1aTimo Sirainen map->hdr.log_file_seq = prev_seq;
7bd3f5614e0dd2324dd1015f084de72c0b069a1aTimo Sirainen map->hdr.log_file_tail_offset = 0;
7bd3f5614e0dd2324dd1015f084de72c0b069a1aTimo Sirainen }
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen }
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen map->hdr.log_file_head_offset = prev_offset;
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen}
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_map *map)
bbf796c17f02538058d7559bfe96d677e5b55015Timo Sirainen{
e6d7d19c328e7043ad35d5a52c1617bde915a16fTimo Sirainen struct mail_index_view *view = ctx->view;
d7095f3a4466fbb78b2d5eb3d322bc15a5b0ab1fTimo Sirainen
153de7823e64c67678b3fc95719c41a8ec5b864dTimo Sirainen i_assert(view->map != map);
153de7823e64c67678b3fc95719c41a8ec5b864dTimo Sirainen
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen mail_index_sync_update_log_offset(ctx, view->map, FALSE);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen mail_index_unmap(&view->map);
bbf796c17f02538058d7559bfe96d677e5b55015Timo Sirainen view->map = map;
e6d7d19c328e7043ad35d5a52c1617bde915a16fTimo Sirainen
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
9315dd69233d554452df0c12bc57002d2042a8f4Timo Sirainen view->index->map = map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
538c58fc95200fcc5e91abdda8b912b574a2f968Timo Sirainen
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainenstatic void
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainenmail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_map *map = ctx->view->map;
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) ||
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map->lock_id != 0);
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen if (map->refcount > 1) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map = mail_index_map_clone(map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_replace_map(ctx, map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen mail_index_map_move_to_memory(ctx->view->map);
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen}
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstruct mail_index_map *
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen mail_index_sync_move_to_private_memory(ctx);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mail_index_record_map_move_to_private(ctx->view->map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ctx->view->map->write_atomic = TRUE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return ctx->view->map;
bbf796c17f02538058d7559bfe96d677e5b55015Timo Sirainen}
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainenstatic int
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainenmail_index_header_update_counts(struct mail_index_header *hdr,
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen uint8_t old_flags, uint8_t new_flags,
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen const char **error_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* different seen-flag */
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if ((old_flags & MAIL_SEEN) != 0) {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (hdr->seen_messages_count == 0) {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen *error_r = "Seen counter wrong";
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen return -1;
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr->seen_messages_count--;
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen } else {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (hdr->seen_messages_count >= hdr->messages_count) {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen *error_r = "Seen counter wrong";
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen return -1;
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen }
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (++hdr->seen_messages_count == hdr->messages_count)
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen hdr->first_unseen_uid_lowwater = hdr->next_uid;
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* different deleted-flag */
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if ((old_flags & MAIL_DELETED) == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen hdr->deleted_messages_count++;
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (hdr->deleted_messages_count > hdr->messages_count) {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen *error_r = "Deleted counter wrong";
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen return -1;
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen }
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen } else {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (hdr->deleted_messages_count == 0 ||
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen hdr->deleted_messages_count > hdr->messages_count) {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen *error_r = "Deleted counter wrong";
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen return -1;
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen }
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (--hdr->deleted_messages_count == 0)
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen hdr->first_deleted_uid_lowwater = hdr->next_uid;
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenstatic void
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenmail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen uint32_t uid,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen uint8_t old_flags, uint8_t new_flags)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen{
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_map *const *maps;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen const char *error;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen unsigned int i, count;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen maps = array_get(&ctx->view->map->rec_map->maps, &count);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen for (i = 0; i < count; i++) {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen if (uid >= maps[i]->hdr.next_uid)
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen continue;
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (mail_index_header_update_counts(&maps[i]->hdr,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen old_flags, new_flags,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen &error) < 0)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_sync_set_corrupted(ctx, "%s", error);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen}
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen uint32_t uid, uint8_t old_flags,
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen uint8_t new_flags, bool all)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const char *error;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (all) {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mail_index_sync_header_update_counts_all(ctx, uid, old_flags,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen new_flags);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen } else {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen i_assert(uid < ctx->view->map->hdr.next_uid);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (mail_index_header_update_counts(&ctx->view->map->hdr,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen old_flags, new_flags,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen &error) < 0)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_sync_set_corrupted(ctx, "%s", error);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen}
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainenstatic void
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenmail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen uint32_t uid, enum mail_flags flags)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen struct mail_index_map *const *maps;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen unsigned int i, count;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen maps = array_get(&ctx->view->map->rec_map->maps, &count);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen for (i = 0; i < count; i++) {
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen if ((flags & MAIL_SEEN) == 0 &&
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen uid < maps[i]->hdr.first_unseen_uid_lowwater)
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen maps[i]->hdr.first_unseen_uid_lowwater = uid;
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen if ((flags & MAIL_DELETED) != 0 &&
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen uid < maps[i]->hdr.first_deleted_uid_lowwater)
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen maps[i]->hdr.first_deleted_uid_lowwater = uid;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainenstatic void
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainensync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t seq1, uint32_t seq2)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_index_expunge_handler *eh;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_record *rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int i, count;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
9315dd69233d554452df0c12bc57002d2042a8f4Timo Sirainen /* call expunge handlers only when syncing index file */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE)
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen return;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!ctx->expunge_handlers_set)
7889c9f65e23c83fc31cecf304cab4ab070d6aa1Timo Sirainen mail_index_sync_init_expunge_handlers(ctx);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!array_is_created(&ctx->expunge_handlers))
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen return;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen eh = array_get(&ctx->expunge_handlers, &count);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0; i < count; i++, eh++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (; seq1 <= seq2; seq1++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen rec = MAIL_INDEX_MAP_IDX(ctx->view->map, seq1-1);
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen /* FIXME: does expunge handler's return value matter?
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen we probably shouldn't disallow expunges if the
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen handler returns failure.. should it be just changed
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen to return void? */
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen (void)eh->handler(ctx, seq1,
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen PTR_OFFSET(rec, eh->record_offset),
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen eh->sync_context, eh->context);
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen }
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainensync_expunge(const struct mail_transaction_expunge *e, unsigned int count,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_sync_map_ctx *ctx)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_map *map = ctx->view->map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_record *rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t seq_count, seq, seq1, seq2;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int i;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0; i < count; i++, e++) {
3da614c39dd29f536c485089e67839b4cf89fed3Timo Sirainen if (!mail_index_lookup_seq_range(ctx->view, e->uid1, e->uid2,
3da614c39dd29f536c485089e67839b4cf89fed3Timo Sirainen &seq1, &seq2)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* everything expunged already */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen continue;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen sync_expunge_call_handlers(ctx, seq1, seq2);
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen
075f90abe6b6b12dc72bca21bfce8086b4b190ecTimo Sirainen map = mail_index_sync_get_atomic_map(ctx);
9bc6e10d9c6d6ffb4a2ed49a3b3d2a180f2a87a3Timo Sirainen for (seq = seq1; seq <= seq2; seq++) {
9bc6e10d9c6d6ffb4a2ed49a3b3d2a180f2a87a3Timo Sirainen rec = MAIL_INDEX_MAP_IDX(map, seq-1);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mail_index_sync_header_update_counts(ctx, rec->uid,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec->flags, 0,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen FALSE);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* @UNSAFE */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen memmove(MAIL_INDEX_MAP_IDX(map, seq1-1),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen MAIL_INDEX_MAP_IDX(map, seq2),
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen (map->rec_map->records_count - seq2) *
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->hdr.record_size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen seq_count = seq2 - seq1 + 1;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map->records_count -= seq_count;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.messages_count -= seq_count;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen }
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return 1;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenvoid mail_index_sync_write_seq_update(struct mail_index_sync_map_ctx *ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t seq1, uint32_t seq2)
abfcd9f73b9ad1eeef4fe6e9940383defabf68c3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_map *map = ctx->view->map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (map->rec_map->write_seq_first == 0 ||
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map->write_seq_first > seq1)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map->write_seq_first = seq1;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (map->rec_map->write_seq_last < seq2)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map->write_seq_last = seq2;
abfcd9f73b9ad1eeef4fe6e9940383defabf68c3Timo Sirainen}
abfcd9f73b9ad1eeef4fe6e9940383defabf68c3Timo Sirainen
cd5ee8630497fdbd853ef588a858b4ef619a5e03Timo Sirainenstatic int sync_append(const struct mail_index_record *rec,
cd5ee8630497fdbd853ef588a858b4ef619a5e03Timo Sirainen struct mail_index_sync_map_ctx *ctx)
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen{
7394389230750c45b105cdefb5850c81cae8cdc0Timo Sirainen struct mail_index_view *view = ctx->view;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen struct mail_index_map *map = view->map;
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen enum mail_flags new_flags;
7e94cf9d70ce9fdeccb7a85ff400b899e6386f36Timo Sirainen void *dest;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen size_t append_pos;
7e94cf9d70ce9fdeccb7a85ff400b899e6386f36Timo Sirainen
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen if (rec->uid < map->hdr.next_uid) {
9887c39c5ba429169389153ca99de49e084a73f0Timo Sirainen mail_index_sync_set_corrupted(ctx,
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen "Append with UID %u, but next_uid = %u",
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen rec->uid, map->hdr.next_uid);
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return -1;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen }
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* move to memory. the mapping is written when unlocking so we don't
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen waste time re-mmap()ing multiple times or waste space growing index
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file too large */
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen mail_index_sync_move_to_private_memory(ctx);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map = view->map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen if (rec->uid <= map->rec_map->last_appended_uid) {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen i_assert(map->hdr.messages_count < map->rec_map->records_count);
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen /* the flags may have changed since it was added to map.
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen use the updated flags already, so flag counters won't get
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen broken. */
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen new_flags =
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count)->flags;
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen } else {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen /* don't rely on buffer->used being at the correct position.
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen at least expunges can move it */
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen append_pos = map->rec_map->records_count * map->hdr.record_size;
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen dest = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen map->hdr.record_size);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen map->rec_map->records =
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen buffer_get_modifiable_data(map->rec_map->buffer, NULL);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen memcpy(dest, rec, sizeof(*rec));
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen map->hdr.record_size - sizeof(*rec));
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen map->rec_map->records_count++;
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen map->rec_map->last_appended_uid = rec->uid;
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen new_flags = rec->flags;
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen
2ebfb5c0608e2323b73271208f4036a7ea7d7f3aTimo Sirainen mail_index_sync_write_seq_update(ctx,
2ebfb5c0608e2323b73271208f4036a7ea7d7f3aTimo Sirainen map->rec_map->records_count,
2ebfb5c0608e2323b73271208f4036a7ea7d7f3aTimo Sirainen map->rec_map->records_count);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen }
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen map->hdr.messages_count++;
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen map->hdr.next_uid = rec->uid+1;
41bb0aa8e357876bc9a1916a37c9e3e78e5f8185Timo Sirainen
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
dd8de60250511cc729b67249e61dfc6b4debff11Timo Sirainen
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mail_index_sync_header_update_counts(ctx, rec->uid,
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen 0, new_flags, FALSE);
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainenstatic int sync_flag_update(const struct mail_transaction_flag_update *u,
cd5ee8630497fdbd853ef588a858b4ef619a5e03Timo Sirainen struct mail_index_sync_map_ctx *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
7394389230750c45b105cdefb5850c81cae8cdc0Timo Sirainen struct mail_index_view *view = ctx->view;
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen struct mail_index_record *rec;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint8_t flag_mask, old_flags;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen uint32_t idx, seq1, seq2;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen
3da614c39dd29f536c485089e67839b4cf89fed3Timo Sirainen if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2))
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_write_seq_update(ctx, seq1, seq2);
41bb0aa8e357876bc9a1916a37c9e3e78e5f8185Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen flag_mask = ~u->remove_flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
51920d00fa50edf7b2e9b1019288d64b7abee7f3Timo Sirainen if (((u->add_flags | u->remove_flags) &
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen (MAIL_SEEN | MAIL_DELETED)) == 0) {
51920d00fa50edf7b2e9b1019288d64b7abee7f3Timo Sirainen /* we're not modifying any counted/lowwatered flags */
21ec6628c567eeff025af35d8027be01044b0b1aTimo Sirainen for (idx = seq1-1; idx < seq2; idx++) {
21ec6628c567eeff025af35d8027be01044b0b1aTimo Sirainen rec = MAIL_INDEX_MAP_IDX(view->map, idx);
21ec6628c567eeff025af35d8027be01044b0b1aTimo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
21ec6628c567eeff025af35d8027be01044b0b1aTimo Sirainen }
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen } else {
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen for (idx = seq1-1; idx < seq2; idx++) {
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen rec = MAIL_INDEX_MAP_IDX(view->map, idx);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen old_flags = rec->flags;
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen mail_index_header_update_lowwaters(ctx, rec->uid,
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen rec->flags);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mail_index_sync_header_update_counts(ctx, rec->uid,
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen old_flags,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen rec->flags, TRUE);
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return 1;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen}
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainenstatic int sync_header_update(const struct mail_transaction_header_update *u,
cd5ee8630497fdbd853ef588a858b4ef619a5e03Timo Sirainen struct mail_index_sync_map_ctx *ctx)
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen{
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen struct mail_index_map *map = ctx->view->map;
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen if (u->offset >= map->hdr.base_header_size ||
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen u->offset + u->size > map->hdr.base_header_size) {
9887c39c5ba429169389153ca99de49e084a73f0Timo Sirainen mail_index_sync_set_corrupted(ctx,
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen "Header update outside range: %u + %u > %u",
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen u->offset, u->size, map->hdr.base_header_size);
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen return -1;
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen }
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen map->hdr_base = map->hdr_copy_buf->data;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->write_base_header = TRUE;
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen /* @UNSAFE */
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen memcpy(PTR_OFFSET(&map->hdr, u->offset),
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen u + 1, u->size);
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen } else if (u->offset < sizeof(map->hdr)) {
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen memcpy(PTR_OFFSET(&map->hdr, u->offset),
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen u + 1, sizeof(map->hdr) - u->offset);
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen }
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen /* the tail offset updates are intended for internal transaction
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen log handling. we'll update the offset in the header only when
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen the sync is finished. */
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen return 1;
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen}
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainenint mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_transaction_header *hdr,
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const void *data)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen{
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen int ret = 0;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_APPEND: {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_index_record *rec, *end;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen end = CONST_PTR_OFFSET(data, hdr->size);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen for (rec = data; rec < end; rec++) {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen ret = sync_append(rec, ctx);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if (ret <= 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_EXPUNGE:
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_transaction_expunge *rec = data, *end;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen /* this is simply a request for expunge */
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen break;
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen end = CONST_PTR_OFFSET(data, hdr->size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = sync_expunge(rec, end - rec, ctx);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_FLAG_UPDATE: {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_transaction_flag_update *rec, *end;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen end = CONST_PTR_OFFSET(data, hdr->size);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen for (rec = data; rec < end; rec++) {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen ret = sync_flag_update(rec, ctx);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if (ret <= 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_HEADER_UPDATE: {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_transaction_header_update *rec;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen unsigned int i;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen for (i = 0; i < hdr->size; ) {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen rec = CONST_PTR_OFFSET(data, i);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen ret = sync_header_update(rec, ctx);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if (ret <= 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen i += sizeof(*rec) + rec->size;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if ((i % 4) != 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen i += 4 - (i % 4);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_EXT_INTRO: {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_transaction_ext_intro *rec = data;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen unsigned int i;
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen uint32_t prev_seq;
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen uoff_t prev_offset;
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen &prev_seq, &prev_offset);
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen ctx->ext_intro_seq = prev_seq;
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen ctx->ext_intro_offset = prev_offset;
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen ctx->ext_intro_end_offset =
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen prev_offset + hdr->size + sizeof(*hdr);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen for (i = 0; i < hdr->size; ) {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if (i + sizeof(*rec) > hdr->size) {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen /* should be just extra padding */
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen rec = CONST_PTR_OFFSET(data, i);
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen if (i + sizeof(*rec) + rec->name_size > hdr->size) {
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen mail_index_sync_set_corrupted(ctx,
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "ext intro: name_size too large");
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen ret = -1;
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen break;
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen }
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen
a0d34d3982507f513a9d800082481e9faeb9a943Timo Sirainen ret = mail_index_sync_ext_intro(ctx, rec);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if (ret <= 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen i += sizeof(*rec) + rec->name_size;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if ((i % 4) != 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen i += 4 - (i % 4);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen case MAIL_TRANSACTION_EXT_RESET: {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen const struct mail_transaction_ext_reset *rec = data;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen if (hdr->size != sizeof(*rec)) {
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen mail_index_sync_set_corrupted(ctx,
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "ext reset: invalid record size");
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen ret = -1;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen break;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen }
a0d34d3982507f513a9d800082481e9faeb9a943Timo Sirainen ret = mail_index_sync_ext_reset(ctx, rec);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_transaction_ext_hdr_update *rec = data;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen unsigned int i;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen for (i = 0; i < hdr->size; ) {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen rec = CONST_PTR_OFFSET(data, i);
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen if (i + sizeof(*rec) > hdr->size ||
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen i + sizeof(*rec) + rec->size > hdr->size) {
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen mail_index_sync_set_corrupted(ctx,
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "ext hdr update: invalid record size");
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen ret = -1;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen break;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen }
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen
a0d34d3982507f513a9d800082481e9faeb9a943Timo Sirainen ret = mail_index_sync_ext_hdr_update(ctx, rec);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if (ret <= 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen i += sizeof(*rec) + rec->size;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if ((i % 4) != 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen i += 4 - (i % 4);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_EXT_REC_UPDATE: {
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen const struct mail_transaction_ext_rec_update *rec;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_index_ext *ext;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen unsigned int i, record_size;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen if (ctx->cur_ext_map_idx == (uint32_t)-1) {
2d79e603e20a32bdae4c2b516ead5c5c9169545aTimo Sirainen mail_index_sync_set_corrupted(ctx,
86d52f310fe939090c66b780a3b6ffe5d10dc8faTimo Sirainen "Extension record updated "
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen "without intro prefix");
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen ret = -1;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen }
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen if (ctx->cur_ext_ignore) {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen ret = 1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen break;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen ext = array_idx(&ctx->view->map->extensions,
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen ctx->cur_ext_map_idx);
2d79e603e20a32bdae4c2b516ead5c5c9169545aTimo Sirainen /* the record is padded to 32bits in the transaction log */
2d79e603e20a32bdae4c2b516ead5c5c9169545aTimo Sirainen record_size = (sizeof(*rec) + ext->record_size + 3) & ~3;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen for (i = 0; i < hdr->size; i += record_size) {
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen rec = CONST_PTR_OFFSET(data, i);
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen if (i + record_size > hdr->size) {
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen mail_index_sync_set_corrupted(ctx,
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "ext rec update: invalid record size");
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen ret = -1;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen break;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen }
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen
a0d34d3982507f513a9d800082481e9faeb9a943Timo Sirainen ret = mail_index_sync_ext_rec_update(ctx, rec);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if (ret <= 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen case MAIL_TRANSACTION_KEYWORD_UPDATE: {
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen const struct mail_transaction_keyword_update *rec = data;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen ret = mail_index_sync_keywords(ctx, hdr, rec);
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen break;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen }
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen case MAIL_TRANSACTION_KEYWORD_RESET: {
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen const struct mail_transaction_keyword_reset *rec = data;
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen break;
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen default:
20261d71760e4199cb8d906ab9704a4561d954d7Timo Sirainen mail_index_sync_set_corrupted(ctx,
20261d71760e4199cb8d906ab9704a4561d954d7Timo Sirainen "Unknown transaction record type 0x%x",
20261d71760e4199cb8d906ab9704a4561d954d7Timo Sirainen (hdr->type & MAIL_TRANSACTION_TYPE_MASK));
20261d71760e4199cb8d906ab9704a4561d954d7Timo Sirainen ret = -1;
20261d71760e4199cb8d906ab9704a4561d954d7Timo Sirainen break;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen }
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen return ret;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen}
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenvoid mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen struct mail_index_view *view,
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen enum mail_index_sync_handler_type type)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen{
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen sync_map_ctx->view = view;
7ded22760598b78ee29f9418eacc0abe3fb51055Timo Sirainen sync_map_ctx->cur_ext_map_idx = (uint32_t)-1;
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen sync_map_ctx->type = type;
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen mail_index_sync_init_handlers(sync_map_ctx);
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen}
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainenvoid mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen{
91d4c7b37580b031ed7b0154ae10c643521803f3Timo Sirainen if (sync_map_ctx->unknown_extensions != NULL)
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen buffer_free(&sync_map_ctx->unknown_extensions);
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen if (sync_map_ctx->expunge_handlers_used)
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen mail_index_sync_deinit_handlers(sync_map_ctx);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen}
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
ccffbed92cb02c24fd717808a84138240bf1885bTimo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_index_record *rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int i;
ccffbed92cb02c24fd717808a84138240bf1885bTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* do we have dirty flags anymore? */
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen for (i = 0; i < map->rec_map->records_count; i++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen rec = MAIL_INDEX_MAP_IDX(map, i);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
ccffbed92cb02c24fd717808a84138240bf1885bTimo Sirainen}
ccffbed92cb02c24fd717808a84138240bf1885bTimo Sirainen
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen#ifdef DEBUG
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainenvoid mail_index_map_check(struct mail_index_map *map)
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen{
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen const struct mail_index_header *hdr = &map->hdr;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen unsigned int i, del = 0, seen = 0;
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen
d12f05c7c391786d0d9795ec3aa4377280bbfaeaTimo Sirainen i_assert(hdr->messages_count <= map->rec_map->records_count);
d12f05c7c391786d0d9795ec3aa4377280bbfaeaTimo Sirainen for (i = 0; i < hdr->messages_count; i++) {
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen const struct mail_index_record *rec;
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen rec = MAIL_INDEX_MAP_IDX(map, i);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen if (rec->flags & MAIL_DELETED) {
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen del++;
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen }
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen if (rec->flags & MAIL_SEEN)
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen seen++;
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen else
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen }
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen i_assert(del == hdr->deleted_messages_count);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen i_assert(seen == hdr->seen_messages_count);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen}
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen#endif
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenint mail_index_sync_map(struct mail_index_map **_map,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen enum mail_index_sync_handler_type type, bool force)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_map *map = *_map;
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mail_index *index = map->index;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_view *view;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen const struct mail_transaction_header *thdr;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const void *tdata;
c3248869ddd406a7a46b8c05633f0cccc72fcb77Timo Sirainen uint32_t prev_seq;
c3248869ddd406a7a46b8c05633f0cccc72fcb77Timo Sirainen uoff_t start_offset, prev_offset;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen int ret;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen bool had_dirty, reset;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!force) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* see if we'd prefer to reopen the index file instead of
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen syncing the current map from the transaction log */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uoff_t log_size, index_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen if (index->log->head == NULL)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen if (index->fd == -1 &&
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen index->log->head->hdr.prev_file_seq != 0) {
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen /* we don't know the index's size, so use the
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen smallest index size we're willing to read */
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen } else {
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen index_size = map->hdr.header_size +
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->rec_map->records_count *
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen map->hdr.record_size;
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* this isn't necessary correct currently, but it should be
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen close enough */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen log_size = index->log->head->last_size;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if (log_size > map->hdr.log_file_tail_offset &&
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen log_size - map->hdr.log_file_tail_offset > index_size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen view = mail_index_view_open_with_map(index, map);
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen ret = mail_transaction_log_view_set(view->log_view,
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen map->hdr.log_file_seq, start_offset,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen (uint32_t)-1, (uoff_t)-1, &reset);
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen if (ret <= 0) {
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen if (force && ret == 0) {
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen /* the seq/offset is probably broken */
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen (void)mail_index_fsck(index);
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* can't use it. sync by re-reading index. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_view_close(&view);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
31ddc75584c5cde53d2e78a737587f2e7fdcb0d2Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* view referenced the map. avoid unnecessary map cloning by
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unreferencing the map while view exists. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->refcount--;
5c1a8aee989af87bddefd71e2aa83aa2bd695155Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (had_dirty) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->write_base_header = TRUE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
31ddc75584c5cde53d2e78a737587f2e7fdcb0d2Timo Sirainen
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen if (map->hdr_base != map->hdr_copy_buf->data) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* if syncing updates the header, it updates hdr_copy_buf
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen and updates hdr_base to hdr_copy_buf. so the buffer must
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen initially contain a valid header or we'll break it when
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen writing it. */
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen buffer_reset(map->hdr_copy_buf);
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen buffer_append(map->hdr_copy_buf, map->hdr_base,
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen map->hdr.header_size);
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen map->hdr_base = map->hdr_copy_buf->data;
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen }
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_map_init(&sync_map_ctx, view, type);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen if (reset) {
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* Reset the entire index. Leave only indexid and
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen log_file_seq. */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen &prev_seq, &prev_offset);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen map = mail_index_map_alloc(index);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen map->hdr.log_file_seq = prev_seq;
7bd3f5614e0dd2324dd1015f084de72c0b069a1aTimo Sirainen map->hdr.log_file_tail_offset = 0;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen mail_index_sync_replace_map(&sync_map_ctx, map);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map = NULL;
b9ce555e8624a5593b3bfd81b572b7d2e1e1fca5Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* FIXME: when transaction sync lock is removed, we'll need to handle
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen the case when a transaction is committed while mailbox is being
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen synced ([synced transactions][new transaction][ext transaction]).
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen this means int_offset contains [synced] and ext_offset contains
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen all */
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen &tdata)) > 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen &prev_seq, &prev_offset);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if (LOG_IS_BEFORE(prev_seq, prev_offset,
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen view->map->hdr.log_file_seq,
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen view->map->hdr.log_file_head_offset)) {
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen /* this has been synced already. we're here only to call
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen expunge handlers and extension update handlers. */
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen continue;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
2a734f36105e33ab452d057df6bc7a2b7d9f96f0Timo Sirainen continue;
2a734f36105e33ab452d057df6bc7a2b7d9f96f0Timo Sirainen }
2a734f36105e33ab452d057df6bc7a2b7d9f96f0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* we'll just skip over broken entries */
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen T_FRAME(
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen (void)mail_index_sync_record(&sync_map_ctx,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen thdr, tdata);
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen );
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map = view->map;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (had_dirty)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_update_hdr_dirty_flag(map);
ccffbed92cb02c24fd717808a84138240bf1885bTimo Sirainen
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen#ifdef DEBUG
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen mail_index_map_check(map);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen#endif
9ae24544b49e9cbf12bfa888279988a97235e10dTimo Sirainen i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen /* transaction log tracks internally the current tail offset.
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen besides using header updates, it also updates the offset to skip
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen over following external transactions to avoid extra unneeded log
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen reading. */
88b6f36ef1b453c08e8d9cadb229b39fc9bb4a1cTimo Sirainen i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq);
88b6f36ef1b453c08e8d9cadb229b39fc9bb4a1cTimo Sirainen if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) {
88b6f36ef1b453c08e8d9cadb229b39fc9bb4a1cTimo Sirainen map->hdr.log_file_tail_offset =
88b6f36ef1b453c08e8d9cadb229b39fc9bb4a1cTimo Sirainen index->log->head->max_tail_offset;
88b6f36ef1b453c08e8d9cadb229b39fc9bb4a1cTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen map->hdr_copy_buf->used);
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen }
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* restore refcount before closing the view. this is necessary also
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if map got cloned, because view closing would otherwise destroy it */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->refcount++;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_map_deinit(&sync_map_ctx);
1b317fe22572738681d34d80da8012638e1e1281Timo Sirainen mail_index_view_close(&view);
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen if (mail_index_map_check_header(map) <= 0) {
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen mail_index_set_error(index,
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen "Synchronization corrupted index header: %s",
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen index->filepath);
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen (void)mail_index_fsck(index);
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen map = index->map;
f31720ad5ece28ca3fa793920f1501165ad9603fTimo Sirainen } else if (sync_map_ctx.errors) {
f31720ad5ece28ca3fa793920f1501165ad9603fTimo Sirainen /* make sure the index looks valid now */
f31720ad5ece28ca3fa793920f1501165ad9603fTimo Sirainen (void)mail_index_fsck(index);
f31720ad5ece28ca3fa793920f1501165ad9603fTimo Sirainen map = index->map;
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen }
f7656d7bc15510a4259ed74ddda3c560de8a51c1Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen *_map = map;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return ret < 0 ? -1 : 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}