bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen/* If we have less than this many bytes to sync from log file, don't bother
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen reading the main index */
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainenmail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
20c892309312df8f4f73cfcaf8acd2ededda8b05Timo Sirainen /* handling lost changes in view syncing */
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen if (prev_offset == ctx->ext_intro_end_offset &&
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen /* previous transaction was an extension introduction.
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen we probably came here from
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen mail_index_sync_ext_reset(). if there are any more
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen views which want to continue syncing it needs the
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen intro. so back up a bit more.
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen don't do this in case the last transaction in the
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen log is the extension intro, so we don't keep trying
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen to sync it over and over again. */
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen mail_index_sync_update_log_offset(ctx, view->map, FALSE);
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainenmail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen mail_index_map_move_to_memory(ctx->view->map);
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)mail_index_sync_move_to_private_memory(ctx);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen mail_index_record_map_move_to_private(ctx->view->map);
7de1c472fd23ddac6b4dc5cbeee6fa6a8418b071Timo Sirainen mail_index_modseq_sync_map_replaced(ctx->modseq_ctx);
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainenmail_index_header_update_counts(struct mail_index_header *hdr,
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen const char **error_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* different seen-flag */
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (hdr->seen_messages_count >= hdr->messages_count) {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (++hdr->seen_messages_count == hdr->messages_count)
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen hdr->first_unseen_uid_lowwater = hdr->next_uid;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* different deleted-flag */
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen if (hdr->deleted_messages_count > hdr->messages_count) {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen hdr->deleted_messages_count > hdr->messages_count) {
1cad0dd34667548ba39f794ddeb9fc486cf4c666Timo Sirainen hdr->first_deleted_uid_lowwater = hdr->next_uid;
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenmail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen maps = array_get(&ctx->view->map->rec_map->maps, &count);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen for (i = 0; i < count; i++) {
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (mail_index_header_update_counts(&maps[i]->hdr,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_sync_set_corrupted(ctx, "%s", error);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
23079bf0a6e7489c5f542b0b897a71bdfd884a51Timo Sirainen mail_index_sync_set_corrupted(ctx, "uid %u >= next_uid %u",
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen if (mail_index_header_update_counts(&ctx->view->map->hdr,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen mail_index_sync_set_corrupted(ctx, "%s", error);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainenmail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen maps = array_get(&ctx->view->map->rec_map->maps, &count);
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen for (i = 0; i < count; i++) {
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen uid < maps[i]->hdr.first_deleted_uid_lowwater)
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen maps[i]->hdr.first_deleted_uid_lowwater = uid;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainensync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
1df526903ed039e8ff966a223c43b8d04eddf3c7Phil Carmody rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->map, seq);
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen /* FIXME: does expunge handler's return value matter?
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen we probably shouldn't disallow expunges if the
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen handler returns failure.. should it be just changed
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen to return void? */
b32de04eb77234b25e2e411884a2503a1bf3c255Phil Carmodysync_expunge_handlers_init(struct mail_index_sync_map_ctx *ctx)
b32de04eb77234b25e2e411884a2503a1bf3c255Phil Carmody /* call expunge handlers only when syncing index file */
dfdd228ab20092705da0e8812b3cd0a039319375Timo Sirainensync_expunge_range(struct mail_index_sync_map_ctx *ctx, const ARRAY_TYPE(seq_range) *seqs)
74896b89e1d82819d710f9322cf7c9e72d5841adPhil Carmody /* call the expunge handlers first */
74896b89e1d82819d710f9322cf7c9e72d5841adPhil Carmody for (i = 0; i < count; i++) {
aba9cc9bf97576c0ca653d4e218567e617061029Phil Carmody for (i = 0; i < count; i++) {
89b7d6ce9266288c156e3513f5798680f1e33572Phil Carmody mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags, 0);
53e0e6889bf659b86e4c7d2e83e27d69fd9d6bcbPhil Carmody /* @UNSAFE: move (prev_seq2+1) .. (seq1-1) to its
53e0e6889bf659b86e4c7d2e83e27d69fd9d6bcbPhil Carmody final location in the map if necessary */
53e0e6889bf659b86e4c7d2e83e27d69fd9d6bcbPhil Carmody uint32_t move_count = (seq1-1) - (prev_seq2+1) + 1;
89b7d6ce9266288c156e3513f5798680f1e33572Phil Carmody mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2);
53e0e6889bf659b86e4c7d2e83e27d69fd9d6bcbPhil Carmody /* Final stragglers */
53e0e6889bf659b86e4c7d2e83e27d69fd9d6bcbPhil Carmody uint32_t final_move_count = orig_rec_count - prev_seq2;
bddd52cb7f3e5a894c080f60750aa76b5aeaf103Timo Sirainenstatic void *sync_append_record(struct mail_index_map *map)
bddd52cb7f3e5a894c080f60750aa76b5aeaf103Timo Sirainen append_pos = map->rec_map->records_count * map->hdr.record_size;
bddd52cb7f3e5a894c080f60750aa76b5aeaf103Timo Sirainen ret = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
bddd52cb7f3e5a894c080f60750aa76b5aeaf103Timo Sirainen buffer_get_modifiable_data(map->rec_map->buffer, NULL);
9f19a50d5966643c4d1c5ca06868ac2ad31bc4d5Timo Sirainenstatic bool sync_update_ignored_change(struct mail_index_sync_map_ctx *ctx)
9f19a50d5966643c4d1c5ca06868ac2ad31bc4d5Timo Sirainen struct mail_index_transaction_commit_result *result =
1e2887ae909d9817cc43a7e40ecb50508419f7edTimo Sirainen uoff_t prev_log_offset, trans_start_offset, trans_end_offset;
1e2887ae909d9817cc43a7e40ecb50508419f7edTimo Sirainen /* we'll return TRUE if this modseq change was written within the
1e2887ae909d9817cc43a7e40ecb50508419f7edTimo Sirainen transaction that was just committed */
9f19a50d5966643c4d1c5ca06868ac2ad31bc4d5Timo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
1e2887ae909d9817cc43a7e40ecb50508419f7edTimo Sirainen trans_start_offset = trans_end_offset - result->commit_size;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainensync_modseq_update(struct mail_index_sync_map_ctx *ctx,
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen const struct mail_transaction_modseq_update *u,
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen unsigned int size)
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen const struct mail_transaction_modseq_update *end;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen for (; u < end; u++) {
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen if (u->uid == 0)
4b89231f4ec9cc69f4aea715e1d34f405c7e317dTimo Sirainen else if (!mail_index_lookup_seq(view, u->uid, &seq))
76959d3d6fed45d5f5e1397fcdcf09a5adb87f24Timo Sirainen min_modseq = ((uint64_t)u->modseq_high32 << 32) |
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen "modseqs updated before they were enabled");
9f19a50d5966643c4d1c5ca06868ac2ad31bc4d5Timo Sirainen if (ret == 0 && sync_update_ignored_change(ctx))
9f19a50d5966643c4d1c5ca06868ac2ad31bc4d5Timo Sirainen view->index->sync_commit_result->ignored_modseq_changes++;
cd5ee8630497fdbd853ef588a858b4ef619a5e03Timo Sirainenstatic int sync_append(const struct mail_index_record *rec,
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen "Append with UID %u, but next_uid = %u",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* move to memory. the mapping is written when unlocking so we don't
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen waste time re-mmap()ing multiple times or waste space growing index
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen file too large */
b99f3f908d51f4d1f7628bdf2cc6100cd8587656Timo Sirainen map = mail_index_sync_move_to_private_memory(ctx);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen if (rec->uid <= map->rec_map->last_appended_uid) {
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen i_assert(map->hdr.messages_count < map->rec_map->records_count);
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen /* the flags may have changed since it was added to map.
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen use the updated flags already, so flag counters won't get
1032e5427bf10566098f3b3bb9110e2bc1227e85Timo Sirainen old_rec = MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count);
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen /* don't rely on buffer->used being at the correct position.
a28a6267f48971117dec958b160deefd14ebb7a6Timo Sirainen at least expunges can move it */
945565e0c9cf979b5feeba6fbd4efce3bf4484adTimo Sirainen if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0 &&
945565e0c9cf979b5feeba6fbd4efce3bf4484adTimo Sirainen (view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0)
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
b9ec0443d7d8afebfe61c17a9d692d6fad30c276Timo Sirainen mail_index_sync_header_update_counts(ctx, rec->uid, 0, new_flags);
20b9283d4af31e45e588014da427fb2dbcd3227aTimo Sirainenstatic int sync_flag_update(const struct mail_transaction_flag_update *u,
20b9283d4af31e45e588014da427fb2dbcd3227aTimo Sirainen if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2))
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u)) {
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen mail_index_modseq_update_flags(ctx->modseq_ctx,
945565e0c9cf979b5feeba6fbd4efce3bf4484adTimo Sirainen if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0 &&
945565e0c9cf979b5feeba6fbd4efce3bf4484adTimo Sirainen (view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0)
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
51920d00fa50edf7b2e9b1019288d64b7abee7f3Timo Sirainen /* we're not modifying any counted/lowwatered flags */
21ec6628c567eeff025af35d8027be01044b0b1aTimo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
fa5957ffc9b676bfd649fa9953e63e72ee4ebeb4Timo Sirainen rec->flags = (rec->flags & flag_mask) | u->add_flags;
3342badd8c69adff34db589fb0a221ace5996212Timo Sirainen mail_index_header_update_lowwaters(ctx, rec->uid,
b9ec0443d7d8afebfe61c17a9d692d6fad30c276Timo Sirainen mail_index_sync_header_update_counts_all(ctx, rec->uid,
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainenstatic int sync_header_update(const struct mail_transaction_header_update *u,
fd2118e34f4d1d65cffdccc40d74dda931fae81eTimo Sirainen#define MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, field) \
fd2118e34f4d1d65cffdccc40d74dda931fae81eTimo Sirainen ((u)->offset <= offsetof(struct mail_index_header, field) && \
fd2118e34f4d1d65cffdccc40d74dda931fae81eTimo Sirainen (u)->offset + (u)->size > offsetof(struct mail_index_header, field))
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen u->offset + u->size > map->hdr.base_header_size) {
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen "Header update outside range: %u + %u > %u",
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen u->offset, u->size, map->hdr.base_header_size);
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
1af0c5d8814c2304d09d8ca844a84f0b9b0c1f61Timo Sirainen i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen /* @UNSAFE */
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
5efcaaace97d50e58e57424ee5bc51c7a0e942b4Timo Sirainen /* next_uid update tried to shrink its value. this can happen
5efcaaace97d50e58e57424ee5bc51c7a0e942b4Timo Sirainen in some race conditions with e.g. with dsync, so just
5efcaaace97d50e58e57424ee5bc51c7a0e942b4Timo Sirainen silently ignore it. */
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen /* the tail offset updates are intended for internal transaction
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen log handling. we'll update the offset in the header only when
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen the sync is finished. */
5a7b52012bf77132bb8f466d07e0e88c63fdba42Timo Sirainen map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
6d25194a4230037c0999aa6a01ae6814e03ec06cTimo Sirainenmail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx,
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_transaction_expunge *rec = data, *end;
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen /* this is simply a request for expunge */
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen const struct mail_transaction_expunge_guid *rec = data, *end;
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
0c909e3461607eadcd66f4eac69b7f34e37fccf1Timo Sirainen /* this is simply a request for expunge */
dfdd228ab20092705da0e8812b3cd0a039319375Timo Sirainen if (mail_index_lookup_seq(ctx->view, rec->uid, &seq))
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_transaction_flag_update *rec, *end;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_transaction_header_update *rec;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen unsigned int i;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if ((i % 4) != 0)
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen const struct mail_transaction_ext_intro *rec = data;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen unsigned int i;
2c677e9d339bc91d5b54376ba2986f71476c06abTimo Sirainen mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen /* should be just extra padding */
8f41f9935b056a26375755a8011340893e7e0802Timo Sirainen /* name_size checked by _log_view_next() */
8f41f9935b056a26375755a8011340893e7e0802Timo Sirainen i_assert(i + sizeof(*rec) + rec->name_size <= hdr->size);
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if ((i % 4) != 0)
6e07b4251bf6a3cf34019c351a32a65c08392e58Timo Sirainen /* old versions have only new_reset_id */
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "ext reset: invalid record size");
6e07b4251bf6a3cf34019c351a32a65c08392e58Timo Sirainen memcpy(&rec, data, I_MIN(hdr->size, sizeof(rec)));
8bb360f9e5de1c25e4f875205bb06e8bf15dae14Timo Sirainen const struct mail_transaction_ext_hdr_update *rec;
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen unsigned int i;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "ext hdr update: invalid record size");
e76f5e07be5bec4e5ca99c3e093ff7f11edbe1b7Timo Sirainen ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
e76f5e07be5bec4e5ca99c3e093ff7f11edbe1b7Timo Sirainen if ((i % 4) != 0)
8bb360f9e5de1c25e4f875205bb06e8bf15dae14Timo Sirainen const struct mail_transaction_ext_hdr_update32 *rec;
e76f5e07be5bec4e5ca99c3e093ff7f11edbe1b7Timo Sirainen unsigned int i;
e76f5e07be5bec4e5ca99c3e093ff7f11edbe1b7Timo Sirainen "ext hdr update: invalid record size");
e76f5e07be5bec4e5ca99c3e093ff7f11edbe1b7Timo Sirainen ret = mail_index_sync_ext_hdr_update(ctx, rec->offset,
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen if ((i % 4) != 0)
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen const struct mail_transaction_ext_rec_update *rec;
86d52f310fe939090c66b780a3b6ffe5d10dc8faTimo Sirainen "Extension record updated "
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen "without intro prefix");
2d79e603e20a32bdae4c2b516ead5c5c9169545aTimo Sirainen /* the record is padded to 32bits in the transaction log */
3403e054eacf125e757bf6c66abf0ea9f086a4b6Timo Sirainen record_size = (sizeof(*rec) + ctx->cur_ext_record_size + 3) & ~3;
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen for (i = 0; i < hdr->size; i += record_size) {
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "ext rec update: invalid record size");
a0d34d3982507f513a9d800082481e9faeb9a943Timo Sirainen ret = mail_index_sync_ext_rec_update(ctx, rec);
c5794838af9995f50bfecb06a3cd4f9a0ac77858Timo Sirainen const struct mail_transaction_ext_atomic_inc *rec, *end;
c5794838af9995f50bfecb06a3cd4f9a0ac77858Timo Sirainen "Extension record updated "
c5794838af9995f50bfecb06a3cd4f9a0ac77858Timo Sirainen "without intro prefix");
c5794838af9995f50bfecb06a3cd4f9a0ac77858Timo Sirainen ret = mail_index_sync_ext_atomic_inc(ctx, rec);
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen const struct mail_transaction_keyword_update *rec = data;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen ret = mail_index_sync_keywords(ctx, hdr, rec);
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen const struct mail_transaction_keyword_reset *rec = data;
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen const struct mail_transaction_modseq_update *rec = data;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen ret = sync_modseq_update(ctx, rec, hdr->size);
d041ddb437ee7000174161405581ab85c0ba314aTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
d041ddb437ee7000174161405581ab85c0ba314aTimo Sirainen /* next sync finishes the deletion */
d041ddb437ee7000174161405581ab85c0ba314aTimo Sirainen ctx->view->index->index_delete_requested = TRUE;
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen /* transaction log reading handles this */
8872e5c991430f96138a46e36b7f3c2c40d8e5c2Timo Sirainen ctx->view->index->index_delete_requested = FALSE;
20261d71760e4199cb8d906ab9704a4561d954d7Timo Sirainen "Unknown transaction record type 0x%x",
6d25194a4230037c0999aa6a01ae6814e03ec06cTimo Sirainenint mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
6d25194a4230037c0999aa6a01ae6814e03ec06cTimo Sirainen ret = mail_index_sync_record_real(ctx, hdr, data);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenvoid mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen sync_map_ctx->modseq_ctx = mail_index_modseq_sync_begin(sync_map_ctx);
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainenvoid mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
6307d76096764e66bddc63d4a3e5a1aa19cc528fJosef 'Jeff' Sipek buffer_free(&sync_map_ctx->unknown_extensions);
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen mail_index_sync_deinit_handlers(sync_map_ctx);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
945565e0c9cf979b5feeba6fbd4efce3bf4484adTimo Sirainen if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 ||
945565e0c9cf979b5feeba6fbd4efce3bf4484adTimo Sirainen (map->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) != 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* do we have dirty flags anymore? */
2f16d2e0b4408370cd44db359759b23a8c0656d3Phil Carmody for (seq = 1; seq <= map->rec_map->records_count; seq++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainenvoid mail_index_map_check(struct mail_index_map *map)
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen const struct mail_index_header *hdr = &map->hdr;
d12f05c7c391786d0d9795ec3aa4377280bbfaeaTimo Sirainen i_assert(hdr->messages_count <= map->rec_map->records_count);
2f16d2e0b4408370cd44db359759b23a8c0656d3Phil Carmody for (seq = 1; seq <= hdr->messages_count; seq++) {
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainenint mail_index_sync_map(struct mail_index_map **_map,
6a4bfb2b0bb9f53fb1d4e705bf3948ef4d1ecccbTimo Sirainen enum mail_index_sync_handler_type type, bool force,
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
cff21b6a2e9e54086544dfdc0e33fe8321e6bf02Timo Sirainen start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
cff21b6a2e9e54086544dfdc0e33fe8321e6bf02Timo Sirainen map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen if (!force && (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* see if we'd prefer to reopen the index file instead of
32339680e0197f50f1f5b40a28099a9e0f19ab23Timo Sirainen syncing the current map from the transaction log.
32339680e0197f50f1f5b40a28099a9e0f19ab23Timo Sirainen don't check this if mmap is disabled, because reopening
32339680e0197f50f1f5b40a28099a9e0f19ab23Timo Sirainen index causes sync to get lost. */
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen /* we don't know the index's size, so use the
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen smallest index size we're willing to read */
d41573018e85896ec836d897fd554e87126147f5Timo Sirainen index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* this isn't necessary correct currently, but it should be
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen close enough */
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen view = mail_index_view_open_with_map(index, map);
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen ret = mail_transaction_log_view_set(view->log_view,
d697aa99bbcd1b0356afbc2b0ab87c88fc7d725cTimo Sirainen /* if we failed because of a syscall error, make sure
d697aa99bbcd1b0356afbc2b0ab87c88fc7d725cTimo Sirainen we return a failure. */
1dec807061d7d428dba5c5a92cd2a5ff843a2039Timo Sirainen /* the seq/offset is probably broken */
db95d1a82dc3ddd93a09ef4720f5855c5a1f34c9Timo Sirainen mail_index_set_error(index, "Index %s: Lost log for "
6a4bfb2b0bb9f53fb1d4e705bf3948ef4d1ecccbTimo Sirainen "(initial_mapped=%d, reason=%s)", index->filepath,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* can't use it. sync by re-reading index. */
686c00553a7cea22272548d9fb8c888170965ec9Timo Sirainen mail_transaction_log_get_head(index->log, &prev_seq, &prev_offset);
3a78329166819e06f2929ce44e360514c6a80a8eTimo Sirainen index->optimization_set.index.rewrite_min_log_bytes) {
686c00553a7cea22272548d9fb8c888170965ec9Timo Sirainen /* we're reading more from log than we would have preferred.
686c00553a7cea22272548d9fb8c888170965ec9Timo Sirainen remember that we probably want to rewrite index soon. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* view referenced the map. avoid unnecessary map cloning by
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unreferencing the map while view exists. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen if (map->hdr_base != map->hdr_copy_buf->data) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* if syncing updates the header, it updates hdr_copy_buf
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen and updates hdr_base to hdr_copy_buf. so the buffer must
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen initially contain a valid header or we'll break it when
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen writing it. */
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen buffer_append(map->hdr_copy_buf, map->hdr_base,
6e07b4251bf6a3cf34019c351a32a65c08392e58Timo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_sync_map_init(&sync_map_ctx, view, type);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* Reset the entire index. Leave only indexid and
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen log_file_seq. */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
02d6628c1fea2990c67c60b111c8e68867160885Timo Sirainen if ((index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0)
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen mail_index_sync_replace_map(&sync_map_ctx, map);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* FIXME: when transaction sync lock is removed, we'll need to handle
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen the case when a transaction is committed while mailbox is being
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen synced ([synced transactions][new transaction][ext transaction]).
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen this means int_offset contains [synced] and ext_offset contains
5a07b37a9df398b5189c14872a600384208ab74bTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen /* this has been synced already. we're here only to call
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen expunge handlers and extension update handlers. */
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* we'll just skip over broken entries */
6d25194a4230037c0999aa6a01ae6814e03ec06cTimo Sirainen (void)mail_index_sync_record(&sync_map_ctx, thdr, tdata);
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen mail_index_modseq_sync_end(&sync_map_ctx.modseq_ctx);
f81a4d2002da0db33d11ca694d3a91b3ee2a0fdbTimo Sirainen mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);
9ae24544b49e9cbf12bfa888279988a97235e10dTimo Sirainen i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen /* transaction log tracks internally the current tail offset.
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen besides using header updates, it also updates the offset to skip
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen over following external transactions to avoid extra unneeded log
88b6f36ef1b453c08e8d9cadb229b39fc9bb4a1cTimo Sirainen i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq);
88b6f36ef1b453c08e8d9cadb229b39fc9bb4a1cTimo Sirainen if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) {
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
3697080532ccd9f51fac108be6079b616c7a2ddfTimo Sirainen memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* restore refcount before closing the view. this is necessary also
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if map got cloned, because view closing would otherwise destroy it */
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen if (mail_index_map_check_header(map, &error) <= 0) {
45b0d8d0b97be14d10e3a3c12c169e4b352b2aacTimo Sirainen "Synchronization corrupted index header %s: %s",
f31720ad5ece28ca3fa793920f1501165ad9603fTimo Sirainen /* make sure the index looks valid now */