mail-index-view-sync.c revision ffb79efcac37f807f75ab0a04c7b17a3d4af7167
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2003-2012 Dovecot authors, see the included COPYING file */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen#include "lib.h"
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen#include "array.h"
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen#include "buffer.h"
dd3ccdbb29dad006f7781ea138a5ba39727963c4Timo Sirainen#include "mail-index-view-private.h"
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen#include "mail-index-sync-private.h"
419be6cd72f6e11705576bbba683b29c32eaa762Timo Sirainen#include "mail-index-modseq.h"
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen#include "mail-transaction-log.h"
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen
dd3ccdbb29dad006f7781ea138a5ba39727963c4Timo Sirainen#include <stdlib.h>
c4390dad33b03dd51ba2a475f550347c86ebdb9aTimo Sirainen
dd3ccdbb29dad006f7781ea138a5ba39727963c4Timo Sirainenstruct mail_index_view_sync_ctx {
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen struct mail_index_view *view;
6b6011c2242e470b41316f92512b282b5e306dacTimo Sirainen enum mail_index_view_sync_flags flags;
18458233e89c8467e30d8b93b1823128f26bc174Timo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;
ffaa309c211897ab875bbe0b093bc7e709bb1e5dTimo Sirainen
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen /* After syncing view, map is replaced with sync_new_map. */
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen struct mail_index_map *sync_new_map;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen ARRAY_TYPE(seq_range) expunges;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen unsigned int finish_min_msg_count;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen const struct mail_transaction_header *hdr;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen const void *data;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* temporary variables while handling lost transaction logs: */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw;
458acd7b39c84bae0d18c36ff9ddff9a49b4ae4aTimo Sirainen buffer_t *lost_kw_buf;
78a5b3e697af5db96fe0dffed600b0d6370bb8e5Timo Sirainen uint32_t lost_new_ext_idx;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* result of lost transaction logs: */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen ARRAY_TYPE(seq_range) lost_flags;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen unsigned int lost_flag_idx;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen size_t data_offset;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen unsigned int failed:1;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen unsigned int sync_map_update:1;
3b22894b8805b186c73d8b754001e8d7e944be85Timo Sirainen unsigned int skipped_expunges:1;
3b22894b8805b186c73d8b754001e8d7e944be85Timo Sirainen unsigned int last_read:1;
3b22894b8805b186c73d8b754001e8d7e944be85Timo Sirainen unsigned int log_was_lost:1;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen unsigned int hidden:1;
a50f8b9c6196b0f0bb0a7aac3ae9c12b1c42b93bTimo Sirainen};
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainenstatic int
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainenview_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges,
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen bool *reset_r)
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen{
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen const struct mail_index_header *hdr = &view->index->map->hdr;
89502bb187e8285b2a155559894ca80374ac3ae7Timo Sirainen uint32_t start_seq, end_seq;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen uoff_t start_offset, end_offset;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen int ret;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen start_seq = view->log_file_expunge_seq;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen start_offset = view->log_file_expunge_offset;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen end_seq = hdr->log_file_seq;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen end_offset = hdr->log_file_head_offset;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen if (end_seq < view->log_file_head_seq ||
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen (end_seq == view->log_file_head_seq &&
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen end_offset < view->log_file_head_offset)) {
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen mail_index_set_error(view->index,
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen "%s log position went backwards "
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen "(%u,%"PRIuUOFF_T" < %u,%"PRIuUOFF_T")",
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen view->index->filepath, end_seq, end_offset,
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen view->log_file_head_seq, view->log_file_head_offset);
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen return -1;
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen }
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen for (;;) {
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen /* the view begins from the first non-synced transaction */
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen ret = mail_transaction_log_view_set(view->log_view,
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen start_seq, start_offset,
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen end_seq, end_offset,
d65cc3312d3126d34b51ae8eccd6b48215d50029Timo Sirainen reset_r);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (ret <= 0)
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen return ret;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen if (!*reset_r || sync_expunges)
b015d3d3af5b1c0fd526f31d78229bf9f633db56Timo Sirainen break;
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* log was reset, but we don't want to sync expunges.
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen we can't do this, so sync only up to the reset. */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen &end_seq, &end_offset);
fcd443a32b01c4da131f36649d5a5fa5f8452dcfTimo Sirainen end_seq--; end_offset = (uoff_t)-1;
fcd443a32b01c4da131f36649d5a5fa5f8452dcfTimo Sirainen if (end_seq < start_seq) {
d65cc3312d3126d34b51ae8eccd6b48215d50029Timo Sirainen /* we have only this reset log */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen mail_transaction_log_view_clear(view->log_view,
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen view->log_file_expunge_seq);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen break;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen }
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen }
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen return 1;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen}
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainenstatic unsigned int
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainenview_sync_expunges2seqs(struct mail_index_view_sync_ctx *ctx)
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen{
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen struct mail_index_view *view = ctx->view;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen struct seq_range *src, *src_end, *dest;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen unsigned int count, expunge_count = 0;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen uint32_t prev_seq = 0;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen /* convert UIDs to sequences */
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen src = dest = array_get_modifiable(&ctx->expunges, &count);
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen src_end = src + count;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen for (; src != src_end; src++) {
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen &dest->seq1, &dest->seq2))
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen count--;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen else {
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen i_assert(dest->seq1 > prev_seq);
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen prev_seq = dest->seq2;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen expunge_count += dest->seq2 - dest->seq1 + 1;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen dest++;
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen }
4fbf59562594dbbbe037f8d4c480dbf88f3fc708Timo Sirainen }
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen array_delete(&ctx->expunges, count,
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen array_count(&ctx->expunges) - count);
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen return expunge_count;
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen}
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainenstatic void
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainenview_sync_add_expunge_range(ARRAY_TYPE(seq_range) *dest,
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen const struct seq_range *src, size_t src_size)
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen{
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen unsigned int i, src_count;
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen i_assert(src_size % sizeof(*src) == 0);
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen src_count = src_size / sizeof(*src);
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen for (i = 0; i < src_count; i++)
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen seq_range_array_add_range(dest, src[i].seq1, src[i].seq2);
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen}
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainenstatic void
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainenview_sync_add_expunge_guids(ARRAY_TYPE(seq_range) *dest,
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen const struct mail_transaction_expunge_guid *src,
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen size_t src_size)
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen{
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen unsigned int i, src_count;
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen i_assert(src_size % sizeof(*src) == 0);
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen src_count = src_size / sizeof(*src);
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen for (i = 0; i < src_count; i++)
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen seq_range_array_add(dest, 0, src[i].uid);
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen}
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainenstatic int
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainenview_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen unsigned int *expunge_count_r)
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen{
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen struct mail_index_view *view = ctx->view;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen const struct mail_transaction_header *hdr;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen const void *data;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen int ret;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen /* get a list of expunge transactions. there may be some that we have
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen already synced, but it doesn't matter because they'll get dropped
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen out when converting to sequences. the uid ranges' validity has
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen already been verified, so we can use them directly. */
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen mail_transaction_log_view_mark(view->log_view);
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen &hdr, &data)) > 0) {
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen /* skip expunge requests */
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen continue;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen }
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) {
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen view_sync_add_expunge_guids(&ctx->expunges,
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen data, hdr->size);
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen } else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen view_sync_add_expunge_range(&ctx->expunges,
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen data, hdr->size);
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen }
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen }
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen mail_transaction_log_view_rewind(view->log_view);
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen *expunge_count_r = view_sync_expunges2seqs(ctx);
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen return ret;
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen}
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainenstatic bool have_existing_expunges(struct mail_index_view *view,
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen const struct seq_range *range, size_t size)
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen{
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen const struct seq_range *range_end;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen uint32_t seq1, seq2;
65514ab6ccc1889e1667211fddb0cca4b51017dfTimo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen range_end = CONST_PTR_OFFSET(range, size);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen for (; range != range_end; range++) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (mail_index_lookup_seq_range(view, range->seq1, range->seq2,
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen &seq1, &seq2))
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen return TRUE;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen }
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen return FALSE;
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen}
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainenstatic bool
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainenhave_existing_guid_expunge(struct mail_index_view *view,
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen const struct mail_transaction_expunge_guid *expunges,
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen size_t size)
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen{
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen const struct mail_transaction_expunge_guid *expunges_end;
e93ab1b206e3792a8dabc460ad2ee60aaf6830b1Timo Sirainen uint32_t seq;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
e93ab1b206e3792a8dabc460ad2ee60aaf6830b1Timo Sirainen expunges_end = CONST_PTR_OFFSET(expunges, size);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen for (; expunges != expunges_end; expunges++) {
65514ab6ccc1889e1667211fddb0cca4b51017dfTimo Sirainen if (mail_index_lookup_seq(view, expunges->uid, &seq))
65514ab6ccc1889e1667211fddb0cca4b51017dfTimo Sirainen return TRUE;
65514ab6ccc1889e1667211fddb0cca4b51017dfTimo Sirainen }
65514ab6ccc1889e1667211fddb0cca4b51017dfTimo Sirainen return FALSE;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen}
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainenstatic bool view_sync_have_expunges(struct mail_index_view *view)
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen{
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen const struct mail_transaction_header *hdr;
d65cc3312d3126d34b51ae8eccd6b48215d50029Timo Sirainen const void *data;
6b6011c2242e470b41316f92512b282b5e306dacTimo Sirainen bool have_expunges = FALSE;
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen int ret;
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen mail_transaction_log_view_mark(view->log_view);
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen
6de95f63d4590814354fa5e35e92946c58d4562eTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
6b6011c2242e470b41316f92512b282b5e306dacTimo Sirainen &hdr, &data)) > 0) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* skip expunge requests */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen continue;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen }
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) {
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen /* we have an expunge. see if it still exists. */
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if (have_existing_guid_expunge(view, data, hdr->size)) {
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen have_expunges = TRUE;
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen break;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen }
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen } else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen /* we have an expunge. see if it still exists. */
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if (have_existing_expunges(view, data, hdr->size)) {
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen have_expunges = TRUE;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen break;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen }
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen }
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen }
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen mail_transaction_log_view_rewind(view->log_view);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen /* handle failures as having expunges (which is safer).
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen we'll probably fail later. */
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen return ret < 0 || have_expunges;
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen}
009217abb57a24a4076092e8e4e165545747839eStephan Bosch
009217abb57a24a4076092e8e4e165545747839eStephan Boschstatic int uint_cmp(const void *p1, const void *p2)
009217abb57a24a4076092e8e4e165545747839eStephan Bosch{
009217abb57a24a4076092e8e4e165545747839eStephan Bosch const unsigned int *u1 = p1, *u2 = p2;
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch if (*u1 < *u2)
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch return -1;
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch if (*u1 > *u2)
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch return 1;
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch return 0;
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch}
c93aca832ee532010ead91b85fa9f614132e1be2Stephan Bosch
accba2f170529d73097e098d0bb6c13b7b4d6abfTimo Sirainenstatic bool view_sync_lost_keywords_equal(struct mail_index_view_sync_ctx *ctx)
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen{
dfe53c824fb6f6e3736ee2958665dd6f545724b1Wido den Hollander unsigned int *old_idx, *new_idx;
dfe53c824fb6f6e3736ee2958665dd6f545724b1Wido den Hollander unsigned int old_count, new_count;
accba2f170529d73097e098d0bb6c13b7b4d6abfTimo Sirainen
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen old_idx = array_get_modifiable(&ctx->lost_old_kw, &old_count);
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen new_idx = array_get_modifiable(&ctx->lost_new_kw, &new_count);
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen if (old_count != new_count)
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen return FALSE;
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp);
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp);
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen return memcmp(old_idx, new_idx, old_count * sizeof(old_idx)) == 0;
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen}
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainenstatic int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx,
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen uint32_t uid)
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen{
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen struct mail_transaction_header thdr;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen struct mail_transaction_keyword_update kw_up;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen const unsigned int *kw_idx;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen const char *const *kw_names;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen unsigned int i, count;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen kw_idx = array_get(&ctx->lost_new_kw, &count);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if (count == 0)
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen return 0;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen kw_names = array_idx(&ctx->view->index->keywords, 0);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen memset(&thdr, 0, sizeof(thdr));
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen thdr.type = MAIL_TRANSACTION_KEYWORD_UPDATE | MAIL_TRANSACTION_EXTERNAL;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen memset(&kw_up, 0, sizeof(kw_up));
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen kw_up.modify_type = MODIFY_ADD;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen /* add new flags one by one */
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen for (i = 0; i < count; i++) {
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen kw_up.name_size = strlen(kw_names[kw_idx[i]]);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen buffer_set_used_size(ctx->lost_kw_buf, 0);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen buffer_append(ctx->lost_kw_buf, &kw_up, sizeof(kw_up));
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen buffer_append(ctx->lost_kw_buf, kw_names[kw_idx[i]],
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen kw_up.name_size);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if (ctx->lost_kw_buf->used % 4 != 0) {
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen buffer_append_zero(ctx->lost_kw_buf,
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen 4 - ctx->lost_kw_buf->used % 4);
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen }
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen thdr.size = ctx->lost_kw_buf->used;
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen ctx->lost_kw_buf->data) < 0)
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen return -1;
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen }
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen return 0;
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen}
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainenstatic int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen uint32_t old_seq, uint32_t new_seq)
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen{
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen struct mail_index_map *old_map = ctx->view->map;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen struct mail_index_map *new_map = ctx->view->index->map;
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller const struct mail_index_record *old_rec, *new_rec;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen struct mail_transaction_header thdr;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen const struct mail_index_ext *ext;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen const uint64_t *modseqp;
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen uint64_t new_modseq;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen bool changed = FALSE;
86fafb22c02f4b85a9d59f2b60059a48286286fbTimo Sirainen
3b22894b8805b186c73d8b754001e8d7e944be85Timo Sirainen old_rec = MAIL_INDEX_MAP_IDX(old_map, old_seq - 1);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen new_rec = MAIL_INDEX_MAP_IDX(new_map, new_seq - 1);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen memset(&thdr, 0, sizeof(thdr));
86fafb22c02f4b85a9d59f2b60059a48286286fbTimo Sirainen if (old_rec->flags != new_rec->flags) {
86fafb22c02f4b85a9d59f2b60059a48286286fbTimo Sirainen struct mail_transaction_flag_update flag_update;
86fafb22c02f4b85a9d59f2b60059a48286286fbTimo Sirainen
86fafb22c02f4b85a9d59f2b60059a48286286fbTimo Sirainen /* check this before syncing the record, since it updates
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen old_rec. */
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if ((old_rec->flags & MAIL_INDEX_FLAGS_MASK) !=
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen (new_rec->flags & MAIL_INDEX_FLAGS_MASK))
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen changed = TRUE;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen thdr.type = MAIL_TRANSACTION_FLAG_UPDATE |
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen MAIL_TRANSACTION_EXTERNAL;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen thdr.size = sizeof(flag_update);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen memset(&flag_update, 0, sizeof(flag_update));
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen flag_update.uid1 = flag_update.uid2 = new_rec->uid;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen flag_update.add_flags = new_rec->flags;
3b22894b8805b186c73d8b754001e8d7e944be85Timo Sirainen flag_update.remove_flags = ~new_rec->flags & 0xff;
3b22894b8805b186c73d8b754001e8d7e944be85Timo Sirainen if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen &flag_update) < 0)
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen return -1;
3b22894b8805b186c73d8b754001e8d7e944be85Timo Sirainen }
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen mail_index_map_lookup_keywords(old_map, old_seq, &ctx->lost_old_kw);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen mail_index_map_lookup_keywords(new_map, new_seq, &ctx->lost_new_kw);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if (!view_sync_lost_keywords_equal(ctx)) {
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen struct mail_transaction_keyword_reset kw_reset;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen thdr.type = MAIL_TRANSACTION_KEYWORD_RESET |
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen MAIL_TRANSACTION_EXTERNAL;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen thdr.size = sizeof(kw_reset);
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen /* remove all old flags by resetting them */
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen memset(&kw_reset, 0, sizeof(kw_reset));
eed0a07ecb946ec9d021f5b413fb33eb36e135fdTimo Sirainen kw_reset.uid1 = kw_reset.uid2 = new_rec->uid;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen &kw_reset) < 0)
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen return -1;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen view_sync_update_keywords(ctx, new_rec->uid);
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen changed = TRUE;
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen }
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen if (changed) {
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen /* flags or keywords changed */
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller } else if (ctx->view->highest_modseq != 0 &&
d2a2e0b6d302c17c33638a6fd48bd665a1c81e46Timo Sirainen ctx->lost_new_ext_idx != (uint32_t)-1) {
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen /* if modseq has changed include this message in changed flags
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen list, even if we didn't see any changes above. */
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen ext = array_idx(&new_map->extensions, ctx->lost_new_ext_idx);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen modseqp = CONST_PTR_OFFSET(new_rec, ext->record_offset);
7aa2dd189748ed57c2fb8d8d9b7cb3c4ac2f7ea5Timo Sirainen new_modseq = *modseqp;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if (new_modseq > ctx->view->highest_modseq)
9508ac436fff0e1dcea975855c139cd251deb703Timo Sirainen changed = TRUE;
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen }
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen /* without modseqs lost_flags isn't updated perfectly correctly, because
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen by the time we're comparing old flags it may have changed from what
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen we last sent to the client (because the map is shared). This could
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen be avoided by always keeping a private copy of the map in the view,
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen but that's a waste of memory for as rare of a problem as this. */
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen if (changed)
7aa2dd189748ed57c2fb8d8d9b7cb3c4ac2f7ea5Timo Sirainen seq_range_array_add(&ctx->lost_flags, 0, new_rec->uid);
7aa2dd189748ed57c2fb8d8d9b7cb3c4ac2f7ea5Timo Sirainen return 0;
7aa2dd189748ed57c2fb8d8d9b7cb3c4ac2f7ea5Timo Sirainen}
7aa2dd189748ed57c2fb8d8d9b7cb3c4ac2f7ea5Timo Sirainen
7aa2dd189748ed57c2fb8d8d9b7cb3c4ac2f7ea5Timo Sirainenstatic int
7aa2dd189748ed57c2fb8d8d9b7cb3c4ac2f7ea5Timo Sirainenview_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx,
7aa2dd189748ed57c2fb8d8d9b7cb3c4ac2f7ea5Timo Sirainen unsigned int *expunge_count_r)
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen{
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen struct mail_index_view *view = ctx->view;
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen struct mail_index_map *old_map = view->map;
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen struct mail_index_map *new_map = view->index->map;
32efae185f1f86167b3f00ea84f8502940c6c677Timo Sirainen const unsigned int old_count = old_map->hdr.messages_count;
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen const unsigned int new_count = new_map->hdr.messages_count;
d163fac99c7ca5b2ce2edfa83a1b2922cf66aeacTimo Sirainen const struct mail_index_record *old_rec, *new_rec;
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen struct mail_transaction_header thdr;
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen unsigned int i, j;
ddbdc644a15f56f4b43596f1b8c0fc196c101445Timo Sirainen
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen /* we don't update the map in the same order as it's typically done.
3322dfd56c9cae5015a33bab4c80124c08c3d9caTimo Sirainen map->rec_map may already have some messages appended that we don't
e93ab1b206e3792a8dabc460ad2ee60aaf6830b1Timo Sirainen want. get an atomic map to make sure these get removed. */
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen (void)mail_index_sync_get_atomic_map(&ctx->sync_map_ctx);
e93ab1b206e3792a8dabc460ad2ee60aaf6830b1Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if (!mail_index_map_get_ext_idx(new_map, view->index->modseq_ext_id,
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen &ctx->lost_new_ext_idx))
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen ctx->lost_new_ext_idx = (uint32_t)-1;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen i_array_init(&ctx->lost_flags, 64);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen t_array_init(&ctx->lost_old_kw, 32);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen t_array_init(&ctx->lost_new_kw, 32);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen ctx->lost_kw_buf = buffer_create_dynamic(pool_datastack_create(), 128);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen /* handle expunges and sync flags */
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen i = j = 0;
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen while (i < old_count && j < new_count) {
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen old_rec = MAIL_INDEX_MAP_IDX(old_map, i);
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen new_rec = MAIL_INDEX_MAP_IDX(new_map, j);
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen if (old_rec->uid == new_rec->uid) {
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen /* message found - check if flags have changed */
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen if (view_sync_apply_lost_changes(ctx, i + 1, j + 1) < 0)
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen return -1;
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen i++; j++;
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen } else if (old_rec->uid < new_rec->uid) {
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen /* message expunged */
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen seq_range_array_add(&ctx->expunges, 0, old_rec->uid);
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen i++;
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen } else {
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen /* new message appeared out of nowhere */
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen mail_index_set_error(view->index,
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen "%s view is inconsistent: "
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen "uid=%u inserted in the middle of mailbox",
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen view->index->filepath, new_rec->uid);
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen return -1;
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen }
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen }
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen /* if there are old messages left, they're all expunged */
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen for (; i < old_count; i++) {
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen old_rec = MAIL_INDEX_MAP_IDX(old_map, i);
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen seq_range_array_add(&ctx->expunges, 0, old_rec->uid);
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen }
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen /* if there are new messages left, they're all new messages */
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen thdr.type = MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL;
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen thdr.size = sizeof(*new_rec);
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen for (; j < new_count; j++) {
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen new_rec = MAIL_INDEX_MAP_IDX(new_map, j);
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen if (mail_index_sync_record(&ctx->sync_map_ctx,
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen &thdr, new_rec) < 0)
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen return -1;
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen mail_index_map_lookup_keywords(new_map, j + 1,
a338794c56fc9674121e262fcb67c3dc1da31436Timo Sirainen &ctx->lost_new_kw);
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen view_sync_update_keywords(ctx, new_rec->uid);
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller }
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller *expunge_count_r = view_sync_expunges2seqs(ctx);
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen
7b22071c71a99bcb71ee9fcfbb83ed5bf5e08a2dTimo Sirainen /* we have no idea how far we've synced - make sure these aren't used */
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller old_map->hdr.log_file_seq = 0;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen old_map->hdr.log_file_head_offset = 0;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen old_map->hdr.log_file_tail_offset = 0;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
32efae185f1f86167b3f00ea84f8502940c6c677Timo Sirainen array_clear(&ctx->expunges);
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen ctx->skipped_expunges = *expunge_count_r > 0;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen } else {
7b22071c71a99bcb71ee9fcfbb83ed5bf5e08a2dTimo Sirainen view->log_file_head_seq = new_map->hdr.log_file_seq;
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller view->log_file_head_offset = new_map->hdr.log_file_head_offset;
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller }
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller return 0;
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller}
7b22071c71a99bcb71ee9fcfbb83ed5bf5e08a2dTimo Sirainen
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Wellerstatic int mail_index_view_sync_init_fix(struct mail_index_view_sync_ctx *ctx)
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen{
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen struct mail_index_view *view = ctx->view;
7b22071c71a99bcb71ee9fcfbb83ed5bf5e08a2dTimo Sirainen uint32_t seq;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen uoff_t offset;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen bool reset;
7b22071c71a99bcb71ee9fcfbb83ed5bf5e08a2dTimo Sirainen
7b22071c71a99bcb71ee9fcfbb83ed5bf5e08a2dTimo Sirainen /* replace the view's map */
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen view->index->map->refcount++;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen mail_index_unmap(&view->map);
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen view->map = view->index->map;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen /* update log positions */
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen view->log_file_head_seq = seq = view->map->hdr.log_file_seq;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen view->log_file_head_offset = offset =
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen view->map->hdr.log_file_head_offset;
2ac5f36aa7c2e7a07ba8815d43a6d7483f62e74cTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen if (mail_transaction_log_view_set(view->log_view, seq, offset,
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen seq, offset, &reset) <= 0)
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen return -1;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen view->inconsistent = FALSE;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen return 0;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen}
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainenstruct mail_index_view_sync_ctx *
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainenmail_index_view_sync_begin(struct mail_index_view *view,
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen enum mail_index_view_sync_flags flags)
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen{
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen struct mail_index_view_sync_ctx *ctx;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen struct mail_index_map *tmp_map;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen unsigned int expunge_count = 0;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen bool reset, sync_expunges, have_expunges;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen int ret;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen i_assert(!view->syncing);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen i_assert(view->transactions == 0);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen view->syncing = TRUE;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen /* Syncing the view invalidates all previous looked up records.
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen Unreference the mappings this view keeps because of them. */
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen mail_index_view_unref_maps(view);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ctx = i_new(struct mail_index_view_sync_ctx, 1);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ctx->view = view;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ctx->flags = flags;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen if (sync_expunges)
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen i_array_init(&ctx->expunges, 64);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) != 0) {
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen /* just get this view synced - don't return anything */
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen i_assert(sync_expunges);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen if (mail_index_view_sync_init_fix(ctx) < 0)
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ctx->failed = TRUE;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen return ctx;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen }
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen if (mail_index_view_is_inconsistent(view)) {
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen mail_index_set_error(view->index, "%s view is inconsistent",
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen view->index->filepath);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ctx->failed = TRUE;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen return ctx;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen }
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ret = view_sync_set_log_view_range(view, sync_expunges, &reset);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen if (ret < 0) {
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ctx->failed = TRUE;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen return ctx;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen }
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen if (ret == 0) {
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ctx->log_was_lost = TRUE;
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen if (!sync_expunges)
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen i_array_init(&ctx->expunges, 64);
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen mail_index_sync_map_init(&ctx->sync_map_ctx, view,
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen MAIL_INDEX_SYNC_HANDLER_VIEW);
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen ret = view_sync_get_log_lost_changes(ctx, &expunge_count);
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen mail_index_sync_map_deinit(&ctx->sync_map_ctx);
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen if (ret < 0) {
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen mail_index_set_error(view->index,
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen "%s view syncing failed to apply changes",
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen view->index->filepath);
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen view->inconsistent = TRUE;
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen ctx->failed = TRUE;
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen return ctx;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen }
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen have_expunges = expunge_count > 0;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen } else if (sync_expunges) {
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen /* get list of all expunges first */
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen if (view_sync_get_expunges(ctx, &expunge_count) < 0) {
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen ctx->failed = TRUE;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen return ctx;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen }
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen have_expunges = expunge_count > 0;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen } else {
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen have_expunges = view_sync_have_expunges(view);
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen }
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen ctx->finish_min_msg_count = reset ? 0 :
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen view->map->hdr.messages_count - expunge_count;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen if (reset && view->map->hdr.messages_count > 0) {
61d3fd14828b68d789f3df73d1dbed56e37b7931Timo Sirainen view->inconsistent = TRUE;
61d3fd14828b68d789f3df73d1dbed56e37b7931Timo Sirainen mail_index_set_error(view->index,
61d3fd14828b68d789f3df73d1dbed56e37b7931Timo Sirainen "%s reset, view is now inconsistent",
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen view->index->filepath);
61d3fd14828b68d789f3df73d1dbed56e37b7931Timo Sirainen }
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen if (!have_expunges) {
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen /* no expunges, we can just replace the map */
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen if (view->index->map->hdr.messages_count <
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen ctx->finish_min_msg_count) {
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen mail_index_set_error(view->index,
2ed2459dbd183bb371da4a0aecb2d2b74ae7c815Timo Sirainen "Index %s lost messages without expunging "
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen "(%u -> %u)", view->index->filepath,
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen view->map->hdr.messages_count,
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen view->index->map->hdr.messages_count);
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen ctx->finish_min_msg_count = 0;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen view->inconsistent = TRUE;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen }
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen view->index->map->refcount++;
02e8b5f55210f413853998a0ba5832eb3f9a6dc6Timo Sirainen mail_index_unmap(&view->map);
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen view->map = view->index->map;
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen } else {
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen /* expunges seen. create a private map which we update.
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen if we're syncing expunges the map will finally be replaced
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen with the head map to remove the expunged messages. */
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen ctx->sync_map_update = TRUE;
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen if (view->map->refcount > 1) {
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen tmp_map = mail_index_map_clone(view->map);
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen mail_index_unmap(&view->map);
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen view->map = tmp_map;
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen }
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen if (sync_expunges) {
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen ctx->sync_new_map = view->index->map;
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen ctx->sync_new_map->refcount++;
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen }
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen }
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen mail_index_sync_map_init(&ctx->sync_map_ctx, view,
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen MAIL_INDEX_SYNC_HANDLER_VIEW);
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen
553a131eddb46d97967fd9610408b7f0829eab6fTimo Sirainen#ifdef DEBUG
3edfbca796edb8f1efeb48da916e109850adfc0fTimo Sirainen mail_index_map_check(view->map);
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen#endif
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen return ctx;
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen}
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainenstatic bool
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainenview_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen{
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen const struct mail_index_view_log_sync_area *sync;
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen if (!array_is_created(&view->syncs_hidden))
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen return FALSE;
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen array_foreach(&view->syncs_hidden, sync) {
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainen if (sync->log_file_offset <= offset &&
553a131eddb46d97967fd9610408b7f0829eab6fTimo Sirainen offset - sync->log_file_offset < sync->length &&
553a131eddb46d97967fd9610408b7f0829eab6fTimo Sirainen sync->log_file_seq == seq)
553a131eddb46d97967fd9610408b7f0829eab6fTimo Sirainen return TRUE;
553a131eddb46d97967fd9610408b7f0829eab6fTimo Sirainen }
553a131eddb46d97967fd9610408b7f0829eab6fTimo Sirainen
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen return FALSE;
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen}
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainenstatic bool
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainenmail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen const struct mail_transaction_header *hdr)
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen{
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen struct mail_index_view *view = ctx->view;
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen uint32_t seq;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen uoff_t offset, next_offset;
f6008a666d9c3df66fd8a437369133e8519b4e24Timo Sirainen
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen next_offset = offset + sizeof(*hdr) + hdr->size;
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE |
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen MAIL_TRANSACTION_EXPUNGE_GUID)) != 0 &&
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen i_assert(!LOG_IS_BEFORE(seq, offset,
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen view->log_file_expunge_seq,
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen view->log_file_expunge_offset));
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen if (!ctx->skipped_expunges) {
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen view->log_file_expunge_seq = seq;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen view->log_file_expunge_offset = offset;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen ctx->skipped_expunges = TRUE;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen }
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen return FALSE;
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen }
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
e5b06a21b5f857d7037fd6ff41ba3bd449d1232aTimo Sirainen view->log_file_expunge_offset)) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* already synced */
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen return FALSE;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen }
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen }
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen view->log_file_head_offset)) {
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen /* already synced */
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen return FALSE;
65514ab6ccc1889e1667211fddb0cca4b51017dfTimo Sirainen }
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller
65514ab6ccc1889e1667211fddb0cca4b51017dfTimo Sirainen view->log_file_head_seq = seq;
d65cc3312d3126d34b51ae8eccd6b48215d50029Timo Sirainen view->log_file_head_offset = next_offset;
d65cc3312d3126d34b51ae8eccd6b48215d50029Timo Sirainen return TRUE;
458acd7b39c84bae0d18c36ff9ddff9a49b4ae4aTimo Sirainen}
972c9172e9e6a0fc6053efb3d2ee9d354b67727fLennart Weller
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainenstatic int
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainenmail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen{
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen struct mail_transaction_log_view *log_view = ctx->view->log_view;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen struct mail_index_view *view = ctx->view;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen const struct mail_transaction_header *hdr;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen uint32_t seq;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen uoff_t offset;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen int ret;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen bool synced_to_map;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen do {
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen /* Get the next transaction from log. */
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen &ctx->data);
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen if (ret <= 0) {
1e217aa0dad9bbfad64525396c3f226f170f985cTimo Sirainen if (ret < 0)
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen return -1;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen ctx->hdr = NULL;
d16cbc7557bc2e9f790c9c907212eb5df57a4ab2Timo Sirainen ctx->last_read = TRUE;
d16cbc7557bc2e9f790c9c907212eb5df57a4ab2Timo Sirainen return 0;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen }
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen hdr = ctx->hdr;
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen /* skip records we've already synced */
c2ebc8f28b5504f280cd5d4adfe57ed70f9a7d83Timo Sirainen } while (!mail_index_view_sync_want(ctx, hdr));
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
f6008a666d9c3df66fd8a437369133e8519b4e24Timo Sirainen mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
f6008a666d9c3df66fd8a437369133e8519b4e24Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* If we started from a map that we didn't create ourself,
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen some of the transactions may already be synced. at the end
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen of this view sync we'll update file_seq=0 so that this check
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen always becomes FALSE for subsequent syncs. */
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen synced_to_map = view->map->hdr.log_file_seq != 0 &&
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen LOG_IS_BEFORE(seq, offset, view->map->hdr.log_file_seq,
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen view->map->hdr.log_file_head_offset);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen /* Apply transaction to view's mapping if needed (meaning we
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen didn't just re-map the view to head mapping). */
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen if (ctx->sync_map_update && !synced_to_map) {
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE |
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen MAIL_TRANSACTION_EXPUNGE_GUID)) == 0) {
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen ret = mail_index_sync_record(&ctx->sync_map_ctx,
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen hdr, ctx->data);
191a22ee13135fd752fcf03a43ef2f1b070937f5Timo Sirainen }
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen if (ret < 0)
a795ff053b41be95cb94a33cd5501f8d2e843e20Timo Sirainen return -1;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen }
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen
e7b1f9db0128384b152b58ea65303d66484c9d7bTimo Sirainen ctx->hidden = view_sync_is_hidden(view, seq, offset);
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen return 1;
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen}
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainenstatic bool
bc37bab86f0ae7b9d7149d2418340b88e6ef212fTimo Sirainenmail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen struct mail_index_view_sync_rec *rec)
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen{
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen const struct mail_transaction_header *hdr = ctx->hdr;
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen const void *data = ctx->data;
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen case MAIL_TRANSACTION_FLAG_UPDATE: {
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen const struct mail_transaction_flag_update *update =
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen CONST_PTR_OFFSET(data, ctx->data_offset);
ced943b0a9b49a5be38516302fe1631c1883debaTimo Sirainen
ddf6020f6fefe71c4422a8c55d012a62648ae3adTimo Sirainen /* data contains mail_transaction_flag_update[] */
42abccd9b2a5a4190bd3c14ec2dcc10d51c0f491Timo Sirainen for (;;) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen ctx->data_offset += sizeof(*update);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(update))
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen break;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
2ba9851abbaffe9592c06c0060dce40bb6da3b78Timo Sirainen /* skip internal flag changes */
2ba9851abbaffe9592c06c0060dce40bb6da3b78Timo Sirainen if (ctx->data_offset == ctx->hdr->size)
2ba9851abbaffe9592c06c0060dce40bb6da3b78Timo Sirainen return FALSE;
2ba9851abbaffe9592c06c0060dce40bb6da3b78Timo Sirainen
2ba9851abbaffe9592c06c0060dce40bb6da3b78Timo Sirainen update = CONST_PTR_OFFSET(data, ctx->data_offset);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen }
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen rec->uid1 = update->uid1;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen rec->uid2 = update->uid2;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen break;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen }
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen case MAIL_TRANSACTION_KEYWORD_UPDATE: {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen const struct mail_transaction_keyword_update *update = data;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen const uint32_t *uids;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
d65cc3312d3126d34b51ae8eccd6b48215d50029Timo Sirainen /* data contains mail_transaction_keyword_update header,
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen the keyword name and an array of { uint32_t uid1, uid2; } */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (ctx->data_offset == 0) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* skip over the header and name */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen ctx->data_offset = sizeof(*update) + update->name_size;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if ((ctx->data_offset % 4) != 0)
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen ctx->data_offset += 4 - (ctx->data_offset % 4);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen }
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen uids = CONST_PTR_OFFSET(data, ctx->data_offset);
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen rec->uid1 = uids[0];
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen rec->uid2 = uids[1];
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen ctx->data_offset += sizeof(uint32_t) * 2;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen break;
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen }
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen case MAIL_TRANSACTION_KEYWORD_RESET: {
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen const struct mail_transaction_keyword_reset *reset =
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen CONST_PTR_OFFSET(data, ctx->data_offset);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen /* data contains mail_transaction_keyword_reset[] */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen rec->uid1 = reset->uid1;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen rec->uid2 = reset->uid2;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen ctx->data_offset += sizeof(*reset);
7ffa9e9809f10884a9167c19c09c980e3c1792b1Timo Sirainen break;
8c30a8e508dd05b63c9b1fa7ae9c637d132dac6fTimo Sirainen }
16dfe1f9c1c1a5abc5e5a12efa25619586f7e4e6Timo Sirainen default:
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen ctx->hdr = NULL;
da0420f1b5d5f5ab1079d1204d70dd51866ce025Timo Sirainen return FALSE;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen }
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen
18458233e89c8467e30d8b93b1823128f26bc174Timo Sirainen rec->hidden = ctx->hidden;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen return TRUE;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen}
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainenstatic bool
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainenmail_index_view_sync_next_lost(struct mail_index_view_sync_ctx *ctx,
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen struct mail_index_view_sync_rec *sync_rec)
7ffa9e9809f10884a9167c19c09c980e3c1792b1Timo Sirainen{
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen const struct seq_range *range;
7ffa9e9809f10884a9167c19c09c980e3c1792b1Timo Sirainen unsigned int count;
7ffa9e9809f10884a9167c19c09c980e3c1792b1Timo Sirainen
7ffa9e9809f10884a9167c19c09c980e3c1792b1Timo Sirainen range = array_get(&ctx->lost_flags, &count);
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen if (ctx->lost_flag_idx == count) {
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen ctx->last_read = TRUE;
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen return FALSE;
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen }
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen sync_rec->uid1 = range[ctx->lost_flag_idx].seq1;
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen sync_rec->uid2 = range[ctx->lost_flag_idx].seq2;
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen sync_rec->hidden = FALSE;
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen ctx->lost_flag_idx++;
a566d9f8797dd66b4d5432a20ab2b9f7c8f76102Timo Sirainen return TRUE;
da0420f1b5d5f5ab1079d1204d70dd51866ce025Timo Sirainen}
da0420f1b5d5f5ab1079d1204d70dd51866ce025Timo Sirainen
da0420f1b5d5f5ab1079d1204d70dd51866ce025Timo Sirainenbool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
da0420f1b5d5f5ab1079d1204d70dd51866ce025Timo Sirainen struct mail_index_view_sync_rec *sync_rec)
d65cc3312d3126d34b51ae8eccd6b48215d50029Timo Sirainen{
ca4526e3b5fbf5ea3dd477a2098522a44c9ac52cTimo Sirainen int ret;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
61d3fd14828b68d789f3df73d1dbed56e37b7931Timo Sirainen if (ctx->log_was_lost)
61d3fd14828b68d789f3df73d1dbed56e37b7931Timo Sirainen return mail_index_view_sync_next_lost(ctx, sync_rec);
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen do {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen ret = mail_index_view_sync_get_next_transaction(ctx);
18458233e89c8467e30d8b93b1823128f26bc174Timo Sirainen if (ret <= 0) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (ret < 0)
8c30a8e508dd05b63c9b1fa7ae9c637d132dac6fTimo Sirainen ctx->failed = TRUE;
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen return FALSE;
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen }
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen ctx->data_offset = 0;
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen }
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen } while (!mail_index_view_sync_get_rec(ctx, sync_rec));
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen return TRUE;
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen}
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainenvoid mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen const ARRAY_TYPE(seq_range) **expunges_r)
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen{
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen *expunges_r = &ctx->expunges;
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen}
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainenstatic void
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainenmail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen{
0f5dc4da3982053036be65190e44bf28a67b1ca2Timo Sirainen const struct mail_index_view_log_sync_area *syncs;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen unsigned int i, count;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
44f3f472a49078312432b785fddcfe7b95928391Timo Sirainen if (!array_is_created(&view->syncs_hidden))
44f3f472a49078312432b785fddcfe7b95928391Timo Sirainen return;
8c30a8e508dd05b63c9b1fa7ae9c637d132dac6fTimo Sirainen
d25877a69c21fcd004f77bbfb1b8d0895d78e4ddTimo Sirainen /* Clean up to view's tail */
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen syncs = array_get(&view->syncs_hidden, &count);
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen for (i = 0; i < count; i++) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if ((syncs[i].log_file_offset +
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen syncs[i].length > view->log_file_expunge_offset &&
6b6011c2242e470b41316f92512b282b5e306dacTimo Sirainen syncs[i].log_file_seq == view->log_file_expunge_seq) ||
6b6011c2242e470b41316f92512b282b5e306dacTimo Sirainen syncs[i].log_file_seq > view->log_file_expunge_seq)
6b6011c2242e470b41316f92512b282b5e306dacTimo Sirainen break;
da0420f1b5d5f5ab1079d1204d70dd51866ce025Timo Sirainen }
6b6011c2242e470b41316f92512b282b5e306dacTimo Sirainen if (i > 0)
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen array_delete(&view->syncs_hidden, 0, i);
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen}
38f624b427aa8b6fad3765e6efd97c85a7f97a09Timo Sirainen
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainenint mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx,
71564d36ea3aba6dd6a0d2560da46ffda733b679Timo Sirainen bool *delayed_expunges_r)
71564d36ea3aba6dd6a0d2560da46ffda733b679Timo Sirainen{
71564d36ea3aba6dd6a0d2560da46ffda733b679Timo Sirainen struct mail_index_view_sync_ctx *ctx = *_ctx;
71564d36ea3aba6dd6a0d2560da46ffda733b679Timo Sirainen struct mail_index_view *view = ctx->view;
9c45821d7448c6f63391d318a6dff785e46e58eeTimo Sirainen int ret = ctx->failed ? -1 : 0;
bfd8127fc0836b181b1ae65680bd6e3806b64fd1Timo Sirainen
16dfe1f9c1c1a5abc5e5a12efa25619586f7e4e6Timo Sirainen i_assert(view->syncing);
16dfe1f9c1c1a5abc5e5a12efa25619586f7e4e6Timo Sirainen
16dfe1f9c1c1a5abc5e5a12efa25619586f7e4e6Timo Sirainen *_ctx = NULL;
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen *delayed_expunges_r = ctx->skipped_expunges;
16dfe1f9c1c1a5abc5e5a12efa25619586f7e4e6Timo Sirainen
16dfe1f9c1c1a5abc5e5a12efa25619586f7e4e6Timo Sirainen if ((!ctx->last_read || view->inconsistent) &&
16dfe1f9c1c1a5abc5e5a12efa25619586f7e4e6Timo Sirainen (ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
16dfe1f9c1c1a5abc5e5a12efa25619586f7e4e6Timo Sirainen /* we didn't sync everything */
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen view->inconsistent = TRUE;
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen ret = -1;
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen }
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (ctx->sync_map_ctx.modseq_ctx != NULL)
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen if (ctx->sync_new_map != NULL) {
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen mail_index_unmap(&view->map);
eed03830015b7138b9d4522e72bef650aa24b45fTimo Sirainen view->map = ctx->sync_new_map;
aeea3dbd1f4031634f7b318614adf51dcfc79f42Timo Sirainen } else if (ctx->sync_map_update) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* log offsets have no meaning in views. make sure they're not
1ee1a73bde20380ba39d572681397ad05dcdf9a1Timo Sirainen tried to be used wrong by setting them to zero. */
1ee1a73bde20380ba39d572681397ad05dcdf9a1Timo Sirainen view->map->hdr.log_file_seq = 0;
1ee1a73bde20380ba39d572681397ad05dcdf9a1Timo Sirainen view->map->hdr.log_file_head_offset = 0;
1ee1a73bde20380ba39d572681397ad05dcdf9a1Timo Sirainen view->map->hdr.log_file_tail_offset = 0;
b9dce659b9135c87c7708b2bb0f14e8742db7e15Timo Sirainen }
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
2ed2459dbd183bb371da4a0aecb2d2b74ae7c815Timo Sirainen i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count);
170834eda87ac8e6b8b6577738ef373fc68f117dTimo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (!ctx->skipped_expunges) {
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen view->log_file_expunge_seq = view->log_file_head_seq;
9a935c34e98ba7a9cc90784ceb63b2fbdab4105fTimo Sirainen view->log_file_expunge_offset = view->log_file_head_offset;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen }
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
b9dce659b9135c87c7708b2bb0f14e8742db7e15Timo Sirainen if (ctx->sync_map_ctx.view != NULL)
b9dce659b9135c87c7708b2bb0f14e8742db7e15Timo Sirainen mail_index_sync_map_deinit(&ctx->sync_map_ctx);
b9dce659b9135c87c7708b2bb0f14e8742db7e15Timo Sirainen mail_index_view_sync_clean_log_syncs(ctx->view);
b9dce659b9135c87c7708b2bb0f14e8742db7e15Timo Sirainen
b9dce659b9135c87c7708b2bb0f14e8742db7e15Timo Sirainen#ifdef DEBUG
b9dce659b9135c87c7708b2bb0f14e8742db7e15Timo Sirainen mail_index_map_check(view->map);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen#endif
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen /* set log view to empty range so unneeded memory gets freed */
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen mail_transaction_log_view_clear(view->log_view,
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen view->log_file_expunge_seq);
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (array_is_created(&ctx->expunges))
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen array_free(&ctx->expunges);
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen if (array_is_created(&ctx->lost_flags))
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen array_free(&ctx->lost_flags);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen view->highest_modseq = mail_index_map_modseq_get_highest(view->map);
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen view->syncing = FALSE;
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen i_free(ctx);
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen return ret;
d65cc3312d3126d34b51ae8eccd6b48215d50029Timo Sirainen}
ca4526e3b5fbf5ea3dd477a2098522a44c9ac52cTimo Sirainen
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainenvoid mail_index_view_add_hidden_transaction(struct mail_index_view *view,
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen uint32_t log_file_seq,
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen uoff_t log_file_offset,
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen unsigned int length)
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen{
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen struct mail_index_view_log_sync_area *area;
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen if (!array_is_created(&view->syncs_hidden))
ee77aba32f3d05ecd87cca445c77e1f1095035e2Timo Sirainen i_array_init(&view->syncs_hidden, 32);
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen area = array_append_space(&view->syncs_hidden);
efb0def040a3e98da05ea77c170cd6711a9c8747Timo Sirainen area->log_file_seq = log_file_seq;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen area->log_file_offset = log_file_offset;
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen area->length = length;
315ce5be539bfe8bc7777ab0654499c49583cea2Timo Sirainen}
b02e7bac5b8b19795884229f64e6fbbd93a0d2deTimo Sirainen