mail-index-view-sync.c revision 2a6af811ea3de3cf9e2f15e446674dd21b0705f3
183bea41fa640dc8117f3eb45ff935cd81377a84Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "lib.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "array.h"
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen#include "buffer.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-index-view-private.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-index-sync-private.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-transaction-log.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-transaction-util.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstruct mail_index_view_sync_ctx {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_index_view *view;
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen enum mail_transaction_type visible_sync_mask;
2cd2518bab14292a67cf8a490b58ab9ef89879daTimo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ARRAY_TYPE(seq_range) expunges;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_transaction_header *hdr;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const void *data;
6ec925f52d04ec8700e47cb005bd7ddc65ac5614Timo Sirainen
c60178b2610a9b193ff72aa18448398ef72529a1Timo Sirainen size_t data_offset;
4d4d6d4745682790c20d759ba93dbea46b812c5dTimo Sirainen unsigned int skipped_some:1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int last_read:1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int sync_map_update:1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen};
cf2e6953d03a1c22f272ec19432fc03c136ac1bbTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainenmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct seq_range *src, size_t src_size)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Note that all the sequences are actually still UIDs at this point */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct seq_range *src_end;
617e13833c798435e2be425b99c27ecaad1b8393Timo Sirainen struct seq_range *dest, new_exp;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int first, i, dest_count;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(src_size % sizeof(*src) == 0);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen /* @UNSAFE */
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen dest = array_get_modifiable(expunges, &dest_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (dest_count == 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen array_append(expunges, src, src_size / sizeof(*src));
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen }
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen src_end = CONST_PTR_OFFSET(src, src_size);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen for (i = 0; src != src_end; src++) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* src[] must be sorted. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (src->seq1 > src->seq2 ||
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (src+1 != src_end && src->seq2 >= src[1].seq1))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen for (; i < dest_count; i++) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (src->seq1 < dest[i].seq1)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen break;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen new_exp = *src;
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen first = i;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen /* we can/must merge with next record */
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen if (new_exp.seq2 < dest[i].seq2)
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen new_exp.seq2 = dest[i].seq2;
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen i++;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen }
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
230ef558135f16a66b86cbe3762524eaa9ae9d81Timo Sirainen if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
230ef558135f16a66b86cbe3762524eaa9ae9d81Timo Sirainen /* continue previous record */
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen if (dest[first-1].seq2 < new_exp.seq2)
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen dest[first-1].seq2 = new_exp.seq2;
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen } else if (i == first) {
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen array_insert(expunges, i, &new_exp, 1);
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen i++; first++;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen dest = array_get_modifiable(expunges, &dest_count);
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen } else {
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen /* use next record */
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen dest[first] = new_exp;
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen first++;
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen }
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen if (i > first) {
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen array_delete(expunges, first, i - first);
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
617e13833c798435e2be425b99c27ecaad1b8393Timo Sirainen i = first;
617e13833c798435e2be425b99c27ecaad1b8393Timo Sirainen }
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen }
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen return 0;
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen}
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainenstatic int view_sync_set_log_view_range(struct mail_index_view *view,
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen enum mail_transaction_type type_mask)
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen{
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen const struct mail_index_header *hdr = view->index->hdr;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen int ret;
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen
617e13833c798435e2be425b99c27ecaad1b8393Timo Sirainen /* the view begins from the first non-synced transaction */
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen ret = mail_transaction_log_view_set(view->log_view,
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen view->log_file_seq,
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen view->log_file_offset,
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen hdr->log_file_seq,
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen hdr->log_file_index_int_offset,
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen type_mask);
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen if (ret <= 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (ret == 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* FIXME: use the new index to get needed changes */
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen mail_index_set_error(view->index,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen "Transaction log got desynced for index %s",
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen view->index->filepath);
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen mail_index_set_inconsistent(view->index);
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen }
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen return -1;
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen }
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen return 0;
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen}
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainenstatic int
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainenview_sync_get_expunges(struct mail_index_view *view,
dc07b75b7ea83ff5f447970a20419032725271a7Timo Sirainen ARRAY_TYPE(seq_range) *expunges_r)
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen{
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen const struct mail_transaction_header *hdr;
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen struct seq_range *src, *src_end, *dest;
efc5c69c572e83db7bf7eab5d4698c0ab0d3d886Timo Sirainen const void *data;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int count;
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen int ret;
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen if (view_sync_set_log_view_range(view, MAIL_TRANSACTION_EXPUNGE) < 0)
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen return -1;
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen /* get a list of expunge transactions. there may be some that we have
29f32cdcf44cda9688576bfdc7450a8a15e90e86Timo Sirainen already synced, but it doesn't matter because they'll get dropped
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen out when converting to sequences */
b955a1c1b6d466977d971c029a9305bee492f73cTimo Sirainen i_array_init(expunges_r, 64);
b955a1c1b6d466977d971c029a9305bee492f73cTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
b955a1c1b6d466977d971c029a9305bee492f73cTimo Sirainen &hdr, &data, NULL)) > 0) {
b955a1c1b6d466977d971c029a9305bee492f73cTimo Sirainen i_assert((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0);
b955a1c1b6d466977d971c029a9305bee492f73cTimo Sirainen if (mail_transaction_log_sort_expunges(expunges_r, data,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen hdr->size) < 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_transaction_log_view_set_corrupted(view->log_view,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen "Corrupted expunge record");
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ret = -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen break;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (ret < 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen array_free(expunges_r);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* convert UIDs to sequences */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen src = dest = array_get_modifiable(expunges_r, &count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen src_end = src + count;
601b455f4d5e780044b9e4fac5f687c1b07ae145Timo Sirainen for (; src != src_end; src++) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ret = mail_index_lookup_uid_range(view, src->seq1,
e9ac518aaf49a06d976bf7f24ab14a3e2d6d86abTimo Sirainen src->seq2,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen &dest->seq1,
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen &dest->seq2);
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen i_assert(ret == 0);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (dest->seq1 == 0)
fb7ac3e31c92627efe076318319976ac1c27ae2aTimo Sirainen count--;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen else
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen dest++;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
72cc352b25ad401b923436c6ed0f1f3adaffa737Timo Sirainen array_delete(expunges_r, count, array_count(expunges_r) - count);
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
614529ee060755c0b282102b70daf56bcd64222dTimo Sirainen
9ebd0c59de5f8240c0dbc58773fe5679391199dbTimo Sirainenstatic void mail_index_view_hdr_update(struct mail_index_view *view,
6ec925f52d04ec8700e47cb005bd7ddc65ac5614Timo Sirainen struct mail_index_map *map)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Keep message count the same. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map->hdr.next_uid = view->hdr.next_uid;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map->hdr.messages_count = view->hdr.messages_count;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Keep the old message flag counts also, although they may be
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen somewhat stale already. We just don't want them to be more than
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen our old messages_count. */
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen map->hdr.recent_messages_count = view->hdr.recent_messages_count;
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen map->hdr.seen_messages_count = view->hdr.seen_messages_count;
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen map->hdr.deleted_messages_count = view->hdr.deleted_messages_count;
e1f866daa1bd1a5cd7516f3b19c6f197bcf6cc8aTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Keep log position so we know where to continue syncing */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map->hdr.log_file_seq = view->hdr.log_file_seq;
4b2a1030f9b51e90f6ff4cdbc115871a398c1e0fTimo Sirainen map->hdr.log_file_index_int_offset =
72cc352b25ad401b923436c6ed0f1f3adaffa737Timo Sirainen view->hdr.log_file_index_int_offset;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen map->hdr.log_file_index_ext_offset =
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen view->hdr.log_file_index_ext_offset;
72cc352b25ad401b923436c6ed0f1f3adaffa737Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->hdr = map->hdr;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen#ifdef DEBUG
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic void mail_index_view_check(struct mail_index_view *view)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int i, del = 0, recent = 0, seen = 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(view->hdr.deleted_messages_count ==
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->map->hdr.deleted_messages_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(view->hdr.recent_messages_count ==
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->map->hdr.recent_messages_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(view->hdr.seen_messages_count ==
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->map->hdr.seen_messages_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen for (i = 0; i < view->map->records_count; i++) {
f5e1d3d6b34ec152aa1ff15c7bd3d3552e9227eaTimo Sirainen const struct mail_index_record *rec;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rec = MAIL_INDEX_MAP_IDX(view->map, i);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (rec->flags & MAIL_DELETED) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(rec->uid >= view->hdr.first_deleted_uid_lowwater);
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen del++;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (rec->flags & MAIL_RECENT) {
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen i_assert(rec->uid >= view->hdr.first_recent_uid_lowwater);
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen recent++;
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen }
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen if (rec->flags & MAIL_SEEN)
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen seen++;
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen else
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(rec->uid >= view->hdr.first_unseen_uid_lowwater);
d482079eb385cd071bbc9637cacee225e4aff968Timo Sirainen }
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen i_assert(del == view->hdr.deleted_messages_count);
3a0f9aa9504497e4e47f32df54fbf47fdc7423b6Timo Sirainen i_assert(recent == view->hdr.recent_messages_count);
3a0f9aa9504497e4e47f32df54fbf47fdc7423b6Timo Sirainen i_assert(seen == view->hdr.seen_messages_count);
1ffb2afe6d7e8860a2231a4827078cf2ef9c22cdTimo Sirainen}
1ffb2afe6d7e8860a2231a4827078cf2ef9c22cdTimo Sirainen#endif
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen#define MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK \
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen (MAIL_INDEX_SYNC_TYPE_FLAGS | \
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET | \
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD | MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#define MAIL_TRANSACTION_VISIBLE_SYNC_MASK \
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_APPEND | \
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen MAIL_TRANSACTION_FLAG_UPDATE | MAIL_TRANSACTION_KEYWORD_UPDATE | \
0547a828ee9647fc5fd2e359b145774740a697feTimo Sirainen MAIL_TRANSACTION_KEYWORD_RESET)
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen#define VIEW_IS_SYNCED_TO_SAME(view, hdr) \
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen ((hdr)->log_file_seq == (view)->log_file_seq && \
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen (hdr)->log_file_index_int_offset == (view)->log_file_offset && \
43d7e7ce608f5451e4907b5f5c48c00beb265802Timo Sirainen (hdr)->log_file_index_ext_offset == (view)->log_file_offset && \
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen (!array_is_created(&view->syncs_done) || \
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen array_count(&view->syncs_done) == 0))
5d46f4d076fc13ae2779c365cf4bd9bda3a5bc69Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenint mail_index_view_sync_begin(struct mail_index_view *view,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen enum mail_index_sync_type sync_mask,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_index_view_sync_ctx **ctx_r)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen struct mail_index_view_sync_ctx *ctx;
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen struct mail_index_map *map;
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen enum mail_transaction_type log_get_mask, visible_mask;
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen ARRAY_TYPE(seq_range) expunges = ARRAY_INIT;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen /* We must sync flags as long as view is mmap()ed, as the flags may
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen have already changed under us. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert((sync_mask & MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK) ==
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Currently we're not handling correctly expunges + no-appends case */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) == 0 ||
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen (sync_mask & MAIL_INDEX_SYNC_TYPE_APPEND) != 0);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(!view->syncing);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(view->transactions == 0);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (mail_index_view_lock_head(view) < 0)
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
601b455f4d5e780044b9e4fac5f687c1b07ae145Timo Sirainen if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) {
601b455f4d5e780044b9e4fac5f687c1b07ae145Timo Sirainen /* get list of all expunges first */
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen if (view_sync_get_expunges(view, &expunges) < 0)
cdc8485491045d82bb98405d4b995f277d12838eTimo Sirainen return -1;
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen }
b2ed2b25c4c457ec1c99ebe5e9bd66a2e2f89cfdTimo Sirainen
e5097d2c8efecdd274272b222cf2b30a8ae4ca2aTimo Sirainen /* only flags, appends and expunges can be left to be synced later */
e5097d2c8efecdd274272b222cf2b30a8ae4ca2aTimo Sirainen visible_mask = mail_transaction_type_mask_get(sync_mask);
e5097d2c8efecdd274272b222cf2b30a8ae4ca2aTimo Sirainen i_assert((visible_mask & ~MAIL_TRANSACTION_VISIBLE_SYNC_MASK) == 0);
e5097d2c8efecdd274272b222cf2b30a8ae4ca2aTimo Sirainen
e5097d2c8efecdd274272b222cf2b30a8ae4ca2aTimo Sirainen /* we want to also get non-visible changes. especially because we use
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen the returned skipped-flag in mail_transaction_log_view_next() to
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen tell us if any visible changes were skipped. */
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen log_get_mask = visible_mask | (MAIL_TRANSACTION_TYPE_MASK ^
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen MAIL_TRANSACTION_VISIBLE_SYNC_MASK);
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (view_sync_set_log_view_range(view, log_get_mask) < 0) {
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (array_is_created(&expunges))
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen array_free(&expunges);
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen return -1;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen }
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen ctx = i_new(struct mail_index_view_sync_ctx, 1);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ctx->view = view;
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen ctx->visible_sync_mask = visible_mask;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ctx->expunges = expunges;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_sync_map_init(&ctx->sync_map_ctx, view,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_INDEX_SYNC_HANDLER_VIEW);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0 &&
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen (sync_mask & MAIL_INDEX_SYNC_TYPE_APPEND) != 0) {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen view->sync_new_map = view->index->map;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen view->sync_new_map->refcount++;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* since we're syncing everything, the counters get fixed */
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen view->broken_counters = FALSE;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* keep the old mapping without expunges until we're
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen fully synced */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen } else {
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen /* We need a private copy of the map if we don't want to
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen sync expunges.
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen If view's map is the head map, it means that it contains
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen already all the latest changes and there's no need for us
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen to apply any changes to it. This can only happen if there
3a0f9aa9504497e4e47f32df54fbf47fdc7423b6Timo Sirainen hadn't been any expunges. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t old_records_count = view->map->records_count;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen if (view->map != view->index->map) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Using non-head mapping. We have to apply
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen transactions to it to get latest changes into it. */
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen ctx->sync_map_update = TRUE;
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen /* Unless map was synced at the exact same position as
4f4943f6ef1bc45c23de73eebe83779712b3c8cbTimo Sirainen view, the message flags can't be reliably used to
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen update flag counters. note that map->hdr may contain
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen old information if another process updated the
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen index file since. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (view->map->mmap_base != NULL) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_index_header *hdr;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen hdr = view->map->mmap_base;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->map->hdr = *hdr;
2cd2518bab14292a67cf8a490b58ab9ef89879daTimo Sirainen }
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen ctx->sync_map_ctx.unreliable_flags =
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen !VIEW_IS_SYNCED_TO_SAME(view, &view->map->hdr);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Copy only the mails that we see currently, since
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen we're going to append the new ones when we see
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen their transactions. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(view->map->records_count >=
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->hdr.messages_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->map->records_count = view->hdr.messages_count;
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen
d8552f9f65e5ff64be5de9faf9a8171799a0bbecTimo Sirainen#ifdef DEBUG
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen if (!ctx->sync_map_ctx.unreliable_flags) {
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen i_assert(view->map->hdr.messages_count ==
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->hdr.messages_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_view_check(view);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen#endif
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen }
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen map = mail_index_map_clone(view->map);
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen view->map->records_count = old_records_count;
bad5fa318c6c1384ab83bd72d53ce06593274c18Timo Sirainen mail_index_unmap(view->index, &view->map);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->map = map;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (ctx->sync_map_update) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Start the sync using our old view's header.
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen The old view->hdr may differ from map->hdr if
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen another view sharing the map with us had synced
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen itself. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(map->hdr_base == map->hdr_copy_buf->data);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_view_hdr_update(view, map);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert(map->records_count == map->hdr.messages_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Syncing the view invalidates all previous looked up records.
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen Unreference the mappings this view keeps because of them. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_index_view_unref_maps(view);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->syncing = TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen *ctx_r = ctx;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic void
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenview_add_synced_transaction(struct mail_index_view *view,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t log_file_seq, uoff_t log_file_offset)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_index_view_log_sync_area *pos;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!array_is_created(&view->syncs_done))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_array_init(&view->syncs_done, 32);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen pos = array_append_space(&view->syncs_done);
614529ee060755c0b282102b70daf56bcd64222dTimo Sirainen pos->log_file_seq = log_file_seq;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen pos->log_file_offset = log_file_offset;
a3dcda4b01461c7690c655a013ec12851cdb78d4Timo Sirainen pos->length = 1;
61618d4c58080570f689614fec204ae14e90cef2Timo Sirainen}
61618d4c58080570f689614fec204ae14e90cef2Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic bool view_sync_area_find(ARRAY_TYPE(view_log_sync_area) *sync_arr,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t seq, uoff_t offset)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_index_view_log_sync_area *syncs;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int i, count;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!array_is_created(sync_arr))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen syncs = array_get(sync_arr, &count);
129db31de1780783a175633eba5811e44c361a81Timo Sirainen for (i = 0; i < count; i++) {
129db31de1780783a175633eba5811e44c361a81Timo Sirainen if (syncs[i].log_file_offset <= offset &&
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen offset - syncs[i].log_file_offset < syncs[i].length &&
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen syncs[i].log_file_seq == seq)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return TRUE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenmail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_transaction_log_view *log_view = ctx->view->log_view;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_index_view *view = ctx->view;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uint32_t seq;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen uoff_t offset;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen int ret;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen bool skipped, synced_to_map;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
63849db64682675a2fd3e1aea05c10ecbc6d473aTimo Sirainen for (;;) {
114162093a3eb36c23a4ce4d4f2a43541dc18cc2Timo Sirainen /* Get the next transaction from log. */
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen &ctx->data, &skipped);
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (skipped) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* We skipped some (visible) transactions that were
114162093a3eb36c23a4ce4d4f2a43541dc18cc2Timo Sirainen outside our sync mask. Note that we may get here
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen even when ret=0. */
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen ctx->skipped_some = TRUE;
ca502ca792708238ac2c23481c3bf94eda1f743fTimo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (ret <= 0) {
63849db64682675a2fd3e1aea05c10ecbc6d473aTimo Sirainen if (ret < 0)
129db31de1780783a175633eba5811e44c361a81Timo Sirainen return -1;
129db31de1780783a175633eba5811e44c361a81Timo Sirainen
dfcc56dc6c485fe45a3fa18325f047bfaae65019Timo Sirainen ctx->hdr = NULL;
dfcc56dc6c485fe45a3fa18325f047bfaae65019Timo Sirainen ctx->last_read = TRUE;
63849db64682675a2fd3e1aea05c10ecbc6d473aTimo Sirainen return 0;
63849db64682675a2fd3e1aea05c10ecbc6d473aTimo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (!ctx->skipped_some) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* We haven't skipped anything while syncing this view.
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen Update this view's synced log offset. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->log_file_seq = seq;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view->log_file_offset = offset + sizeof(*ctx->hdr) +
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen ctx->hdr->size;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* skip everything we've already synced */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (view_sync_area_find(&view->syncs_done, seq, offset))
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen continue;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
55bc6a7a0940ec48a68558ef70838991c5d301d2Timo Sirainen if (ctx->skipped_some) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* We've been skipping some transactions, which means
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen we'll go through these same transactions again
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen later. Since we're syncing this one, we don't want
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen to do it again. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen view_add_synced_transaction(view, seq, offset);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* if we started from a map that we didn't create ourself,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen some of the external transactions may already be synced.
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen at the end of view sync we'll update the ext_offset in the
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen header so that this check always becomes FALSE for
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen subsequent syncs. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen synced_to_map = offset < view->hdr.log_file_index_ext_offset &&
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen seq == view->hdr.log_file_seq &&
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen (ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* Apply transaction to view's mapping if needed (meaning we
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen didn't just re-map the view to head mapping). */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (ctx->sync_map_update && !synced_to_map) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_assert((ctx->hdr->type &
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MAIL_TRANSACTION_EXPUNGE) == 0);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (mail_index_sync_record(&ctx->sync_map_ctx,
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen ctx->hdr, ctx->data) < 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen return -1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen if ((ctx->hdr->type & ctx->visible_sync_mask) == 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* non-visible change that we just wanted to update
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen to map. */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen continue;
800fc30be46bc6f8380f6de04aea7c19ea839ddfTimo Sirainen }
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* skip changes committed by hidden transactions (eg. in IMAP
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen store +flags.silent command) */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (view_sync_area_find(&view->syncs_hidden, seq, offset))
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen continue;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen break;
800fc30be46bc6f8380f6de04aea7c19ea839ddfTimo Sirainen }
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen return 1;
800fc30be46bc6f8380f6de04aea7c19ea839ddfTimo Sirainen}
800fc30be46bc6f8380f6de04aea7c19ea839ddfTimo Sirainen
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen#define FLAG_UPDATE_IS_INTERNAL(u) \
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen ((((u)->add_flags | (u)->remove_flags) & \
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0)
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenmail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct mail_index_view_sync_rec *rec)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_transaction_header *hdr = ctx->hdr;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const void *data = ctx->data;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen case MAIL_TRANSACTION_APPEND: {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* data contains the appended records, but we don't care */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen rec->uid1 = rec->uid2 = 0;
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen ctx->data_offset += hdr->size;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen break;
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen }
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen case MAIL_TRANSACTION_EXPUNGE: {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen const struct mail_transaction_expunge *exp =
9c8f854d95d8d895022a75f140a0a500eb200d39Timo Sirainen CONST_PTR_OFFSET(data, ctx->data_offset);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* data contains mail_transaction_expunge[] */
rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
rec->uid1 = exp->uid1;
rec->uid2 = exp->uid2;
ctx->data_offset += sizeof(*exp);
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *update =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_flag_update[] */
for (;;) {
ctx->data_offset += sizeof(*update);
if (!FLAG_UPDATE_IS_INTERNAL(update))
break;
/* skip internal flag changes */
if (ctx->data_offset == ctx->hdr->size)
return 0;
update = CONST_PTR_OFFSET(data, ctx->data_offset);
}
rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
rec->uid1 = update->uid1;
rec->uid2 = update->uid2;
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *update = data;
const uint32_t *uids;
/* data contains mail_transaction_keyword_update header,
the keyword name and an array of { uint32_t uid1, uid2; } */
if (ctx->data_offset == 0) {
/* skip over the header and name */
ctx->data_offset = sizeof(*update) + update->name_size;
if ((ctx->data_offset % 4) != 0)
ctx->data_offset += 4 - (ctx->data_offset % 4);
}
uids = CONST_PTR_OFFSET(data, ctx->data_offset);
rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
rec->uid1 = uids[0];
rec->uid2 = uids[1];
ctx->data_offset += sizeof(uint32_t) * 2;
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *reset =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_keyword_reset[] */
rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
rec->uid1 = reset->uid1;
rec->uid2 = reset->uid2;
ctx->data_offset += sizeof(*reset);
break;
}
default:
i_unreached();
}
return 1;
}
int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
int ret;
do {
if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
ret = mail_index_view_sync_get_next_transaction(ctx);
if (ret <= 0)
return ret;
ctx->data_offset = 0;
}
} while (!mail_index_view_sync_get_rec(ctx, sync_rec));
return 1;
}
void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
const ARRAY_TYPE(seq_range) **expunges_r)
{
*expunges_r = &ctx->expunges;
}
static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view_sync_ctx *ctx,
ARRAY_TYPE(view_log_sync_area) *sync_arr,
bool fast_clean)
{
struct mail_index_view *view = ctx->view;
const struct mail_index_view_log_sync_area *syncs;
uint32_t seq = view->log_file_seq;
uoff_t offset = view->log_file_offset;
unsigned int i, count;
if (!array_is_created(sync_arr))
return;
if (!ctx->skipped_some && fast_clean) {
/* Nothing skipped. Clean it up the quick way. */
array_clear(sync_arr);
return;
}
/* Clean up until view's current syncing position */
syncs = array_get(sync_arr, &count);
for (i = 0; i < count; i++) {
if ((syncs[i].log_file_offset + syncs[i].length > offset &&
syncs[i].log_file_seq == seq) ||
syncs[i].log_file_seq > seq)
break;
}
if (i > 0)
array_delete(sync_arr, 0, i);
}
void mail_index_view_sync_end(struct mail_index_view_sync_ctx **_ctx)
{
struct mail_index_view_sync_ctx *ctx = *_ctx;
struct mail_index_view *view = ctx->view;
i_assert(view->syncing);
*_ctx = NULL;
mail_index_sync_map_deinit(&ctx->sync_map_ctx);
mail_index_view_sync_clean_log_syncs(ctx, &view->syncs_done, TRUE);
mail_index_view_sync_clean_log_syncs(ctx, &view->syncs_hidden, FALSE);
if (!ctx->last_read) {
/* we didn't sync everything */
view->inconsistent = TRUE;
}
if (view->sync_new_map != NULL) {
mail_index_unmap(view->index, &view->map);
view->map = view->sync_new_map;
view->sync_new_map = NULL;
}
if (ctx->sync_map_update) {
if (view->log_file_seq != view->map->hdr.log_file_seq) {
i_assert(view->log_file_seq >
view->map->hdr.log_file_seq);
view->map->hdr.log_file_seq = view->log_file_seq;
view->map->hdr.log_file_index_ext_offset =
view->log_file_offset;
} else {
i_assert(view->log_file_offset >=
view->map->hdr.log_file_index_int_offset);
if (view->log_file_offset >
view->map->hdr.log_file_index_ext_offset) {
view->map->hdr.log_file_index_ext_offset =
view->log_file_offset;
}
}
view->map->hdr.log_file_index_int_offset =
view->log_file_offset;
buffer_write(view->map->hdr_copy_buf, 0,
&view->map->hdr, sizeof(view->map->hdr));
} else {
i_assert(view->inconsistent ||
view->log_file_offset >=
view->map->hdr.log_file_index_int_offset);
}
view->hdr = view->map->hdr;
#ifdef DEBUG
if (!view->broken_counters && !ctx->sync_map_ctx.unreliable_flags)
mail_index_view_check(view);
#endif
/* set log view to empty range so unneeded memory gets freed */
(void)mail_transaction_log_view_set(view->log_view,
view->log_file_seq,
view->log_file_offset,
view->log_file_seq,
view->log_file_offset,
MAIL_TRANSACTION_TYPE_MASK);
if (array_is_created(&ctx->expunges))
array_free(&ctx->expunges);
view->syncing = FALSE;
i_free(ctx);
}
void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset,
unsigned int length)
{
struct mail_index_view_log_sync_area *area;
if (!array_is_created(&view->syncs_hidden))
i_array_init(&view->syncs_hidden, 32);
area = array_append_space(&view->syncs_hidden);
area->log_file_seq = log_file_seq;
area->log_file_offset = log_file_offset;
area->length = length;
}