mail-index-view-sync.c revision 94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cb
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen#include "lib.h"
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen#include "array.h"
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen#include "buffer.h"
c2ee17c9c263efdc9c0a339c4836c3d43f5cd3d9Sascha Wilde#include "mail-index-view-private.h"
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen#include "mail-index-sync-private.h"
cda217260716cfd8d8ec5e56f91708c64c140538Timo Sirainen#include "mail-transaction-log.h"
7a7d2aa11e46195e2d92d6c337d7e78052a5ce67Timo Sirainen#include "mail-transaction-util.h"
d00ae137b6772f0b047cc98cb153f11c5246f82bTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainenstruct mail_index_view_sync_ctx {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen struct mail_index_view *view;
c2ee17c9c263efdc9c0a339c4836c3d43f5cd3d9Sascha Wilde enum mail_transaction_type visible_sync_mask;
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;
c2ee17c9c263efdc9c0a339c4836c3d43f5cd3d9Sascha Wilde ARRAY_TYPE(seq_range) expunges;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen const struct mail_transaction_header *hdr;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen const void *data;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen size_t data_offset;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen unsigned int skipped_some:1;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen unsigned int last_read:1;
f325d795b52ce2053f914072b22ebca9c4f0dc7eTimo Sirainen unsigned int sync_map_update:1;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen};
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainenstatic void
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainenmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen const struct seq_range *src, size_t src_size)
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* Note that all the sequences are actually still UIDs at this point */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen const struct seq_range *src_end;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen struct seq_range *dest, new_exp;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen unsigned int first, i, dest_count;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen i_assert(src_size % sizeof(*src) == 0);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
139cbf8d9e1cc0c65d985f525756fe47a7bfada6Timo Sirainen /* @UNSAFE */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (dest_count == 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen array_append(expunges, src, src_size / sizeof(*src));
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen }
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen src_end = CONST_PTR_OFFSET(src, src_size);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen for (i = 0; src != src_end; src++) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* src[] must be sorted. */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen i_assert(src+1 == src_end || src->seq2 < src[1].seq1);
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen i_assert(src->seq1 <= src->seq2);
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen
0f17bb103602d0c4394e3784cb96d788530fc79eTimo Sirainen for (; i < dest_count; i++) {
0f17bb103602d0c4394e3784cb96d788530fc79eTimo Sirainen if (src->seq1 < dest[i].seq1)
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen break;
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen }
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen new_exp = *src;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen first = i;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen /* we can/must merge with next record */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (new_exp.seq2 < dest[i].seq2)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen new_exp.seq2 = dest[i].seq2;
c8cf8a605e0ddea7cb36fe04551aeca5090e684bTimo Sirainen i++;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen }
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
c8cf8a605e0ddea7cb36fe04551aeca5090e684bTimo Sirainen /* continue previous record */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (dest[first-1].seq2 < new_exp.seq2)
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen dest[first-1].seq2 = new_exp.seq2;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen } else if (i == first) {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen array_insert(expunges, i, &new_exp, 1);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen i++; first++;
139cbf8d9e1cc0c65d985f525756fe47a7bfada6Timo Sirainen
f6d63a21010540d3ddf08f2e7664ffca3ea70489Timo Sirainen dest = array_get_modifiable(expunges, &dest_count);
139cbf8d9e1cc0c65d985f525756fe47a7bfada6Timo Sirainen } else {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen /* use next record */
139cbf8d9e1cc0c65d985f525756fe47a7bfada6Timo Sirainen dest[first] = new_exp;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen first++;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen }
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (i > first) {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen array_delete(expunges, first, i - first);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen i = first;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen }
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen }
dc912088f84c263db1609435c2f5d7cb29bf1a33Timo Sirainen}
a5bb2908b44c8bf5ce41160a64c67fb840a20006Timo Sirainen
a5bb2908b44c8bf5ce41160a64c67fb840a20006Timo Sirainenstatic int view_sync_set_log_view_range(struct mail_index_view *view,
a5bb2908b44c8bf5ce41160a64c67fb840a20006Timo Sirainen enum mail_transaction_type type_mask)
a5bb2908b44c8bf5ce41160a64c67fb840a20006Timo Sirainen{
a5bb2908b44c8bf5ce41160a64c67fb840a20006Timo Sirainen const struct mail_index_header *hdr = view->index->hdr;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen int ret;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen ret = mail_transaction_log_view_set(view->log_view,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen view->log_file_seq,
8d90e4f9a8f79f79c393aca23d0a897471dc2d8fTimo Sirainen view->log_file_offset,
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen hdr->log_file_seq,
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen hdr->log_file_int_offset,
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen type_mask);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (ret <= 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (ret == 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* FIXME: use the new index to get needed changes */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_set_error(view->index,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen "Transaction log got desynced for index %s",
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen view->index->filepath);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_set_inconsistent(view->index);
67c47dbb3fde79218320fd38a45c33f61bbf3012Timo Sirainen }
7fc0f80480063a9d4cb9e8c07b50db2a5627799eTimo Sirainen return -1;
67c47dbb3fde79218320fd38a45c33f61bbf3012Timo Sirainen }
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen return 0;
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen}
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen
708efcd8581258763289b95cde119ca9423641d8Timo Sirainenstatic int
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainenview_sync_get_expunges(struct mail_index_view *view,
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen ARRAY_TYPE(seq_range) *expunges_r)
7fc0f80480063a9d4cb9e8c07b50db2a5627799eTimo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen const struct mail_transaction_header *hdr;
67c47dbb3fde79218320fd38a45c33f61bbf3012Timo Sirainen struct seq_range *src, *src_end, *dest;
67c47dbb3fde79218320fd38a45c33f61bbf3012Timo Sirainen const void *data;
c2ee17c9c263efdc9c0a339c4836c3d43f5cd3d9Sascha Wilde unsigned int count;
c2ee17c9c263efdc9c0a339c4836c3d43f5cd3d9Sascha Wilde int ret;
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen if (view_sync_set_log_view_range(view, MAIL_TRANSACTION_EXPUNGE) < 0)
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen return -1;
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen i_array_init(expunges_r, 64);
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen &hdr, &data, NULL)) > 0) {
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen i_assert((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0);
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen mail_transaction_log_sort_expunges(expunges_r, data, hdr->size);
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen }
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen if (ret < 0) {
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen array_free(expunges_r);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return -1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* convert UIDs to sequences */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen src = dest = array_get_modifiable(expunges_r, &count);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen src_end = src + count;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen for (; src != src_end; src++) {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen ret = mail_index_lookup_uid_range(view, src->seq1,
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen src->seq2,
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen &dest->seq1,
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen &dest->seq2);
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen i_assert(ret == 0);
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (dest->seq1 == 0)
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen count--;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen else
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen dest++;
67c47dbb3fde79218320fd38a45c33f61bbf3012Timo Sirainen }
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen array_delete(expunges_r, count, array_count(expunges_r) - count);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen return 0;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen}
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainenstatic void mail_index_view_hdr_update(struct mail_index_view *view,
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen struct mail_index_map *map)
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen{
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen /* Keep message count the same. */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen map->hdr.next_uid = view->hdr.next_uid;
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen map->hdr.messages_count = view->hdr.messages_count;
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen /* Keep the old message flag counts also, although they may be
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen somewhat stale already. We just don't want them to be more than
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen our old messages_count. */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen map->hdr.recent_messages_count = view->hdr.recent_messages_count;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen map->hdr.seen_messages_count = view->hdr.seen_messages_count;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen map->hdr.deleted_messages_count = view->hdr.deleted_messages_count;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen /* Keep log position so we know where to continue syncing */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen map->hdr.log_file_seq = view->hdr.log_file_seq;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen map->hdr.log_file_int_offset = view->hdr.log_file_int_offset;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen map->hdr.log_file_ext_offset = view->hdr.log_file_ext_offset;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen view->hdr = map->hdr;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen}
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen#define MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK \
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen (MAIL_INDEX_SYNC_TYPE_FLAGS | \
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET | \
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD | MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE)
35083063d0e432d0cf78206b5929750e613ad772Timo Sirainen
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen#define MAIL_TRANSACTION_VISIBLE_SYNC_MASK \
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_APPEND | \
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen MAIL_TRANSACTION_FLAG_UPDATE | MAIL_TRANSACTION_KEYWORD_UPDATE | \
35083063d0e432d0cf78206b5929750e613ad772Timo Sirainen MAIL_TRANSACTION_KEYWORD_RESET)
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainenint mail_index_view_sync_begin(struct mail_index_view *view,
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen enum mail_index_sync_type sync_mask,
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen struct mail_index_view_sync_ctx **ctx_r)
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen{
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen const struct mail_index_header *hdr;
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen struct mail_index_view_sync_ctx *ctx;
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen struct mail_index_map *map;
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen enum mail_transaction_type log_get_mask, visible_mask;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen ARRAY_TYPE(seq_range) expunges = ARRAY_INIT;
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen /* We must sync flags as long as view is mmap()ed, as the flags may
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen have already changed under us. */
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen i_assert((sync_mask & MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK) ==
b1c42176a65dbe9c83a0af766e6bd8315530f3a5Timo Sirainen MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK);
d927bc5618696157fc55eb1f11b5cab05400ed52Timo Sirainen /* Currently we're not handling correctly expunges + no-appends case */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen i_assert((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) == 0 ||
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen (sync_mask & MAIL_INDEX_SYNC_TYPE_APPEND) != 0);
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen i_assert(!view->syncing);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen i_assert(view->transactions == 0);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (mail_index_view_lock_head(view, TRUE) < 0)
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen return -1;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen hdr = view->index->hdr;
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) {
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen /* get list of all expunges first */
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen if (view_sync_get_expunges(view, &expunges) < 0)
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen return -1;
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen }
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen /* only flags, appends and expunges can be left to be synced later */
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen visible_mask = mail_transaction_type_mask_get(sync_mask);
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen i_assert((visible_mask & ~MAIL_TRANSACTION_VISIBLE_SYNC_MASK) == 0);
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen /* we want to also get non-visible changes. especially because we use
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen the returned skipped-flag in mail_transaction_log_view_next() to
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen tell us if any visible changes were skipped. */
0671e0ae0cfd8d5d671a0c2a75a070c8e2a39fecTimo Sirainen log_get_mask = visible_mask | (MAIL_TRANSACTION_TYPE_MASK ^
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen MAIL_TRANSACTION_VISIBLE_SYNC_MASK);
d927bc5618696157fc55eb1f11b5cab05400ed52Timo Sirainen if (view_sync_set_log_view_range(view, log_get_mask) < 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (array_is_created(&expunges))
d927bc5618696157fc55eb1f11b5cab05400ed52Timo Sirainen array_free(&expunges);
d927bc5618696157fc55eb1f11b5cab05400ed52Timo Sirainen return -1;
d927bc5618696157fc55eb1f11b5cab05400ed52Timo Sirainen }
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen ctx = i_new(struct mail_index_view_sync_ctx, 1);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen ctx->view = view;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen ctx->visible_sync_mask = visible_mask;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen ctx->expunges = expunges;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen mail_index_sync_map_init(&ctx->sync_map_ctx, view,
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen MAIL_INDEX_SYNC_HANDLER_VIEW);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0 &&
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen (sync_mask & MAIL_INDEX_SYNC_TYPE_APPEND) != 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen view->sync_new_map = view->index->map;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen view->sync_new_map->refcount++;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* since we're syncing everything, the counters get fixed */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen view->broken_counters = FALSE;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen /* keep the old mapping without expunges until we're
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen fully synced */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen } else {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen /* We need a private copy of the map if we don't want to
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sync expunges.
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
f534c0d42f1470fca8e4ff3493c94927bf600260Timo Sirainen If view's map is the head map, it means that it contains
39993536eaef0a23954105e41040dcf88afd2e7eTimo Sirainen already all the latest changes and there's no need for us
39993536eaef0a23954105e41040dcf88afd2e7eTimo Sirainen to apply any changes to it. This can only happen if there
39993536eaef0a23954105e41040dcf88afd2e7eTimo Sirainen hadn't been any expunges. */
39993536eaef0a23954105e41040dcf88afd2e7eTimo Sirainen uint32_t old_records_count = view->map->records_count;
39993536eaef0a23954105e41040dcf88afd2e7eTimo Sirainen
39993536eaef0a23954105e41040dcf88afd2e7eTimo Sirainen if (view->map != view->index->map) {
39993536eaef0a23954105e41040dcf88afd2e7eTimo Sirainen /* Using non-head mapping. We have to apply
f77ffa31038d46ca9c6d24d93e3d76c9aa8d4d0cTimo Sirainen transactions to it to get latest changes into it. */
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen ctx->sync_map_update = TRUE;
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen /* Unless map was synced at the exact same position as
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen view, the message flags can't be reliably used to
67c47dbb3fde79218320fd38a45c33f61bbf3012Timo Sirainen update flag counters. */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen ctx->sync_map_ctx.unreliable_flags =
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen !(view->map->hdr.log_file_seq ==
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen view->log_file_seq &&
67c47dbb3fde79218320fd38a45c33f61bbf3012Timo Sirainen view->map->hdr.log_file_int_offset ==
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen view->log_file_offset);
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen /* Copy only the mails that we see currently, since
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen we're going to append the new ones when we see
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen their transactions. */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen i_assert(view->map->records_count >=
1ba47b1a31e60c533631c8810400b365f785870aTimo Sirainen view->hdr.messages_count);
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen view->map->records_count = view->hdr.messages_count;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen map = mail_index_map_clone(view->map,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen view->map->hdr.record_size);
9762e4f86950549c8186c7d3d4fa4a6b533ea848Timo Sirainen view->map->records_count = old_records_count;
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen mail_index_unmap(view->index, &view->map);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen view->map = map;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (ctx->sync_map_update) {
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen /* Start the sync using our old view's header.
c4c9be10781e1a16b3b001dc6b0461c4640da101Timo Sirainen The old view->hdr may differ from map->hdr if
60b42c6dfdf9edcca8a96b380ef9a0adc60c2464Timo Sirainen another view sharing the map with us had synced
7fc0f80480063a9d4cb9e8c07b50db2a5627799eTimo Sirainen itself. */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen i_assert(map->hdr_base == map->hdr_copy_buf->data);
4ed1c0fedf33299264a319b2c4e0fe2465ea321bTimo Sirainen mail_index_view_hdr_update(view, map);
4ed1c0fedf33299264a319b2c4e0fe2465ea321bTimo Sirainen }
4ed1c0fedf33299264a319b2c4e0fe2465ea321bTimo Sirainen
4ed1c0fedf33299264a319b2c4e0fe2465ea321bTimo Sirainen i_assert(map->records_count == map->hdr.messages_count);
60b42c6dfdf9edcca8a96b380ef9a0adc60c2464Timo Sirainen }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen /* Syncing the view invalidates all previous looked up records.
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen Unreference the mappings this view keeps because of them. */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen mail_index_view_unref_maps(view);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen view->syncing = TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen *ctx_r = ctx;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainenstatic bool view_sync_pos_find(ARRAY_TYPE(view_log_sync_pos) *sync_arr,
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen uint32_t seq, uoff_t offset)
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen{
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen const struct mail_index_view_log_sync_pos *syncs;
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen unsigned int i, count;
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen
d29e35631075b8c172ce781b85fd1cdaf0a400fdTimo Sirainen if (!array_is_created(sync_arr))
cc2c73be39dfe988f52c0370667e3882d01c63a2Timo Sirainen return FALSE;
cc2c73be39dfe988f52c0370667e3882d01c63a2Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen syncs = array_get(sync_arr, &count);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen for (i = 0; i < count; i++) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (syncs[i].log_file_offset == offset &&
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen syncs[i].log_file_seq == seq)
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen return TRUE;
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen }
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen return FALSE;
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen}
512f492f410fdaefb2f58e19c2b067ef20fb4adfTimo Sirainen
31327a74b86728e201fcedd0acaecf69d077bf1dTimo Sirainenstatic int
31327a74b86728e201fcedd0acaecf69d077bf1dTimo Sirainenmail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
31327a74b86728e201fcedd0acaecf69d077bf1dTimo Sirainen{
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct mail_transaction_log_view *log_view = ctx->view->log_view;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct mail_index_view *view = ctx->view;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen uint32_t seq;
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen uoff_t offset;
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen int ret;
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen bool skipped;
708efcd8581258763289b95cde119ca9423641d8Timo Sirainen
94ce7e7700cda14a8342cb08e7285507b4b531daTimo Sirainen for (;;) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* Get the next transaction from log. */
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen &ctx->data, &skipped);
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (ret <= 0) {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen if (ret < 0)
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen return -1;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen ctx->hdr = NULL;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen ctx->last_read = TRUE;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen return 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
ac26a4607cb12b156f6a42f1ead2881bedd43d94Timo Sirainen mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (skipped) {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen /* We skipped some (visible) transactions that were
d00ae137b6772f0b047cc98cb153f11c5246f82bTimo Sirainen outside our sync mask. */
3e564425db51f3921ce4de11859777135fdedd15Timo Sirainen ctx->skipped_some = TRUE;
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen } else if (!ctx->skipped_some) {
4654f788834c9d7920a351306b89cf5d1c21772eTimo Sirainen /* We haven't skipped anything while syncing this view.
Update this view's synced log offset. */
view->log_file_seq = seq;
view->log_file_offset = offset + sizeof(*ctx->hdr) +
ctx->hdr->size;
}
/* skip everything we've already synced */
if (view_sync_pos_find(&view->syncs_done, seq, offset))
continue;
/* Apply transaction to view's mapping if needed (meaning we
didn't just re-map the view to head mapping). */
if (ctx->sync_map_update) {
i_assert((ctx->hdr->type &
MAIL_TRANSACTION_EXPUNGE) == 0);
if (mail_index_sync_record(&ctx->sync_map_ctx,
ctx->hdr, ctx->data) < 0)
return -1;
}
if ((ctx->hdr->type & ctx->visible_sync_mask) == 0) {
/* non-visible change that we just wanted to update
to map. */
continue;
}
/* skip changes committed by hidden transactions (eg. in IMAP
store +flags.silent command) */
if (view_sync_pos_find(&view->syncs_hidden, seq, offset))
continue;
break;
}
if (ctx->skipped_some) {
/* We've been skipping some transactions, which means we'll
go through these same transaction again later. Since we're
syncing this one, we don't want to do it again. */
mail_index_view_add_synced_transaction(view, seq, offset);
}
return 1;
}
#define FLAG_UPDATE_IS_INTERNAL(u) \
((((u)->add_flags | (u)->remove_flags) & \
~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0)
static int
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *rec)
{
const struct mail_transaction_header *hdr = ctx->hdr;
const void *data = ctx->data;
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_APPEND: {
/* data contains the appended records, but we don't care */
rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
rec->uid1 = rec->uid2 = 0;
ctx->data_offset += hdr->size;
break;
}
case MAIL_TRANSACTION_EXPUNGE: {
const struct mail_transaction_expunge *exp =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_expunge[] */
rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
rec->uid1 = exp->uid1;
rec->uid2 = exp->uid2;
ctx->data_offset += sizeof(*exp);
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *update =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_flag_update[] */
for (;;) {
ctx->data_offset += sizeof(*update);
if (!FLAG_UPDATE_IS_INTERNAL(update))
break;
/* skip internal flag changes */
if (ctx->data_offset == ctx->hdr->size)
return 0;
update = CONST_PTR_OFFSET(data, ctx->data_offset);
}
rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
rec->uid1 = update->uid1;
rec->uid2 = update->uid2;
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *update = data;
const uint32_t *uids;
/* data contains mail_transaction_keyword_update header,
the keyword name and an array of { uint32_t uid1, uid2; } */
if (ctx->data_offset == 0) {
/* skip over the header and name */
ctx->data_offset = sizeof(*update) + update->name_size;
if ((ctx->data_offset % 4) != 0)
ctx->data_offset += 4 - (ctx->data_offset % 4);
}
uids = CONST_PTR_OFFSET(data, ctx->data_offset);
rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
rec->uid1 = uids[0];
rec->uid2 = uids[1];
ctx->data_offset += sizeof(uint32_t) * 2;
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *reset =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_keyword_reset[] */
rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
rec->uid1 = reset->uid1;
rec->uid2 = reset->uid2;
ctx->data_offset += sizeof(*reset);
break;
}
default:
i_unreached();
}
return 1;
}
int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
int ret;
do {
if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
ret = mail_index_view_sync_get_next_transaction(ctx);
if (ret <= 0)
return ret;
ctx->data_offset = 0;
}
} while (!mail_index_view_sync_get_rec(ctx, sync_rec));
return 1;
}
void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
const ARRAY_TYPE(seq_range) **expunges_r)
{
*expunges_r = &ctx->expunges;
}
static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view_sync_ctx *ctx,
ARRAY_TYPE(view_log_sync_pos) *sync_arr)
{
struct mail_index_view *view = ctx->view;
const struct mail_index_view_log_sync_pos *syncs;
unsigned int i, count;
if (!array_is_created(sync_arr))
return;
if (!ctx->skipped_some) {
/* Nothing skipped. Clean it up the quick way. */
array_clear(sync_arr);
return;
}
/* Clean up until view's current syncing position */
syncs = array_get(sync_arr, &count);
for (i = 0; i < count; i++) {
if ((syncs[i].log_file_offset >= view->log_file_offset &&
syncs[i].log_file_seq == view->log_file_seq) ||
syncs[i].log_file_seq > view->log_file_seq)
break;
}
if (i > 0)
array_delete(sync_arr, 0, i);
}
void mail_index_view_sync_end(struct mail_index_view_sync_ctx **_ctx)
{
struct mail_index_view_sync_ctx *ctx = *_ctx;
struct mail_index_view *view = ctx->view;
i_assert(view->syncing);
*_ctx = NULL;
mail_index_sync_map_deinit(&ctx->sync_map_ctx);
mail_index_view_sync_clean_log_syncs(ctx, &view->syncs_done);
mail_index_view_sync_clean_log_syncs(ctx, &view->syncs_hidden);
if (!ctx->last_read && ctx->hdr != NULL &&
ctx->data_offset != ctx->hdr->size) {
/* we didn't sync everything */
view->inconsistent = TRUE;
}
if (view->sync_new_map != NULL) {
mail_index_unmap(view->index, &view->map);
view->map = view->sync_new_map;
view->sync_new_map = NULL;
}
view->hdr = view->map->hdr;
/* set log view to empty range so unneeded memory gets freed */
(void)mail_transaction_log_view_set(view->log_view,
view->log_file_seq,
view->log_file_offset,
view->log_file_seq,
view->log_file_offset,
MAIL_TRANSACTION_TYPE_MASK);
if (array_is_created(&ctx->expunges))
array_free(&ctx->expunges);
view->syncing = FALSE;
i_free(ctx);
}
static void log_sync_pos_add(ARRAY_TYPE(view_log_sync_pos) *sync_arr,
uint32_t log_file_seq, uoff_t log_file_offset)
{
struct mail_index_view_log_sync_pos *pos;
if (!array_is_created(sync_arr))
i_array_init(sync_arr, 32);
pos = array_append_space(sync_arr);
pos->log_file_seq = log_file_seq;
pos->log_file_offset = log_file_offset;
}
void mail_index_view_add_synced_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset)
{
log_sync_pos_add(&view->syncs_done, log_file_seq, log_file_offset);
}
void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset)
{
log_sync_pos_add(&view->syncs_hidden, log_file_seq, log_file_offset);
}