src/lib-index/mail-index-view-sync.c

	mail-index-view-sync.c revision bbe90c4805e7032fdcefde3df125df0ca8be1bef
e59faf65ce864fe95dc00f5d52b8323cdbd0608aTimo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
46552a931924c2d743f045e95b08c3ce6beda91aTimo Sirainen#include "lib.h"
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen#include "array.h"
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen#include "buffer.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "mail-index-view-private.h"
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen#include "mail-index-sync-private.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "mail-transaction-log.h"
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenstruct mail_index_view_sync_ctx {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    struct mail_index_view *view;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    enum mail_transaction_type visible_sync_mask;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    struct mail_index_sync_map_ctx sync_map_ctx;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    ARRAY_TYPE(seq_range) expunges;
d63b4241643b6014d49ff356f14e0f3ee43068a8Timo Sirainen
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen    const struct mail_transaction_header *hdr;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    const void *data;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    size_t data_offset;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    unsigned int skipped_some:1;
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen    unsigned int last_read:1;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    unsigned int sync_map_update:1;
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen};
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenstatic int
79977d368df2efc3f3dc0bbb260748b076538103Timo Sirainenmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen                   const struct seq_range *src, size_t src_size)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
277c0eea825eec176cddc029af68f5a4d942e16eTimo Sirainen    /* Note that all the sequences are actually still UIDs at this point */
02a0492affdff138f43903c19ca366363923044dTimo Sirainen    const struct seq_range *src_end;
02a0492affdff138f43903c19ca366363923044dTimo Sirainen    struct seq_range *dest, new_exp;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    unsigned int first, i, dest_count;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    i_assert(src_size % sizeof(*src) == 0);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    /* @UNSAFE */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    dest = array_get_modifiable(expunges, &dest_count);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (dest_count == 0) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        array_append(expunges, src, src_size / sizeof(*src));
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        return 0;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    src_end = CONST_PTR_OFFSET(src, src_size);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    for (i = 0; src != src_end; src++) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        /* src[] must be sorted. */
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen        if (src->seq1 > src->seq2 ||
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            (src+1 != src_end && src->seq2 >= src[1].seq1))
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            return -1;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        for (; i < dest_count; i++) {
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen            if (src->seq1 < dest[i].seq1)
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen                break;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        new_exp = *src;
79977d368df2efc3f3dc0bbb260748b076538103Timo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        first = i;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
277c0eea825eec176cddc029af68f5a4d942e16eTimo Sirainen            /* we can/must merge with next record */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            if (new_exp.seq2 < dest[i].seq2)
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen                new_exp.seq2 = dest[i].seq2;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            i++;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            /* continue previous record */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            if (dest[first-1].seq2 < new_exp.seq2)
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen                dest[first-1].seq2 = new_exp.seq2;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        } else if (i == first) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            array_insert(expunges, i, &new_exp, 1);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            i++; first++;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            dest = array_get_modifiable(expunges, &dest_count);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        } else {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            /* use next record */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            dest[first] = new_exp;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            first++;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        }
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        if (i > first) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            array_delete(expunges, first, i - first);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            dest = array_get_modifiable(expunges, &dest_count);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen            i = first;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen        }
383d0e8c24451468d6bea17e4b55d74de744abe6Timo Sirainen    }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen    return 0;
383d0e8c24451468d6bea17e4b55d74de744abe6Timo Sirainen}
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenstatic int view_sync_set_log_view_range(struct mail_index_view *view)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen    const struct mail_index_header *hdr = view->index->hdr;
7ceee001c11865a88a311c50c4430867b2d32ce6Timo Sirainen    int ret;
7ceee001c11865a88a311c50c4430867b2d32ce6Timo Sirainen
7ceee001c11865a88a311c50c4430867b2d32ce6Timo Sirainen    /* the view begins from the first non-synced transaction */
7ceee001c11865a88a311c50c4430867b2d32ce6Timo Sirainen    ret = mail_transaction_log_view_set(view->log_view,
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen                        view->log_file_seq,
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen                        view->log_file_offset,
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen                        hdr->log_file_seq,
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen                        hdr->log_file_index_int_offset);
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen    if (ret <= 0) {
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen        if (ret == 0) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            /* FIXME: use the new index to get needed changes */
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen            mail_index_set_error(view->index,
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen                "Transaction log got desynced for index %s",
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen                view->index->filepath);
ce8a6f53ea3ce91b759a54b771e1779564de19a1Timo Sirainen            mail_index_set_inconsistent(view->index);
ce8a6f53ea3ce91b759a54b771e1779564de19a1Timo Sirainen        }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        return -1;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    }
02a0492affdff138f43903c19ca366363923044dTimo Sirainen    return 0;
02a0492affdff138f43903c19ca366363923044dTimo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenstatic int
02a0492affdff138f43903c19ca366363923044dTimo Sirainenview_sync_get_expunges(struct mail_index_view *view,
02a0492affdff138f43903c19ca366363923044dTimo Sirainen               ARRAY_TYPE(seq_range) *expunges_r)
1cbc0c6372d24168962698c5f4d3e15df8943ebfTimo Sirainen{
1cbc0c6372d24168962698c5f4d3e15df8943ebfTimo Sirainen    const struct mail_transaction_header *hdr;
277c0eea825eec176cddc029af68f5a4d942e16eTimo Sirainen    struct seq_range *src, *src_end, *dest;
277c0eea825eec176cddc029af68f5a4d942e16eTimo Sirainen    const void *data;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    unsigned int count;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    int ret;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    if (view_sync_set_log_view_range(view) < 0)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        return -1;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen    /* get a list of expunge transactions. there may be some that we have
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen       already synced, but it doesn't matter because they'll get dropped
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen       out when converting to sequences */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    i_array_init(expunges_r, 64);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    while ((ret = mail_transaction_log_view_next(view->log_view,
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen                             &hdr, &data)) > 0) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            continue;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            /* this is simply a request for expunge */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen            continue;
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen        }
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen        if (mail_transaction_log_sort_expunges(expunges_r, data,
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen                               hdr->size) < 0) {
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen            mail_transaction_log_view_set_corrupted(view->log_view,
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen                "Corrupted expunge record");
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen            ret = -1;
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen            break;
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen        }
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen    }
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen    if (ret < 0) {
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen        array_free(expunges_r);
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen        return -1;
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen    }
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    /* convert UIDs to sequences */
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen    src = dest = array_get_modifiable(expunges_r, &count);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    src_end = src + count;
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    for (; src != src_end; src++) {
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen        ret = mail_index_lookup_uid_range(view, src->seq1,
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen                          src->seq2,
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen                          &dest->seq1,
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen                          &dest->seq2);
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen        i_assert(ret == 0);
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen        if (dest->seq1 == 0)
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen            count--;
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen        else
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen            dest++;
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    }
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    array_delete(expunges_r, count, array_count(expunges_r) - count);
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    return 0;
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen}
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainenstatic void mail_index_view_hdr_update(struct mail_index_view *view,
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen                       struct mail_index_map *map)
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen{
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    /* Keep message count the same. */
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    map->hdr.next_uid = view->hdr.next_uid;
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    map->hdr.messages_count = view->hdr.messages_count;
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen    /* Keep the old message flag counts also, although they may be
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen       somewhat stale already. We just don't want them to be more than
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen       our old messages_count. */
71b5ad1df203c906d46960b0400564609b97a6c8Timo Sirainen    map->hdr.recent_messages_count = view->hdr.recent_messages_count;
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen    map->hdr.seen_messages_count = view->hdr.seen_messages_count;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen    map->hdr.deleted_messages_count = view->hdr.deleted_messages_count;
1358e2c58ce29231485a5cfa454756d429ad3d2cTimo Sirainen
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen    /* Keep log position so we know where to continue syncing */
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen    map->hdr.log_file_seq = view->hdr.log_file_seq;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen    map->hdr.log_file_index_int_offset =
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        view->hdr.log_file_index_int_offset;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen    map->hdr.log_file_index_ext_offset =
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        view->hdr.log_file_index_ext_offset;
1358e2c58ce29231485a5cfa454756d429ad3d2cTimo Sirainen
7ceee001c11865a88a311c50c4430867b2d32ce6Timo Sirainen    view->hdr = map->hdr;
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen    buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
1358e2c58ce29231485a5cfa454756d429ad3d2cTimo Sirainen}
71b5ad1df203c906d46960b0400564609b97a6c8Timo Sirainen
1358e2c58ce29231485a5cfa454756d429ad3d2cTimo Sirainen#ifdef DEBUG
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainenstatic void mail_index_view_check(struct mail_index_view *view)
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen{
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen    unsigned int i, del = 0, recent = 0, seen = 0;
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen
97db4761382024093f441e4bc78ba8b6a056504dTimo Sirainen    i_assert(view->hdr.deleted_messages_count ==
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen         view->map->hdr.deleted_messages_count);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen    i_assert(view->hdr.recent_messages_count ==
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen         view->map->hdr.recent_messages_count);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen    i_assert(view->hdr.seen_messages_count ==
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen         view->map->hdr.seen_messages_count);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen    for (i = 0; i < view->map->records_count; i++) {
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen        const struct mail_index_record *rec;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen        rec = MAIL_INDEX_MAP_IDX(view->map, i);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen        if (rec->flags & MAIL_DELETED) {
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen            i_assert(rec->uid >= view->hdr.first_deleted_uid_lowwater);
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen            del++;
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen        }
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen        if (rec->flags & MAIL_RECENT) {
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen            i_assert(rec->uid >= view->hdr.first_recent_uid_lowwater);
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen            recent++;
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen        }
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen        if (rec->flags & MAIL_SEEN)
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen            seen++;
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen        else
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen            i_assert(rec->uid >= view->hdr.first_unseen_uid_lowwater);
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    }
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    i_assert(del == view->hdr.deleted_messages_count);
ed4cdfe5847a6063d5a4684a6a47b17e943922b5Timo Sirainen    i_assert(recent == view->hdr.recent_messages_count);
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen    i_assert(seen == view->hdr.seen_messages_count);
f059a046515f4b2b15a6c2a10a6f12f6166e39a5Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#endif
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen#define MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK \
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen    (MAIL_INDEX_SYNC_TYPE_FLAGS | \
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen     MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET | \
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen     MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD | MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE)
5d03d9f439e41c90215a3c938ffebe4c2a8ae257Timo Sirainen
#define MAIL_TRANSACTION_VISIBLE_SYNC_MASK \
    (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_APPEND | \
     MAIL_TRANSACTION_FLAG_UPDATE | MAIL_TRANSACTION_KEYWORD_UPDATE | \
     MAIL_TRANSACTION_KEYWORD_RESET)

#define VIEW_IS_SYNCED_TO_SAME(view, hdr) \
    ((hdr)->log_file_seq == (view)->log_file_seq && \
     (hdr)->log_file_index_int_offset == (view)->log_file_offset && \
     (hdr)->log_file_index_ext_offset == (view)->log_file_offset && \
     (!array_is_created(&view->syncs_done) || \
      array_count(&view->syncs_done) == 0))

int mail_index_view_sync_begin(struct mail_index_view *view,
                               enum mail_index_view_sync_type sync_type,
                   struct mail_index_view_sync_ctx **ctx_r)
{
    struct mail_index_view_sync_ctx *ctx;
    struct mail_index_map *map;
    enum mail_transaction_type visible_mask = 0;
    ARRAY_TYPE(seq_range) expunges = ARRAY_INIT;

    i_assert(!view->syncing);
    i_assert(view->transactions == 0);

    if (mail_index_view_lock_head(view) < 0)
        return -1;

    if (sync_type == MAIL_INDEX_VIEW_SYNC_TYPE_ALL) {
        /* get list of all expunges first */
        if (view_sync_get_expunges(view, &expunges) < 0)
            return -1;
    }

    /* only flags, appends and expunges can be left to be synced later */
    switch (sync_type) {
    case MAIL_INDEX_VIEW_SYNC_TYPE_ALL:
        visible_mask = MAIL_TRANSACTION_VISIBLE_SYNC_MASK;
        break;
    case MAIL_INDEX_VIEW_SYNC_TYPE_NOAPPENDS_NOEXPUNGES:
        visible_mask = MAIL_TRANSACTION_VISIBLE_SYNC_MASK &
            ~(MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_APPEND);
        break;
    case MAIL_INDEX_VIEW_SYNC_TYPE_NOEXPUNGES:
        visible_mask = MAIL_TRANSACTION_VISIBLE_SYNC_MASK &
            ~MAIL_TRANSACTION_EXPUNGE;
        break;
    }

    if (view_sync_set_log_view_range(view) < 0) {
        if (array_is_created(&expunges))
            array_free(&expunges);
        return -1;
    }

    ctx = i_new(struct mail_index_view_sync_ctx, 1);
    ctx->view = view;
    ctx->visible_sync_mask = visible_mask;
    ctx->expunges = expunges;
    mail_index_sync_map_init(&ctx->sync_map_ctx, view,
                 MAIL_INDEX_SYNC_HANDLER_VIEW);

    if (sync_type == MAIL_INDEX_VIEW_SYNC_TYPE_ALL) {
        view->sync_new_map = view->index->map;
        view->sync_new_map->refcount++;

        /* since we're syncing everything, the counters get fixed */
        view->broken_counters = FALSE;

        /* keep the old mapping without expunges until we're
           fully synced */
    } else {
        /* We need a private copy of the map if we don't want to
           sync expunges.

           If view's map is the head map, it means that it contains
           already all the latest changes and there's no need for us
           to apply any changes to it. This can only happen if there
           hadn't been any expunges. */
        uint32_t old_records_count = view->map->records_count;

        if (view->map != view->index->map) {
            /* Using non-head mapping. We have to apply
               transactions to it to get latest changes into it. */
            ctx->sync_map_update = TRUE;

            /* Unless map was synced at the exact same position as
               view, the message flags can't be reliably used to
               update flag counters. note that map->hdr may contain
               old information if another process updated the
               index file since. */
            if (view->map->mmap_base != NULL) {
                const struct mail_index_header *hdr;

                hdr = view->map->mmap_base;
                view->map->hdr = *hdr;
            }
            ctx->sync_map_ctx.unreliable_flags =
                !VIEW_IS_SYNCED_TO_SAME(view, &view->map->hdr);

            /* Copy only the mails that we see currently, since
               we're going to append the new ones when we see
               their transactions. */
            i_assert(view->map->records_count >=
                 view->hdr.messages_count);
            view->map->records_count = view->hdr.messages_count;

#ifdef DEBUG
            if (!ctx->sync_map_ctx.unreliable_flags) {
                i_assert(view->map->hdr.messages_count ==
                     view->hdr.messages_count);
                mail_index_view_check(view);
            }
#endif
        }

        map = mail_index_map_clone(view->map);
        view->map->records_count = old_records_count;
        mail_index_unmap(view->index, &view->map);
        view->map = map;

        if (ctx->sync_map_update) {
            /* Start the sync using our old view's header.
               The old view->hdr may differ from map->hdr if
               another view sharing the map with us had synced
               itself. */
            i_assert(map->hdr_base == map->hdr_copy_buf->data);
            mail_index_view_hdr_update(view, map);
        }

        i_assert(map->records_count == map->hdr.messages_count);
    }

    /* Syncing the view invalidates all previous looked up records.
       Unreference the mappings this view keeps because of them. */
    mail_index_view_unref_maps(view);
    view->syncing = TRUE;

    *ctx_r = ctx;
    return 0;
}

static void
view_add_synced_transaction(struct mail_index_view *view,
                uint32_t log_file_seq, uoff_t log_file_offset)
{
    struct mail_index_view_log_sync_area *pos;

    if (!array_is_created(&view->syncs_done))
        i_array_init(&view->syncs_done, 32);

    pos = array_append_space(&view->syncs_done);
    pos->log_file_seq = log_file_seq;
    pos->log_file_offset = log_file_offset;
    pos->length = 1;
}

static bool view_sync_area_find(ARRAY_TYPE(view_log_sync_area) *sync_arr,
                uint32_t seq, uoff_t offset)
{
    const struct mail_index_view_log_sync_area *syncs;
    unsigned int i, count;

    if (!array_is_created(sync_arr))
        return FALSE;

    syncs = array_get(sync_arr, &count);
    for (i = 0; i < count; i++) {
        if (syncs[i].log_file_offset <= offset &&
            offset - syncs[i].log_file_offset < syncs[i].length &&
            syncs[i].log_file_seq == seq)
            return TRUE;
    }

    return FALSE;
}

static bool
mail_index_view_sync_skip(struct mail_index_view_sync_ctx *ctx,
              const struct mail_transaction_header *hdr)
{
    if ((hdr->type & ctx->visible_sync_mask) != 0)
        return FALSE;

    if ((hdr->type & MAIL_TRANSACTION_VISIBLE_SYNC_MASK) == 0)
        return FALSE;

    if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
        (hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
        /* expunge request. this will be ignored */
        return FALSE;
    }

    /* visible record that we want to skip */
    return TRUE;
}

static int
mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
{
        struct mail_transaction_log_view *log_view = ctx->view->log_view;
    struct mail_index_view *view = ctx->view;
    const struct mail_transaction_header *hdr;
    uint32_t seq;
    uoff_t offset;
    int ret;
    bool synced_to_map;

    for (;;) {
        /* Get the next transaction from log. */
        ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
                             &ctx->data);
        if (ret <= 0) {
            if (ret < 0)
                return -1;

            ctx->hdr = NULL;
            ctx->last_read = TRUE;
            return 0;
        }

        hdr = ctx->hdr;
        if (mail_index_view_sync_skip(ctx, hdr)) {
            /* This is a visible record that we don't want to
               sync. */
            ctx->skipped_some = TRUE;
            continue;
        }

        mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);

        if (!ctx->skipped_some) {
            /* We haven't skipped anything while syncing this view.
               Update this view's synced log offset. */
            view->log_file_seq = seq;
            view->log_file_offset = offset + sizeof(*hdr) +
                hdr->size;
        }

        /* skip everything we've already synced */
        if (view_sync_area_find(&view->syncs_done, seq, offset))
            continue;

        if (ctx->skipped_some) {
            /* We've been skipping some transactions, which means
               we'll go through these same transactions again
               later. Since we're syncing this one, we don't want
               to do it again. */
            view_add_synced_transaction(view, seq, offset);
        }

        /* if we started from a map that we didn't create ourself,
           some of the external transactions may already be synced.
           at the end of view sync we'll update the ext_offset in the
           header so that this check always becomes FALSE for
           subsequent syncs. */
        synced_to_map = offset < view->hdr.log_file_index_ext_offset &&
            seq == view->hdr.log_file_seq &&
            (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0;

        /* Apply transaction to view's mapping if needed (meaning we
           didn't just re-map the view to head mapping). */
        if (ctx->sync_map_update && !synced_to_map) {
            i_assert((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0 ||
                 (hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0);

            if (mail_index_sync_record(&ctx->sync_map_ctx,
                           hdr, ctx->data) < 0)
                return -1;
        }

        if ((hdr->type & ctx->visible_sync_mask) == 0) {
            /* non-visible change that we just wanted to update
               to map. */
            continue;
        }

        /* skip changes committed by hidden transactions (eg. in IMAP
           store +flags.silent command) */
        if (view_sync_area_find(&view->syncs_hidden, seq, offset))
            continue;
        break;
    }
    return 1;
}

#define FLAG_UPDATE_IS_INTERNAL(u) \
    ((((u)->add_flags | (u)->remove_flags) & \
      ~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0)

static int
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
                 struct mail_index_view_sync_rec *rec)
{
    const struct mail_transaction_header *hdr = ctx->hdr;
    const void *data = ctx->data;

    switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
    case MAIL_TRANSACTION_APPEND: {
        /* data contains the appended records, but we don't care */
        rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
        rec->uid1 = rec->uid2 = 0;
        ctx->data_offset += hdr->size;
        break;
    }
    case MAIL_TRANSACTION_EXPUNGE: {
        const struct mail_transaction_expunge *exp =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
            /* this is simply a request for expunge */
            ctx->data_offset = ctx->hdr->size;
            return 0;
        }

        /* data contains mail_transaction_expunge[] */
        rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
        rec->uid1 = exp->uid1;
        rec->uid2 = exp->uid2;

        ctx->data_offset += sizeof(*exp);
        break;
    }
    case MAIL_TRANSACTION_FLAG_UPDATE: {
        const struct mail_transaction_flag_update *update =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_flag_update[] */
        for (;;) {
            ctx->data_offset += sizeof(*update);
            if (!FLAG_UPDATE_IS_INTERNAL(update))
                break;

            /* skip internal flag changes */
            if (ctx->data_offset == ctx->hdr->size)
                return 0;

            update = CONST_PTR_OFFSET(data, ctx->data_offset);
        }

        rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
        rec->uid1 = update->uid1;
        rec->uid2 = update->uid2;
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_UPDATE: {
        const struct mail_transaction_keyword_update *update = data;
        const uint32_t *uids;

        /* data contains mail_transaction_keyword_update header,
           the keyword name and an array of { uint32_t uid1, uid2; } */

        if (ctx->data_offset == 0) {
            /* skip over the header and name */
            ctx->data_offset = sizeof(*update) + update->name_size;
            if ((ctx->data_offset % 4) != 0)
                ctx->data_offset += 4 - (ctx->data_offset % 4);
        }

        uids = CONST_PTR_OFFSET(data, ctx->data_offset);
        rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
        rec->uid1 = uids[0];
        rec->uid2 = uids[1];

        ctx->data_offset += sizeof(uint32_t) * 2;
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_RESET: {
        const struct mail_transaction_keyword_reset *reset =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_keyword_reset[] */
        rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
        rec->uid1 = reset->uid1;
        rec->uid2 = reset->uid2;
        ctx->data_offset += sizeof(*reset);
        break;
    }
    default:
        i_unreached();
    }
    return 1;
}

int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
                  struct mail_index_view_sync_rec *sync_rec)
{
    int ret;

    do {
        if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
            ret = mail_index_view_sync_get_next_transaction(ctx);
            if (ret <= 0)
                return ret;

            ctx->data_offset = 0;
        }
    } while (!mail_index_view_sync_get_rec(ctx, sync_rec));

    return 1;
}

void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
                       const ARRAY_TYPE(seq_range) **expunges_r)
{
    *expunges_r = &ctx->expunges;
}

static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view_sync_ctx *ctx,
                     ARRAY_TYPE(view_log_sync_area) *sync_arr,
                     bool fast_clean)
{
    struct mail_index_view *view = ctx->view;
    const struct mail_index_view_log_sync_area *syncs;
    uint32_t seq = view->log_file_seq;
    uoff_t offset = view->log_file_offset;
    unsigned int i, count;

    if (!array_is_created(sync_arr))
        return;

    if (!ctx->skipped_some && fast_clean) {
        /* Nothing skipped. Clean it up the quick way. */
        array_clear(sync_arr);
        return;
    }

    /* Clean up until view's current syncing position */
    syncs = array_get(sync_arr, &count);
    for (i = 0; i < count; i++) {
        if ((syncs[i].log_file_offset + syncs[i].length > offset &&
                     syncs[i].log_file_seq == seq) ||
            syncs[i].log_file_seq > seq)
            break;
    }
    if (i > 0)
        array_delete(sync_arr, 0, i);
}

void mail_index_view_sync_end(struct mail_index_view_sync_ctx **_ctx)
{
        struct mail_index_view_sync_ctx *ctx = *_ctx;
        struct mail_index_view *view = ctx->view;

    i_assert(view->syncing);

    *_ctx = NULL;
    mail_index_sync_map_deinit(&ctx->sync_map_ctx);
    mail_index_view_sync_clean_log_syncs(ctx, &view->syncs_done, TRUE);
    mail_index_view_sync_clean_log_syncs(ctx, &view->syncs_hidden, FALSE);

    if (!ctx->last_read) {
        /* we didn't sync everything */
        view->inconsistent = TRUE;
    }

    if (view->sync_new_map != NULL) {
        mail_index_unmap(view->index, &view->map);
        view->map = view->sync_new_map;
        view->sync_new_map = NULL;
    }

    if (ctx->sync_map_update) {
        if (view->log_file_seq != view->map->hdr.log_file_seq) {
            i_assert(view->log_file_seq >
                 view->map->hdr.log_file_seq);
            view->map->hdr.log_file_seq = view->log_file_seq;
            view->map->hdr.log_file_index_ext_offset =
                view->log_file_offset;
        } else {
            i_assert(view->log_file_offset >=
                 view->map->hdr.log_file_index_int_offset);
            if (view->log_file_offset >
                view->map->hdr.log_file_index_ext_offset) {
                view->map->hdr.log_file_index_ext_offset =
                    view->log_file_offset;
            }
        }
        view->map->hdr.log_file_index_int_offset =
            view->log_file_offset;
        buffer_write(view->map->hdr_copy_buf, 0,
                 &view->map->hdr, sizeof(view->map->hdr));
    } else {
        i_assert(view->inconsistent ||
             view->log_file_offset >=
             view->map->hdr.log_file_index_int_offset);
    }
    view->hdr = view->map->hdr;

#ifdef DEBUG
    if (!view->broken_counters && !ctx->sync_map_ctx.unreliable_flags)
        mail_index_view_check(view);
#endif

    /* set log view to empty range so unneeded memory gets freed */
    (void)mail_transaction_log_view_set(view->log_view,
                        view->log_file_seq,
                        view->log_file_offset,
                        view->log_file_seq,
                        view->log_file_offset);

    if (array_is_created(&ctx->expunges))
        array_free(&ctx->expunges);

    view->syncing = FALSE;
    i_free(ctx);
}

void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
                        uint32_t log_file_seq,
                        uoff_t log_file_offset,
                        unsigned int length)
{
    struct mail_index_view_log_sync_area *area;

    if (!array_is_created(&view->syncs_hidden))
        i_array_init(&view->syncs_hidden, 32);

    area = array_append_space(&view->syncs_hidden);
    area->log_file_seq = log_file_seq;
    area->log_file_offset = log_file_offset;
    area->length = length;
}