src/lib-index/mail-index-view-sync.c

	mail-index-view-sync.c revision 07afd4db18860589803c46a3ee0559bda1c9e1b4
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen/* Copyright (c) 2003-2007 Dovecot authors, see the included COPYING file */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen#include "lib.h"
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen#include "array.h"
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen#include "buffer.h"
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen#include "mail-index-view-private.h"
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen#include "mail-index-sync-private.h"
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen#include "mail-transaction-log.h"
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainenstruct mail_index_view_sync_ctx {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    struct mail_index_view *view;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    enum mail_index_view_sync_flags flags;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    struct mail_index_sync_map_ctx sync_map_ctx;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    ARRAY_TYPE(seq_range) expunges;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    unsigned int finish_min_msg_count;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const struct mail_transaction_header *hdr;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const void *data;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    size_t data_offset;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    unsigned int failed:1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    unsigned int sync_map_update:1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    unsigned int skipped_expunges:1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    unsigned int last_read:1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen};
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainenstatic int
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainenmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                   const struct seq_range *src, size_t src_size)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen{
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    /* Note that all the sequences are actually still UIDs at this point */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const struct seq_range *src_end;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    struct seq_range *dest, new_exp;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    unsigned int first, i, dest_count;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    i_assert(src_size % sizeof(*src) == 0);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
8e361d2906b0e44f7175a20981f8d2280645b58bTimo Sirainen    /* @UNSAFE */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    dest = array_get_modifiable(expunges, &dest_count);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    if (dest_count == 0) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        array_append(expunges, src, src_size / sizeof(*src));
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        return 0;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
3281669db44d09a087a203201248abbc81b3cc1aTimo Sirainen    src_end = CONST_PTR_OFFSET(src, src_size);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    for (i = 0; src != src_end; src++) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        /* src[] must be sorted. */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (src->seq1 > src->seq2 ||
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            (src+1 != src_end && src->seq2 >= src[1].seq1))
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            return -1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        for (; i < dest_count; i++) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            if (src->seq1 < dest[i].seq1)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                break;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen        new_exp = *src;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        first = i;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            /* we can/must merge with next record */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            if (new_exp.seq2 < dest[i].seq2)
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen                new_exp.seq2 = dest[i].seq2;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            i++;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
8e361d2906b0e44f7175a20981f8d2280645b58bTimo Sirainen            /* continue previous record */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            if (dest[first-1].seq2 < new_exp.seq2)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                dest[first-1].seq2 = new_exp.seq2;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        } else if (i == first) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            array_insert(expunges, i, &new_exp, 1);
8e361d2906b0e44f7175a20981f8d2280645b58bTimo Sirainen            i++; first++;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            dest = array_get_modifiable(expunges, &dest_count);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        } else {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            /* use next record */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            dest[first] = new_exp;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            first++;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (i > first) {
8e361d2906b0e44f7175a20981f8d2280645b58bTimo Sirainen            array_delete(expunges, first, i - first);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            dest = array_get_modifiable(expunges, &dest_count);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            i = first;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    return 0;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen}
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
3281669db44d09a087a203201248abbc81b3cc1aTimo Sirainenstatic int
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainenview_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                 bool quick_sync, bool *reset_r)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen{
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const struct mail_index_header *hdr = &view->index->map->hdr;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    uint32_t start_seq, end_seq;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    uoff_t start_offset, end_offset;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    int ret;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    end_seq = hdr->log_file_seq;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    end_offset = hdr->log_file_head_offset;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    if (quick_sync) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        start_seq = end_seq;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        start_offset = end_offset;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        /* we'll just directly to the end */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        view->log_file_head_seq = end_seq;
d19412e30b27e413f99cbedab86c9f9396741638Timo Sirainen        view->log_file_head_offset = end_offset;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    } else {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        start_seq = view->log_file_expunge_seq;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        start_offset = view->log_file_expunge_offset;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    for (;;) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        /* the view begins from the first non-synced transaction */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        ret = mail_transaction_log_view_set(view->log_view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                            start_seq, start_offset,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                            end_seq, end_offset,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                            reset_r);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (ret <= 0) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            if (ret < 0)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                return -1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            /* FIXME: use the new index to get needed
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen               changes */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            mail_index_set_error(view->index,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                "Transaction log got desynced for index %s",
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                view->index->filepath);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            view->inconsistent = TRUE;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            return -1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (!*reset_r || sync_expunges)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            break;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        /* we can't do this. sync only up to reset. */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        mail_transaction_log_view_get_prev_pos(view->log_view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                               &end_seq, &end_offset);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        end_seq--; end_offset = (uoff_t)-1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (end_seq < start_seq) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            /* we have only this reset log */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            mail_transaction_log_view_clear(view->log_view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                view->log_file_expunge_seq);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            break;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    return 0;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen}
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainenstatic int
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainenview_sync_get_expunges(struct mail_index_view *view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen               ARRAY_TYPE(seq_range) *expunges_r,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen               unsigned int *expunge_count_r)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen{
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const struct mail_transaction_header *hdr;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    struct seq_range *src, *src_end, *dest;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const void *data;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    unsigned int count, expunge_count = 0;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    uint32_t prev_seq = 0;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    bool reset;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    int ret;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    if (view_sync_set_log_view_range(view, TRUE, FALSE, &reset) < 0)
e48f289d2e5b2546a2c5dcc90f7ab624cc58cca2Stephan Bosch        return -1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    /* get a list of expunge transactions. there may be some that we have
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen       already synced, but it doesn't matter because they'll get dropped
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen       out when converting to sequences */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    i_array_init(expunges_r, 64);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    while ((ret = mail_transaction_log_view_next(view->log_view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                             &hdr, &data)) > 0) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            continue;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            /* this is simply a request for expunge */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            continue;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (mail_transaction_log_sort_expunges(expunges_r, data,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                               hdr->size) < 0) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            mail_transaction_log_view_set_corrupted(view->log_view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                "Corrupted expunge record");
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            ret = -1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            break;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    if (ret < 0) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        array_free(expunges_r);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        return -1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    /* convert UIDs to sequences */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    src = dest = array_get_modifiable(expunges_r, &count);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    src_end = src + count;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    for (; src != src_end; src++) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                         &dest->seq1, &dest->seq2))
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            count--;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        else {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            i_assert(dest->seq1 > prev_seq);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            prev_seq = dest->seq2;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            expunge_count += dest->seq2 - dest->seq1 + 1;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            dest++;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    array_delete(expunges_r, count, array_count(expunges_r) - count);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    *expunge_count_r = expunge_count;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    return 0;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen}
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainenstatic bool have_existing_expunges(struct mail_index_view *view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                   const struct seq_range *range, size_t size)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen{
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const struct seq_range *range_end;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    uint32_t seq1, seq2;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    range_end = CONST_PTR_OFFSET(range, size);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    for (; range < range_end; range++) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (mail_index_lookup_seq_range(view, range->seq1, range->seq2,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                        &seq1, &seq2))
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            return TRUE;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    return FALSE;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen}
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainenstatic bool view_sync_have_expunges(struct mail_index_view *view)
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen{
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const struct mail_transaction_header *hdr;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    const void *data;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    uint32_t seq;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    uoff_t offset;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    bool have_expunges = FALSE;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    int ret;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    mail_transaction_log_view_get_prev_pos(view->log_view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                           &seq, &offset);
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen    while ((ret = mail_transaction_log_view_next(view->log_view,
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen                             &hdr, &data)) > 0) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen            continue;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            /* this is simply a request for expunge */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            continue;
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        }
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        /* we have an expunge. see if it still exists. */
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen        if (have_existing_expunges(view, data, hdr->size)) {
e9e2d23e1ea5a149a7d8828d2a45b9f2313c3785Timo Sirainen            have_expunges = TRUE;
            break;
        }
    }

    mail_transaction_log_view_seek(view->log_view, seq, offset);

    /* handle failures as having expunges (which is safer).
       we'll probably fail later. */
    return ret < 0 || have_expunges;
}

int mail_index_view_sync_begin(struct mail_index_view *view,
                               enum mail_index_view_sync_flags flags,
                   struct mail_index_view_sync_ctx **ctx_r)
{
    struct mail_index_view_sync_ctx *ctx;
    struct mail_index_map *map;
    ARRAY_TYPE(seq_range) expunges = ARRAY_INIT;
    unsigned int expunge_count = 0;
    bool reset, sync_expunges, quick_sync;

    i_assert(!view->syncing);
    i_assert(view->transactions == 0);

    quick_sync = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) != 0;
    if (mail_index_view_is_inconsistent(view)) {
        if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
            mail_index_set_error(view->index,
                         "%s view is inconsistent",
                         view->index->filepath);
            return -1;
        }
        view->inconsistent = FALSE;
    }

    sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
    if (quick_sync) {
        i_assert(sync_expunges);
        i_array_init(&expunges, 1);
    } else if (sync_expunges) {
        /* get list of all expunges first */
        if (view_sync_get_expunges(view, &expunges, &expunge_count) < 0)
            return -1;
    }

    if (view_sync_set_log_view_range(view, sync_expunges, quick_sync,
                     &reset) < 0) {
        if (array_is_created(&expunges))
            array_free(&expunges);
        return -1;
    }

    ctx = i_new(struct mail_index_view_sync_ctx, 1);
    ctx->view = view;
    ctx->flags = flags;
    ctx->expunges = expunges;
    ctx->finish_min_msg_count = reset || quick_sync ? 0 :
        view->map->hdr.messages_count - expunge_count;
    mail_index_sync_map_init(&ctx->sync_map_ctx, view,
                 MAIL_INDEX_SYNC_HANDLER_VIEW);

    if (reset && view->map->hdr.messages_count > 0 &&
        (flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
        view->inconsistent = TRUE;
        mail_index_set_error(view->index,
                     "%s reset, view is now inconsistent",
                     view->index->filepath);
    }

    if (sync_expunges || !view_sync_have_expunges(view)) {
        if (view->index->map->hdr.messages_count <
            ctx->finish_min_msg_count) {
            mail_index_set_error(view->index,
                "Index %s lost messages without expunging "
                "(%u -> %u)", view->index->filepath,
                view->map->hdr.messages_count,
                view->index->map->hdr.messages_count);
            ctx->finish_min_msg_count = 0;
            view->inconsistent = TRUE;
        }

        view->sync_new_map = view->index->map;
        view->sync_new_map->refcount++;

        /* keep the old mapping without expunges until we're
           fully synced */
    } else {
        /* We need a private copy of the map if we don't want to
           sync expunges.

           If view's map is the head map, it means that it contains
           already all the latest changes and there's no need for us
           to apply any changes to it. This can only happen if there
           hadn't been any expunges. */
        if (view->map != view->index->map) {
            /* Using non-head mapping. We have to apply
               transactions to it to get latest changes into it. */
            ctx->sync_map_update = TRUE;
        }

        if (view->map->refcount > 1) {
            map = mail_index_map_clone(view->map);
            mail_index_unmap(&view->map);
            view->map = map;
        } else {
            map = view->map;
        }
    }

#ifdef DEBUG
    mail_index_map_check(view->map);
#endif

    /* Syncing the view invalidates all previous looked up records.
       Unreference the mappings this view keeps because of them. */
    mail_index_view_unref_maps(view);
    view->syncing = TRUE;

    *ctx_r = ctx;
    return 0;
}

static bool
view_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
{
    const struct mail_index_view_log_sync_area *syncs;
    unsigned int i, count;

    if (!array_is_created(&view->syncs_hidden))
        return FALSE;

    syncs = array_get(&view->syncs_hidden, &count);
    for (i = 0; i < count; i++) {
        if (syncs[i].log_file_offset <= offset &&
            offset - syncs[i].log_file_offset < syncs[i].length &&
            syncs[i].log_file_seq == seq)
            return TRUE;
    }

    return FALSE;
}

static bool
mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
              const struct mail_transaction_header *hdr)
{
        struct mail_index_view *view = ctx->view;
    uint32_t seq;
    uoff_t offset, next_offset;

    mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
    next_offset = offset + sizeof(*hdr) + hdr->size;

    if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
        (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
        if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
            i_assert(!LOG_IS_BEFORE(seq, offset,
                        view->log_file_expunge_seq,
                        view->log_file_expunge_offset));
            if (!ctx->skipped_expunges) {
                view->log_file_expunge_seq = seq;
                view->log_file_expunge_offset = offset;
                ctx->skipped_expunges = TRUE;
            }
            return FALSE;
        }
        if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
                  view->log_file_expunge_offset)) {
            /* already synced */
            return FALSE;
        }
    }

    if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
              view->log_file_head_offset)) {
        /* already synced */
        return FALSE;
    }

    view->log_file_head_seq = seq;
    view->log_file_head_offset = next_offset;
    return TRUE;
}

static int
mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
{
        struct mail_transaction_log_view *log_view = ctx->view->log_view;
    struct mail_index_view *view = ctx->view;
    const struct mail_transaction_header *hdr;
    uint32_t seq;
    uoff_t offset;
    int ret;
    bool synced_to_map;

    for (;;) {
        /* Get the next transaction from log. */
        ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
                             &ctx->data);
        if (ret <= 0) {
            if (ret < 0)
                return -1;

            ctx->hdr = NULL;
            ctx->last_read = TRUE;
            return 0;
        }

        hdr = ctx->hdr;
        if (!mail_index_view_sync_want(ctx, hdr)) {
            /* This is a visible record that we don't want to
               sync. */
            continue;
        }

        mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);

        /* If we started from a map that we didn't create ourself,
           some of the transactions may already be synced. at the end
           of this view sync we'll update file_seq=0 so that this check
           always becomes FALSE for subsequent syncs. */
        synced_to_map = view->map->hdr.log_file_seq != 0 &&
            LOG_IS_BEFORE(seq, offset,
                      view->map->hdr.log_file_seq,
                      view->map->hdr.log_file_head_offset);

        /* Apply transaction to view's mapping if needed (meaning we
           didn't just re-map the view to head mapping). */
        if (ctx->sync_map_update && !synced_to_map) {
            i_assert((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0 ||
                 (hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0);

            if (mail_index_sync_record(&ctx->sync_map_ctx,
                           hdr, ctx->data) < 0)
                return -1;
        }

        /* skip changes committed by hidden transactions (eg. in IMAP
           store +flags.silent command) */
        if (view_sync_is_hidden(view, seq, offset))
            continue;
        break;
    }
    return 1;
}

#define FLAG_UPDATE_IS_INTERNAL(u) \
    ((((u)->add_flags | (u)->remove_flags) & \
      ~MAIL_INDEX_MAIL_FLAG_DIRTY) == 0)

static bool
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
                 struct mail_index_view_sync_rec *rec)
{
    const struct mail_transaction_header *hdr = ctx->hdr;
    const void *data = ctx->data;

    switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
    case MAIL_TRANSACTION_APPEND: {
        /* data contains the appended records, but we don't care */
        rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
        rec->uid1 = rec->uid2 = 0;
        ctx->data_offset += hdr->size;
        break;
    }
    case MAIL_TRANSACTION_EXPUNGE: {
        const struct mail_transaction_expunge *exp =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
            /* this is simply a request for expunge */
            ctx->data_offset = ctx->hdr->size;
            return 0;
        }

        /* data contains mail_transaction_expunge[] */
        rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
        rec->uid1 = exp->uid1;
        rec->uid2 = exp->uid2;

        ctx->data_offset += sizeof(*exp);
        break;
    }
    case MAIL_TRANSACTION_FLAG_UPDATE: {
        const struct mail_transaction_flag_update *update =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_flag_update[] */
        for (;;) {
            ctx->data_offset += sizeof(*update);
            if (!FLAG_UPDATE_IS_INTERNAL(update))
                break;

            /* skip internal flag changes */
            if (ctx->data_offset == ctx->hdr->size)
                return 0;

            update = CONST_PTR_OFFSET(data, ctx->data_offset);
        }

        rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
        rec->uid1 = update->uid1;
        rec->uid2 = update->uid2;
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_UPDATE: {
        const struct mail_transaction_keyword_update *update = data;
        const uint32_t *uids;

        /* data contains mail_transaction_keyword_update header,
           the keyword name and an array of { uint32_t uid1, uid2; } */

        if (ctx->data_offset == 0) {
            /* skip over the header and name */
            ctx->data_offset = sizeof(*update) + update->name_size;
            if ((ctx->data_offset % 4) != 0)
                ctx->data_offset += 4 - (ctx->data_offset % 4);
        }

        uids = CONST_PTR_OFFSET(data, ctx->data_offset);
        rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
        rec->uid1 = uids[0];
        rec->uid2 = uids[1];

        ctx->data_offset += sizeof(uint32_t) * 2;
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_RESET: {
        const struct mail_transaction_keyword_reset *reset =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_keyword_reset[] */
        rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
        rec->uid1 = reset->uid1;
        rec->uid2 = reset->uid2;
        ctx->data_offset += sizeof(*reset);
        break;
    }
    default:
        ctx->hdr = NULL;
        return FALSE;
    }
    return TRUE;
}

bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
                   struct mail_index_view_sync_rec *sync_rec)
{
    int ret;

    do {
        if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
            ret = mail_index_view_sync_get_next_transaction(ctx);
            if (ret <= 0) {
                if (ret < 0)
                    ctx->failed = TRUE;
                return FALSE;
            }

            ctx->data_offset = 0;
        }
    } while (!mail_index_view_sync_get_rec(ctx, sync_rec));

    return TRUE;
}

void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
                       const ARRAY_TYPE(seq_range) **expunges_r)
{
    *expunges_r = &ctx->expunges;
}

static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
{
    const struct mail_index_view_log_sync_area *syncs;
    unsigned int i, count;

    if (!array_is_created(&view->syncs_hidden))
        return;

    /* Clean up to view's tail */
    syncs = array_get(&view->syncs_hidden, &count);
    for (i = 0; i < count; i++) {
        if ((syncs[i].log_file_offset +
             syncs[i].length > view->log_file_expunge_offset &&
                     syncs[i].log_file_seq == view->log_file_expunge_seq) ||
            syncs[i].log_file_seq > view->log_file_expunge_seq)
            break;
    }
    if (i > 0)
        array_delete(&view->syncs_hidden, 0, i);
}

int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx)
{
        struct mail_index_view_sync_ctx *ctx = *_ctx;
        struct mail_index_view *view = ctx->view;
    int ret = ctx->failed ? -1 : 0;

    i_assert(view->syncing);

    *_ctx = NULL;

    if ((!ctx->last_read || view->inconsistent) &&
        (ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
        /* we didn't sync everything */
        view->inconsistent = TRUE;
        ret = -1;
    }

    if (view->sync_new_map != NULL) {
        mail_index_unmap(&view->map);
        view->map = view->sync_new_map;
        view->sync_new_map = NULL;
    }

    i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count);

    if (!ctx->skipped_expunges) {
        view->log_file_expunge_seq = view->log_file_head_seq;
        view->log_file_expunge_offset = view->log_file_head_offset;
    }

    if (ctx->sync_map_update) {
        /* log offsets have no meaning in views. make sure they're not
           tried to be used wrong by setting them to zero. */
        view->map->hdr.log_file_seq = 0;
        view->map->hdr.log_file_head_offset = 0;
        view->map->hdr.log_file_tail_offset = 0;
    }

    mail_index_sync_map_deinit(&ctx->sync_map_ctx);
    mail_index_view_sync_clean_log_syncs(ctx->view);

#ifdef DEBUG
    mail_index_map_check(view->map);
#endif

    /* set log view to empty range so unneeded memory gets freed */
    mail_transaction_log_view_clear(view->log_view,
                    view->log_file_expunge_seq);

    if (array_is_created(&ctx->expunges))
        array_free(&ctx->expunges);

    view->syncing = FALSE;
    i_free(ctx);
    return ret;
}

void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
                        uint32_t log_file_seq,
                        uoff_t log_file_offset,
                        unsigned int length)
{
    struct mail_index_view_log_sync_area *area;

    if (!array_is_created(&view->syncs_hidden))
        i_array_init(&view->syncs_hidden, 32);

    area = array_append_space(&view->syncs_hidden);
    area->log_file_seq = log_file_seq;
    area->log_file_offset = log_file_offset;
    area->length = length;
}