src/lib-index/mail-index-view-sync.c

	mail-index-view-sync.c revision 8afec4d1a32b78f540257a27769b372aad753384
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi/* Copyright (C) 2003-2004 Timo Sirainen */
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen#include "lib.h"
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen#include "array.h"
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen#include "buffer.h"
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi#include "mail-index-view-private.h"
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen#include "mail-index-sync-private.h"
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi#include "mail-transaction-log.h"
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi#include "mail-transaction-util.h"
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvistruct mail_index_view_sync_ctx {
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    struct mail_index_view *view;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    enum mail_transaction_type visible_sync_mask;
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    struct mail_index_sync_map_ctx sync_map_ctx;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    array_t ARRAY_DEFINE(expunges, struct mail_transaction_expunge);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    const struct mail_transaction_header *hdr;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    const void *data;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    size_t data_offset;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    unsigned int skipped_some:1;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    unsigned int last_read:1;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    unsigned int sync_map_update:1;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi};
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvistruct mail_index_view_log_sync_pos {
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    uint32_t log_file_seq;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    uoff_t log_file_offset;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi};
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvistatic void
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvimail_transaction_log_sort_expunges(array_t *expunges,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                   const struct mail_transaction_expunge *src,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                   size_t src_size)
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi{
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    ARRAY_SET_TYPE(expunges, struct mail_transaction_expunge);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    const struct mail_transaction_expunge *src_end;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    struct mail_transaction_expunge *dest;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    struct mail_transaction_expunge new_exp;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    unsigned int first, i, dest_count;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    i_assert(src_size % sizeof(*src) == 0);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    /* @UNSAFE */
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    dest = array_get_modifyable(expunges, &dest_count);
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    if (dest_count == 0) {
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi        array_append(expunges, src, src_size / sizeof(*src));
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        return;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    }
9b5576a265cbadb1f0b3c3d5e40e928e1fed1ec9Timo Sirainen
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    src_end = CONST_PTR_OFFSET(src, src_size);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    for (i = 0; src != src_end; src++) {
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        /* src[] must be sorted. */
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        i_assert(src+1 == src_end || src->uid2 < src[1].uid1);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        i_assert(src->uid1 <= src->uid2);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi        for (; i < dest_count; i++) {
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi            if (src->uid1 < dest[i].uid1)
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                break;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        }
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        new_exp = *src;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        first = i;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        while (i < dest_count && src->uid2 >= dest[i].uid1-1) {
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            /* we can/must merge with next record */
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            if (new_exp.uid2 < dest[i].uid2)
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                new_exp.uid2 = dest[i].uid2;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            i++;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi        }
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        if (first > 0 && new_exp.uid1 <= dest[first-1].uid2+1) {
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            /* continue previous record */
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            if (dest[first-1].uid2 < new_exp.uid2)
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                dest[first-1].uid2 = new_exp.uid2;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        } else if (i == first) {
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            array_insert(expunges, i, &new_exp, 1);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            i++; first++;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi            dest = array_get_modifyable(expunges, &dest_count);
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi        } else {
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen            /* use next record */
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            dest[first] = new_exp;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            first++;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        }
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        if (i > first) {
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen            array_delete(expunges, first, i - first);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            dest = array_get_modifyable(expunges, &dest_count);
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen            i = first;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        }
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    }
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen}
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvistatic int
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomiview_sync_get_expunges(struct mail_index_view *view, array_t *expunges_r)
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi{
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    ARRAY_SET_TYPE(expunges_r, struct mail_transaction_expunge);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    const struct mail_transaction_header *hdr;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    struct mail_transaction_expunge *src, *src_end, *dest;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    const void *data;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    unsigned int count;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    int ret;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    if (mail_transaction_log_view_set(view->log_view,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                      view->log_file_seq,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                      view->log_file_offset,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                      view->index->hdr->log_file_seq,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                      view->index->hdr->log_file_int_offset,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                      MAIL_TRANSACTION_EXPUNGE) < 0)
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        return -1;
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    ARRAY_CREATE(expunges_r, default_pool,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi             struct mail_transaction_expunge, 64);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    while ((ret = mail_transaction_log_view_next(view->log_view,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                             &hdr, &data, NULL)) > 0) {
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        i_assert((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        mail_transaction_log_sort_expunges(expunges_r, data, hdr->size);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    }
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    if (ret < 0) {
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi        array_free(expunges_r);
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi        return -1;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    }
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    /* convert to sequences */
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    src = dest = array_get_modifyable(expunges_r, &count);
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    src_end = src + count;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    for (; src != src_end; src++) {
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi        ret = mail_index_lookup_uid_range(view, src->uid1,
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi                          src->uid2,
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi                          &dest->uid1,
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi                          &dest->uid2);
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi        i_assert(ret == 0);
dead275d577c87b8f8ba476a7709db758dfffb9bTimo Sirainen
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi        if (dest->uid1 == 0)
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi            count--;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi        else
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi            dest++;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    }
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    array_delete(expunges_r, count, array_count(expunges_r) - count);
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    return 0;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi}
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi#define MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK \
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    (MAIL_INDEX_SYNC_TYPE_FLAGS | \
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi     MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET | \
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi     MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD | MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE)
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi#define MAIL_TRANSACTION_VISIBLE_SYNC_MASK \
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_APPEND | \
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi     MAIL_TRANSACTION_FLAG_UPDATE | MAIL_TRANSACTION_KEYWORD_UPDATE | \
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi     MAIL_TRANSACTION_KEYWORD_RESET)
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomiint mail_index_view_sync_begin(struct mail_index_view *view,
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi                               enum mail_index_sync_type sync_mask,
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi                   struct mail_index_view_sync_ctx **ctx_r)
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi{
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    const struct mail_index_header *hdr;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    struct mail_index_view_sync_ctx *ctx;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    struct mail_index_map *map;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    enum mail_transaction_type log_get_mask, visible_mask;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    array_t expunges = { 0, 0 };
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi    /* We must sync flags as long as view is mmap()ed, as the flags may
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen       have already changed under us. */
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    i_assert((sync_mask & MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK) ==
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen         MAIL_INDEX_VIEW_VISIBLE_FLAGS_MASK);
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    /* Currently we're not handling correctly expunges + no-appends case */
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    i_assert((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) == 0 ||
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen         (sync_mask & MAIL_INDEX_SYNC_TYPE_APPEND) != 0);
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    i_assert(!view->syncing);
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi    i_assert(view->transactions == 0);
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    if (mail_index_view_lock_head(view, TRUE) < 0)
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        return -1;
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    hdr = view->index->hdr;
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) {
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen        /* get list of all expunges first */
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen        if (view_sync_get_expunges(view, &expunges) < 0)
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            return -1;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    }
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    /* only flags, appends and expunges can be left to be synced later */
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen    visible_mask = mail_transaction_type_mask_get(sync_mask);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    i_assert((visible_mask & ~MAIL_TRANSACTION_VISIBLE_SYNC_MASK) == 0);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    /* we want to also get non-visible changes. especially because we use
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi       the returned skipped-flag in mail_transaction_log_view_next() to
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi       tell us if any visible changes were skipped. */
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    log_get_mask = visible_mask | (MAIL_TRANSACTION_TYPE_MASK ^
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen                       MAIL_TRANSACTION_VISIBLE_SYNC_MASK);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    if (mail_transaction_log_view_set(view->log_view,
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen                      view->log_file_seq,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                      view->log_file_offset,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                      hdr->log_file_seq,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                      hdr->log_file_int_offset,
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi                      log_get_mask) < 0) {
7ebda61d3fd228451530ca8faa33380105230ebeAki Tuomi        if (array_is_created(&expunges))
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi            array_free(&expunges);
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi        return -1;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    }
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    ctx = i_new(struct mail_index_view_sync_ctx, 1);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    ctx->view = view;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    ctx->visible_sync_mask = visible_mask;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    ctx->expunges = expunges;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    mail_index_sync_map_init(&ctx->sync_map_ctx, view,
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi                 MAIL_INDEX_SYNC_HANDLER_VIEW);
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi    if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0 &&
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi        (sync_mask & MAIL_INDEX_SYNC_TYPE_APPEND) != 0) {
796beea86bcdc92e4ba2f4865414a951b1717e5cTimo Sirainen        view->sync_new_map = view->index->map;
dfe2b5d36666dfc941821dadf59267d28ff58ff5Aki Tuomi        view->sync_new_map->refcount++;
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi
b28403dc4da6ee942ff18315596b43d4c4e9b7bdMartti Rannanjärvi        /* keep the old mapping without expunges until we're
           fully synced */
    } else {
        /* We need a private copy of the map if we don't want to
           sync expunges.

           If view's map is the head map, it means that it contains
           already all the latest changes and there's no need for us
           to apply any changes to it. This can only happen if there
           hadn't been any expunges. */
        uint32_t old_records_count = view->map->records_count;

        if (view->map != view->index->map) {
            /* Using non-head mapping. We have to apply
               transactions to it to get latest changes into it. */
            ctx->sync_map_update = TRUE;

            /* Copy only the mails that we see currently, since
               we're going to append the new ones when we see
               their transactions. */
            i_assert(view->map->records_count >=
                 view->hdr.messages_count);
            view->map->records_count = view->hdr.messages_count;
        }

        map = mail_index_map_clone(view->map,
                       view->map->hdr.record_size);
        view->map->records_count = old_records_count;
        mail_index_unmap(view->index, view->map);
        view->map = map;

        if (ctx->sync_map_update) {
            /* Start the sync using our old view's header.
               The old view->hdr may differ from map->hdr if
               another view sharing the map with us had synced
               itself. */
            i_assert(map->hdr_base == map->hdr_copy_buf->data);
            buffer_write(map->hdr_copy_buf, 0,
                     &view->hdr, sizeof(view->hdr));
            map->hdr = view->hdr;
        }

        i_assert(map->records_count == map->hdr.messages_count);
    }

    /* Syncing the view invalidates all previous looked up records.
       Unreference the mappings this view keeps because of them. */
    mail_index_view_unref_maps(view);
    view->syncing = TRUE;

    *ctx_r = ctx;
    return 0;
}

static int view_is_transaction_synced(struct mail_index_view *view,
                      uint32_t seq, uoff_t offset)
{
    const struct mail_index_view_log_sync_pos *pos;
    unsigned int i, count;

    if (!array_is_created(&view->log_syncs))
        return FALSE;

    pos = array_get(&view->log_syncs, &count);
    for (i = 0; i < count; i++) {
        if (pos[i].log_file_offset == offset &&
            pos[i].log_file_seq == seq)
            return TRUE;
    }

    return FALSE;
}

static int
mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
{
        struct mail_transaction_log_view *log_view = ctx->view->log_view;
    struct mail_index_view *view = ctx->view;
    uint32_t seq;
    uoff_t offset;
    int ret, skipped;

    for (;;) {
        /* Get the next transaction from log. */
        ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
                             &ctx->data, &skipped);
        if (ret <= 0) {
            if (ret < 0)
                return -1;

            ctx->hdr = NULL;
            ctx->last_read = TRUE;
            return 0;
        }

        mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);

        if (skipped) {
            /* We skipped some (visible) transactions that were
               outside our sync mask. */
            ctx->skipped_some = TRUE;
        } else if (!ctx->skipped_some) {
            /* We haven't skipped anything while syncing this view.
               Update this view's synced log offset. */
            view->log_file_seq = seq;
            view->log_file_offset = offset + sizeof(*ctx->hdr) +
                ctx->hdr->size;
        }

        /* skip flag changes that we committed ourself or have
           already synced */
        if (view_is_transaction_synced(view, seq, offset))
            continue;

        /* Apply transaction to view's mapping if needed (meaning we
           didn't just re-map the view to head mapping). */
        if (ctx->sync_map_update) {
            i_assert((ctx->hdr->type &
                  MAIL_TRANSACTION_EXPUNGE) == 0);

            if (mail_index_sync_record(&ctx->sync_map_ctx,
                           ctx->hdr, ctx->data) < 0)
                return -1;
        }

        if ((ctx->hdr->type & ctx->visible_sync_mask) == 0) {
            /* non-visible change that we just wanted to update
               to map. */
            continue;
        }
        break;
    }

    if (ctx->skipped_some) {
        /* We've been skipping some transactions, which means we'll
           go through these same transaction again later. Since we're
           syncing this one, we don't want to do it again. */
        mail_index_view_add_synced_transaction(view, seq, offset);
    }
    return 1;
}

#define FLAG_UPDATE_IS_INTERNAL(u) \
    ((((u)->add_flags | (u)->remove_flags) & \
      ~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0)

static int
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
                 struct mail_index_sync_rec *rec)
{
    const struct mail_transaction_header *hdr = ctx->hdr;
    const void *data = ctx->data;

    switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
    case MAIL_TRANSACTION_APPEND: {
        /* data contains the appended records, but we don't care */
        rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
        rec->uid1 = rec->uid2 = 0;
        ctx->data_offset += hdr->size;
        break;
    }
    case MAIL_TRANSACTION_EXPUNGE: {
        const struct mail_transaction_expunge *exp =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_expunge[] */
        ctx->data_offset += sizeof(*exp);
                mail_index_sync_get_expunge(rec, exp);
        break;
    }
    case MAIL_TRANSACTION_FLAG_UPDATE: {
        const struct mail_transaction_flag_update *update =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_flag_update[] */
        for (;;) {
            ctx->data_offset += sizeof(*update);
            if (!FLAG_UPDATE_IS_INTERNAL(update))
                break;

            /* skip internal flag changes */
            if (ctx->data_offset == ctx->hdr->size)
                return 0;

            update = CONST_PTR_OFFSET(data, ctx->data_offset);
        }
                mail_index_sync_get_update(rec, update);
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_UPDATE: {
        const struct mail_transaction_keyword_update *update = data;
        const uint32_t *uids;

        /* data contains mail_transaction_keyword_update header,
           the keyword name and an array of { uint32_t uid1, uid2; } */

        if (ctx->data_offset == 0) {
            /* skip over the header and name */
            ctx->data_offset = sizeof(*update) + update->name_size;
            if ((ctx->data_offset % 4) != 0)
                ctx->data_offset += 4 - (ctx->data_offset % 4);
        }

        uids = CONST_PTR_OFFSET(data, ctx->data_offset);
        /* FIXME: rec->keyword_idx isn't set, but no-one cares
           currently. perhaps the whole view syncing API should just
           be returning type and uid range.. */
        rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
        rec->uid1 = uids[0];
        rec->uid2 = uids[1];

        ctx->data_offset += sizeof(uint32_t) * 2;
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_RESET: {
        const struct mail_transaction_keyword_reset *reset =
            CONST_PTR_OFFSET(data, ctx->data_offset);

        /* data contains mail_transaction_keyword_reset[] */
        rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
        rec->uid1 = reset->uid1;
        rec->uid2 = reset->uid2;
        ctx->data_offset += sizeof(*reset);
        break;
    }
    default:
        i_unreached();
    }
    return 1;
}

int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
                  struct mail_index_sync_rec *sync_rec)
{
    int ret;

    do {
        if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
            ret = mail_index_view_sync_get_next_transaction(ctx);
            if (ret <= 0)
                return ret;

            ctx->data_offset = 0;
        }
    } while (!mail_index_view_sync_get_rec(ctx, sync_rec));

    return 1;
}

const uint32_t *
mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
                  unsigned int *count_r)
{
    const struct mail_transaction_expunge *data;

    data = array_get(&ctx->expunges, count_r);
    return (const uint32_t *)data;
}

static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view_sync_ctx *ctx)
{
    struct mail_index_view *view = ctx->view;
    const struct mail_index_view_log_sync_pos *pos;
    unsigned int i, count;

    if (!array_is_created(&view->log_syncs))
        return;

    if (!ctx->skipped_some) {
        /* Nothing skipped. Clean it up the quick way. */
        array_clear(&view->log_syncs);
        return;
    }

    /* Clean up until view's current syncing position */
    pos = array_get(&view->log_syncs, &count);
    for (i = 0; i < count; i++) {
        if ((pos[i].log_file_offset >= view->log_file_offset &&
                     pos[i].log_file_seq == view->log_file_seq) ||
            pos[i].log_file_seq > view->log_file_seq)
            break;
    }
    if (i > 0)
        array_delete(&view->log_syncs, 0, i);
}

void mail_index_view_sync_end(struct mail_index_view_sync_ctx *ctx)
{
        struct mail_index_view *view = ctx->view;

    i_assert(view->syncing);

    if (ctx->sync_map_update)
        mail_index_sync_map_deinit(&ctx->sync_map_ctx);
    mail_index_view_sync_clean_log_syncs(ctx);

    if (!ctx->last_read && ctx->hdr != NULL &&
        ctx->data_offset != ctx->hdr->size) {
        /* we didn't sync everything */
        view->inconsistent = TRUE;
    }

    if (view->sync_new_map != NULL) {
        mail_index_unmap(view->index, view->map);
        view->map = view->sync_new_map;
        view->sync_new_map = NULL;
    }
    view->hdr = view->map->hdr;

    (void)mail_transaction_log_view_set(view->log_view,
                        view->log_file_seq,
                        view->log_file_offset,
                        view->log_file_seq,
                        view->log_file_offset,
                        MAIL_TRANSACTION_TYPE_MASK);

    if (array_is_created(&ctx->expunges))
        array_free(&ctx->expunges);

    view->syncing = FALSE;
    i_free(ctx);
}

void mail_index_view_add_synced_transaction(struct mail_index_view *view,
                        uint32_t log_file_seq,
                        uoff_t log_file_offset)
{
    struct mail_index_view_log_sync_pos *pos;

    if (!array_is_created(&view->log_syncs)) {
        ARRAY_CREATE(&view->log_syncs, default_pool,
                             struct mail_index_view_log_sync_pos, 32);
    }

    pos = array_append_space(&view->log_syncs);
    pos->log_file_seq = log_file_seq;
    pos->log_file_offset = log_file_offset;
}