src/lib-index/mail-index-view-sync.c

	mail-index-view-sync.c revision 56561d472db25ebda35ae6afdc7f7deb75c323fc
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "buffer.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-view-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-sync-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-transaction-log.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-transaction-util.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstruct mail_index_view_sync_ctx {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    struct mail_index_view *view;
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    enum mail_transaction_type trans_sync_mask;
56561d472db25ebda35ae6afdc7f7deb75c323fcTimo Sirainen    struct mail_index_sync_map_ctx sync_map_ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    buffer_t *expunges;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    const struct mail_transaction_header *hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    const void *data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    size_t data_offset;
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    unsigned int skipped_some:1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    unsigned int last_read:1;
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen    unsigned int sync_map_update:1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen};
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenview_sync_get_expunges(struct mail_index_view *view, buffer_t **expunges_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    const struct mail_transaction_header *hdr;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    struct mail_transaction_expunge *src, *src_end, *dest;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    const void *data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    size_t size;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    int ret;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen    *expunges_r = buffer_create_dynamic(default_pool, 512);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    /* with mask 0 we don't get anything, we'll just read the expunges
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen       while seeking to end */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (mail_transaction_log_view_set(view->log_view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      view->log_file_seq,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      view->log_file_offset,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      view->index->hdr->log_file_seq,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      view->index->hdr->log_file_offset,
659fe5d24825b160cae512538088020d97a60239Timo Sirainen                      MAIL_TRANSACTION_EXPUNGE) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return -1;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    while ((ret = mail_transaction_log_view_next(view->log_view,
659fe5d24825b160cae512538088020d97a60239Timo Sirainen                             &hdr, &data, NULL)) > 0) {
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        mail_transaction_log_sort_expunges(*expunges_r,
659fe5d24825b160cae512538088020d97a60239Timo Sirainen                           data, hdr->size);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    if (ret == 0) {
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        /* convert to sequences */
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        src = dest = buffer_get_modifyable_data(*expunges_r, &size);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        src_end = src + size / sizeof(*src);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        for (; src != src_end; src++) {
659fe5d24825b160cae512538088020d97a60239Timo Sirainen            ret = mail_index_lookup_uid_range(view, src->uid1,
659fe5d24825b160cae512538088020d97a60239Timo Sirainen                              src->uid2,
659fe5d24825b160cae512538088020d97a60239Timo Sirainen                              &dest->uid1,
659fe5d24825b160cae512538088020d97a60239Timo Sirainen                              &dest->uid2);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen            i_assert(ret == 0);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen
659fe5d24825b160cae512538088020d97a60239Timo Sirainen            if (dest->uid1 == 0)
659fe5d24825b160cae512538088020d97a60239Timo Sirainen                size -= sizeof(*dest);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen            else
659fe5d24825b160cae512538088020d97a60239Timo Sirainen                dest++;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        }
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        buffer_set_used_size(*expunges_r, size);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    } else {
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        buffer_set_used_size(*expunges_r, 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
659fe5d24825b160cae512538088020d97a60239Timo Sirainen
659fe5d24825b160cae512538088020d97a60239Timo Sirainen    return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d6f50f100ce17fa4b3a89e9567a5ff993b38b872Timo Sirainen#define MAIL_INDEX_VIEW_VISIBLE_SYNC_MASK \
d6f50f100ce17fa4b3a89e9567a5ff993b38b872Timo Sirainen    (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_APPEND | \
d6f50f100ce17fa4b3a89e9567a5ff993b38b872Timo Sirainen     MAIL_TRANSACTION_FLAG_UPDATE)
d6f50f100ce17fa4b3a89e9567a5ff993b38b872Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_index_view_sync_begin(struct mail_index_view *view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                               enum mail_index_sync_type sync_mask,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                   struct mail_index_view_sync_ctx **ctx_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    const struct mail_index_header *hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    struct mail_index_view_sync_ctx *ctx;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen    struct mail_index_map *map;
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    enum mail_transaction_type mask, want_mask;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    buffer_t *expunges = NULL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    /* We must sync flags as long as view is mmap()ed, as the flags may
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen       have already changed under us. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    i_assert((sync_mask & MAIL_INDEX_SYNC_TYPE_FLAGS) != 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    i_assert(!view->syncing);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
31ddc75584c5cde53d2e78a737587f2e7fdcb0d2Timo Sirainen    if (mail_index_view_lock_head(view, TRUE) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    hdr = view->index->hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        /* get list of all expunges first */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        if (view_sync_get_expunges(view, &expunges) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    /* only flags, appends and expunges can be left to be synced later */
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    want_mask = mail_transaction_type_mask_get(sync_mask);
d6f50f100ce17fa4b3a89e9567a5ff993b38b872Timo Sirainen    i_assert((want_mask & ~MAIL_INDEX_VIEW_VISIBLE_SYNC_MASK) == 0);
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    mask = want_mask |
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen        (MAIL_TRANSACTION_TYPE_MASK ^
d6f50f100ce17fa4b3a89e9567a5ff993b38b872Timo Sirainen         MAIL_INDEX_VIEW_VISIBLE_SYNC_MASK);
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (mail_transaction_log_view_set(view->log_view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      view->log_file_seq,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      view->log_file_offset,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      hdr->log_file_seq,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      hdr->log_file_offset, mask) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        if (expunges != NULL)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            buffer_free(expunges);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    ctx = i_new(struct mail_index_view_sync_ctx, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    ctx->view = view;
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    ctx->trans_sync_mask = want_mask;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    ctx->expunges = expunges;
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen
56561d472db25ebda35ae6afdc7f7deb75c323fcTimo Sirainen    ctx->sync_map_ctx.view = view;
56561d472db25ebda35ae6afdc7f7deb75c323fcTimo Sirainen    ctx->sync_map_ctx.last_ext_id = (uint32_t)-1;
56561d472db25ebda35ae6afdc7f7deb75c323fcTimo Sirainen
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen    if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        view->new_map = view->index->map;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        view->new_map->refcount++;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        /* keep the old mapping without expunges until we're
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen           fully synced */
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen    } else {
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen        /* we need a private copy of the map if we don't want to
bbf796c17f02538058d7559bfe96d677e5b55015Timo Sirainen           sync expunges. we need to sync mapping only if we're not
bbf796c17f02538058d7559bfe96d677e5b55015Timo Sirainen           using the latest one. */
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        if (view->map != view->index->map)
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen            ctx->sync_map_update = TRUE;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
7e94cf9d70ce9fdeccb7a85ff400b899e6386f36Timo Sirainen        map = mail_index_map_to_memory(view->map,
7e94cf9d70ce9fdeccb7a85ff400b899e6386f36Timo Sirainen                           view->map->hdr->record_size);
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        mail_index_unmap(view->index, view->map);
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        view->map = map;
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen    }
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen
7c5b51bdf43a98e12c654ad437e0b258c5fffbc1Timo Sirainen    mail_index_view_unref_maps(view);
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen    view->syncing = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    *ctx_r = ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int view_is_transaction_synced(struct mail_index_view *view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                      uint32_t seq, uoff_t offset)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    const unsigned char *data, *end;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    size_t size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (view->log_syncs == NULL)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    data = buffer_get_data(view->log_syncs, &size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    end = data + size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    for (; data < end; ) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        if (*((const uoff_t *)data) == offset &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            *((const uint32_t *)(data + sizeof(uoff_t))) == seq)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        data += sizeof(uoff_t) + sizeof(uint32_t);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mail_index_view_sync_next_trans(struct mail_index_view_sync_ctx *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                       uint32_t *seq_r, uoff_t *offset_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        struct mail_transaction_log_view *log_view = ctx->view->log_view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    struct mail_index_view *view = ctx->view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    int ret, skipped;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    ret = mail_transaction_log_view_next(log_view, &ctx->hdr, &ctx->data,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                         &skipped);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (ret <= 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        if (ret < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d6f50f100ce17fa4b3a89e9567a5ff993b38b872Timo Sirainen        ctx->hdr = NULL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        ctx->last_read = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (skipped)
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen        ctx->skipped_some = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    mail_transaction_log_view_get_prev_pos(log_view, seq_r, offset_r);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    /* skip flag changes that we committed ourself or have already synced */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (view_is_transaction_synced(view, *seq_r, *offset_r))
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen    /* expunges have to be synced afterwards so that caller can still get
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen       information of the messages. otherwise caller most likely wants to
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen       see only updated information. */
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen    if (ctx->sync_map_update &&
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        (ctx->hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0) {
56561d472db25ebda35ae6afdc7f7deb75c323fcTimo Sirainen        if (mail_index_sync_record(&ctx->sync_map_ctx, ctx->hdr,
d143077bd518de129b8d446fb58e003903e50867Timo Sirainen                       ctx->data) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    if ((ctx->hdr->type & ctx->trans_sync_mask) == 0)
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen        return 0;
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen#define FLAG_UPDATE_IS_INTERNAL(u, empty) \
88553367d677170a4b703b9d52aac9eabf91c656Timo Sirainen    ((((u)->add_flags | (u)->remove_flags) & \
88553367d677170a4b703b9d52aac9eabf91c656Timo Sirainen      ~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0 && \
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen     memcmp((u)->add_keywords, empty, INDEX_KEYWORDS_BYTE_COUNT) == 0 && \
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen     memcmp((u)->add_keywords, empty, INDEX_KEYWORDS_BYTE_COUNT) == 0)
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainenstatic int
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainenmail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen                 struct mail_index_sync_rec *rec)
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen{
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen    static keywords_mask_t empty_keywords = { 0, };
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    const struct mail_transaction_header *hdr = ctx->hdr;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    const void *data = ctx->data;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    case MAIL_TRANSACTION_APPEND: {
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen        rec->uid1 = rec->uid2 = 0;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        ctx->data_offset += hdr->size;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        break;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    }
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    case MAIL_TRANSACTION_EXPUNGE: {
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        const struct mail_transaction_expunge *exp =
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen            CONST_PTR_OFFSET(data, ctx->data_offset);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        ctx->data_offset += sizeof(*exp);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen                mail_index_sync_get_expunge(rec, exp);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        break;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    }
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    case MAIL_TRANSACTION_FLAG_UPDATE: {
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        const struct mail_transaction_flag_update *update =
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen            CONST_PTR_OFFSET(data, ctx->data_offset);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen        for (;;) {
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen            ctx->data_offset += sizeof(*update);
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen            if (!FLAG_UPDATE_IS_INTERNAL(update, empty_keywords))
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                break;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen            if (ctx->data_offset == ctx->hdr->size)
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                return 0;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen        }
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen                mail_index_sync_get_update(rec, update);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        break;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    }
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    default:
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen        i_unreached();
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen    }
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen    return 1;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen}
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                  struct mail_index_sync_rec *sync_rec)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    struct mail_index_view *view = ctx->view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    uint32_t seq;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    uoff_t offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    int ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen    do {
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen        if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen            ctx->data_offset = 0;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen            do {
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                ret = mail_index_view_sync_next_trans(ctx, &seq,
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                                      &offset);
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                if (ret < 0)
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                    return -1;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                if (ctx->last_read)
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                    return 0;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen                if (!ctx->skipped_some) {
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                    view->log_file_seq = seq;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                    view->log_file_offset = offset +
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                        sizeof(*ctx->hdr) +
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                        ctx->hdr->size;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                }
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen            } while (ret == 0);
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen            if (ctx->skipped_some) {
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                mail_index_view_add_synced_transaction(view,
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                                       seq,
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen                                       offset);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen            }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        }
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen    } while (!mail_index_view_sync_get_rec(ctx, sync_rec));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenconst uint32_t *
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                  size_t *count_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    const uint32_t *data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    size_t size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    data = buffer_get_data(ctx->expunges, &size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    *count_r = size / (sizeof(uint32_t)*2);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    return data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenvoid mail_index_view_sync_end(struct mail_index_view_sync_ctx *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        struct mail_index_view *view = ctx->view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    i_assert(view->syncing);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    if (view->log_syncs != NULL && !ctx->skipped_some)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        buffer_set_used_size(view->log_syncs, 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (!ctx->last_read && ctx->hdr != NULL &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        ctx->data_offset != ctx->hdr->size) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        /* we didn't sync everything */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        view->inconsistent = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen    if (view->new_map != NULL) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        mail_index_unmap(view->index, view->map);
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        view->map = view->new_map;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen        view->new_map = NULL;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
bcf5af3335a814e4923ba1bf2e0d80eb6dabfb22Timo Sirainen    if ((ctx->trans_sync_mask & MAIL_TRANSACTION_APPEND) != 0)
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen        view->messages_count = view->map->records_count;
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen    (void)mail_transaction_log_view_set(view->log_view,
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen                        view->log_file_seq,
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen                        view->log_file_offset,
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen                        view->log_file_seq,
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen                        view->log_file_offset,
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen                        MAIL_TRANSACTION_TYPE_MASK);
7888a9d2008eab9985096c46e1da9ee985c22a2aTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (ctx->expunges != NULL)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        buffer_free(ctx->expunges);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    view->syncing = FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    i_free(ctx);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenvoid mail_index_view_add_synced_transaction(struct mail_index_view *view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                        uint32_t log_file_seq,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen                        uoff_t log_file_offset)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen    if (view->log_syncs == NULL)
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen        view->log_syncs = buffer_create_dynamic(default_pool, 128);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    buffer_append(view->log_syncs, &log_file_offset,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen              sizeof(log_file_offset));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    buffer_append(view->log_syncs, &log_file_seq, sizeof(log_file_seq));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}