src/lib-index/mail-index-modseq.c

	mail-index-modseq.c revision 5f5870385cff47efd2f58e7892f251cf13761528
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen/* Copyright (c) 2008-2012 Dovecot authors, see the included COPYING file */
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen#include "lib.h"
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen#include "array.h"
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen#include "mail-transaction-log-private.h"
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen#include "mail-index-private.h"
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen#include "mail-index-sync-private.h"
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen#include "mail-index-modseq.h"
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo SirainenARRAY_DEFINE_TYPE(modseqs, uint64_t);
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainenenum modseq_metadata_idx {
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    /* must be in the same order as enum mail_flags */
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    METADATA_MODSEQ_IDX_ANSWERED = 0,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    METADATA_MODSEQ_IDX_FLAGGED,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    METADATA_MODSEQ_IDX_DELETED,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    METADATA_MODSEQ_IDX_SEEN,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    METADATA_MODSEQ_IDX_DRAFT,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    METADATA_MODSEQ_IDX_KEYWORD_START
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen};
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainenstruct metadata_modseqs {
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    ARRAY_TYPE(modseqs) modseqs;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen};
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainenstruct mail_index_map_modseq {
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen    /* indexes use enum modseq_metadata_idx */
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    ARRAY_DEFINE(metadata_modseqs, struct metadata_modseqs);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen};
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainenstruct mail_index_modseq_sync {
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    struct mail_index_sync_map_ctx *sync_map_ctx;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    struct mail_index_view *view;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    struct mail_transaction_log_view *log_view;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    struct mail_index_map_modseq *mmap;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    uint64_t highest_modseq;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen};
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainenvoid mail_index_modseq_init(struct mail_index *index)
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen{
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen    index->modseq_ext_id =
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        mail_index_ext_register(index, MAIL_INDEX_MODSEQ_EXT_NAME,
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen                    sizeof(struct mail_index_modseq_header),
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen                    sizeof(uint64_t), sizeof(uint64_t));
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen}
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainenstatic uint64_t mail_index_modseq_get_head(struct mail_index *index)
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen{
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    return index->log->head == NULL ? 1 :
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen        I_MAX(index->log->head->sync_highest_modseq, 1);
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen}
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainenvoid mail_index_modseq_enable(struct mail_index *index)
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen{
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    struct mail_index_transaction *trans;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    struct mail_index_view *view;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    struct mail_index_modseq_header hdr;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    uint32_t ext_map_idx;
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen    if (index->modseqs_enabled)
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        return;
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen    if (!mail_index_map_get_ext_idx(index->map, index->modseq_ext_id,
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen                    &ext_map_idx)) {
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        /* modseqs not enabled to the index yet, add them. */
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        view = mail_index_view_open(index);
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        trans = mail_index_transaction_begin(view, 0);
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        memset(&hdr, 0, sizeof(hdr));
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        hdr.highest_modseq = mail_index_modseq_get_head(index);
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        mail_index_update_header_ext(trans, index->modseq_ext_id,
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen                         0, &hdr, sizeof(hdr));
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        /* commit also refreshes the index, which syncs the modseqs */
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        (void)mail_index_transaction_commit(&trans);
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        mail_index_view_close(&view);
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        /* get the modseq extension to index map */
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        if (!mail_index_map_get_ext_idx(index->map,
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen                        index->modseq_ext_id,
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen                        &ext_map_idx)) {
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen            /* didn't work for some reason */
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen            return;
d1414c09cf0d58ac983054e2f4e1a1f329272dcfTimo Sirainen        }
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    }
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    index->modseqs_enabled = TRUE;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen}
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenconst struct mail_index_modseq_header *
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainenmail_index_map_get_modseq_header(struct mail_index_map *map)
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen{
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen    const struct mail_index_ext *ext;
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen    uint32_t idx;
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen    if (!mail_index_map_get_ext_idx(map, map->index->modseq_ext_id, &idx))
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen        return NULL;
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen    ext = array_idx(&map->extensions, idx);
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen    if (ext->hdr_size != sizeof(struct mail_index_modseq_header))
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen        return NULL;
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen    return CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
8eea67470c1bd8562a62e7445d930bb2079b1a43Timo Sirainen}
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainenuint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map)
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen{
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    const struct mail_index_modseq_header *modseq_hdr;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    modseq_hdr = mail_index_map_get_modseq_header(map);
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    if (modseq_hdr != NULL && modseq_hdr->highest_modseq != 0)
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen        return modseq_hdr->highest_modseq;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    else {
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen        /* fallback to returning the log head. if modseqs aren't
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen           enabled, we return 0. */
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen        return map->index->log->head == NULL ? 0 :
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen            map->index->log->head->sync_highest_modseq;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    }
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen}
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainenuint64_t mail_index_modseq_get_highest(struct mail_index_view *view)
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen{
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    return mail_index_map_modseq_get_highest(view->map);
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen}
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainenstatic struct mail_index_map_modseq *
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainenmail_index_map_modseq(struct mail_index_view *view)
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen{
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    struct mail_index_map_modseq *mmap = view->map->rec_map->modseq;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    uint32_t ext_map_idx;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    if (mmap != NULL)
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen        return mmap;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    /* don't start tracking until we've seen modseq extension intro */
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen                    &ext_map_idx))
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen        return NULL;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    mmap = i_new(struct mail_index_map_modseq, 1);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    i_array_init(&mmap->metadata_modseqs,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen             METADATA_MODSEQ_IDX_KEYWORD_START +
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen             array_count(&view->index->keywords));
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    view->map->rec_map->modseq = mmap;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    return mmap;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen}
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainenuint64_t mail_index_modseq_lookup(struct mail_index_view *view, uint32_t seq)
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen{
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    struct mail_index_map *map;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    const struct mail_index_ext *ext;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    const struct mail_index_record *rec;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    const uint64_t *modseqp;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    uint32_t ext_map_idx;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    if (mmap == NULL)
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen        return mail_index_modseq_get_head(view->index);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    rec = mail_index_lookup_full(view, seq, &map);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen                    &ext_map_idx)) {
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen        /* not enabled yet */
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen        return mail_index_modseq_get_head(view->index);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    }
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    ext = array_idx(&map->extensions, ext_map_idx);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    modseqp = CONST_PTR_OFFSET(rec, ext->record_offset);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    if (*modseqp == 0) {
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen        /* If we're here because we just enabled modseqs, we'll return
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen           the same modseq (initial highestmodseq) for all messages.
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen           The next sync will change these zeros to initial
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen           highestmodseq or higher.
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen           If we're here because a message got appended but modseq
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen           wasn't set (older Dovecot?), we'll again use the current
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen           highest modseq. This isn't exactly correct, but it gets
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen           fixed after the next sync and this situation shouldn't
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen           normally happen anyway. */
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen        return mail_index_modseq_get_highest(view);
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    }
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen    return *modseqp;
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen}
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainenint mail_index_modseq_set(struct mail_index_view *view,
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen              uint32_t seq, uint64_t min_modseq)
acf3b7bf3a8891b118a71c45e6c48d17bc90b259Timo Sirainen{
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    const struct mail_index_ext *ext;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    struct mail_index_record *rec;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    uint64_t *modseqp;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    uint32_t ext_map_idx;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen    if (mmap == NULL)
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen        return -1;
7a87427770874f38d1d299635b37d699f9772860Timo Sirainen
7a87427770874f38d1d299635b37d699f9772860Timo Sirainen    rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
7a87427770874f38d1d299635b37d699f9772860Timo Sirainen    if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id,
7a87427770874f38d1d299635b37d699f9772860Timo Sirainen                    &ext_map_idx))
7a87427770874f38d1d299635b37d699f9772860Timo Sirainen        return -1;
2a90d8a14b0e7cc1508814bc87d3dfa598ef46a8Timo Sirainen
    ext = array_idx(&view->map->extensions, ext_map_idx);
    modseqp = PTR_OFFSET(rec, ext->record_offset);
    if (*modseqp > min_modseq)
        return 0;
    else {
        *modseqp = min_modseq;
        return 1;
    }
}

static uint64_t
modseq_idx_lookup(struct mail_index_map_modseq *mmap,
          unsigned int idx, uint32_t seq)
{
    const struct metadata_modseqs *metadata;
    const uint64_t *modseqs;
    unsigned int count;

    metadata = array_get(&mmap->metadata_modseqs, &count);
    if (idx >= count || !array_is_created(&metadata[idx].modseqs))
        return 0;

    modseqs = array_get(&metadata[idx].modseqs, &count);
    return seq > count ? 0 : modseqs[seq-1];
}

uint64_t mail_index_modseq_lookup_flags(struct mail_index_view *view,
                    enum mail_flags flags_mask,
                    uint32_t seq)
{
    struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
    unsigned int i;
    uint64_t modseq, highest_modseq = 0;

    if (mmap != NULL) {
        /* first try to find a specific match */
        for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) {
            if ((flags_mask & (1 << i)) != 0) {
                modseq = modseq_idx_lookup(mmap, i, seq);
                if (highest_modseq < modseq)
                    highest_modseq = modseq;
            }
        }
    }

    if (highest_modseq == 0) {
        /* no specific matches, fallback to using the highest */
        highest_modseq = mail_index_modseq_lookup(view, seq);
    }
    return highest_modseq;
}

uint64_t mail_index_modseq_lookup_keywords(struct mail_index_view *view,
                       const struct mail_keywords *keywords,
                       uint32_t seq)
{
    struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
    unsigned int i, metadata_idx;
    uint64_t modseq, highest_modseq = 0;

    if (mmap != NULL) {
        /* first try to find a specific match */
        for (i = 0; i < keywords->count; i++) {
            metadata_idx = METADATA_MODSEQ_IDX_KEYWORD_START +
                keywords->idx[i];

            modseq = modseq_idx_lookup(mmap, metadata_idx, seq);
            if (highest_modseq < modseq)
                highest_modseq = modseq;
        }
    }

    if (highest_modseq == 0) {
        /* no specific matches, fallback to using the highest */
        highest_modseq = mail_index_modseq_lookup(view, seq);
    }
    return highest_modseq;
}

static void
mail_index_modseq_update(struct mail_index_modseq_sync *ctx,
             uint64_t modseq, bool nonzeros,
             uint32_t seq1, uint32_t seq2)
{
    const struct mail_index_ext *ext;
    struct mail_index_record *rec;
    uint32_t ext_map_idx;
    uint64_t *modseqp;

    if (!mail_index_map_get_ext_idx(ctx->view->map,
                    ctx->view->index->modseq_ext_id,
                    &ext_map_idx))
        return;

    if (modseq > ctx->highest_modseq)
        ctx->highest_modseq = modseq;

    ext = array_idx(&ctx->view->map->extensions, ext_map_idx);
    for (; seq1 <= seq2; seq1++) {
        rec = MAIL_INDEX_MAP_IDX(ctx->view->map, seq1-1);
        modseqp = PTR_OFFSET(rec, ext->record_offset);
        if (*modseqp == 0 || (nonzeros && *modseqp < modseq))
            *modseqp = modseq;
    }
}

static bool
mail_index_modseq_update_to_highest(struct mail_index_modseq_sync *ctx,
                    uint32_t seq1, uint32_t seq2)
{
    uint64_t modseq;

    if (ctx->mmap == NULL)
        return FALSE;

    modseq = mail_transaction_log_view_get_prev_modseq(ctx->log_view);
    mail_index_modseq_update(ctx, modseq, TRUE, seq1, seq2);
    return TRUE;
}

static void
mail_index_modseq_update_old_rec(struct mail_index_modseq_sync *ctx,
                 const struct mail_transaction_header *thdr,
                 const void *tdata)
{
    ARRAY_TYPE(seq_range) uids = ARRAY_INIT;
    const struct seq_range *rec;
    buffer_t uid_buf;
    unsigned int i, count;
    uint32_t seq1, seq2;

    switch (thdr->type & MAIL_TRANSACTION_TYPE_MASK) {
    case MAIL_TRANSACTION_APPEND: {
        const struct mail_index_record *appends = tdata;

        count = thdr->size / sizeof(*appends);
        for (i = 0; i < count; i++) {
            if (mail_index_lookup_seq(ctx->view,
                          appends[i].uid, &seq1)) {
                mail_index_modseq_update_to_highest(ctx, seq1,
                                    seq1);
            }
        }
        return;
    }
    case MAIL_TRANSACTION_FLAG_UPDATE: {
        buffer_create_const_data(&uid_buf, tdata, thdr->size);
        array_create_from_buffer(&uids, &uid_buf,
            sizeof(struct mail_transaction_flag_update));
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_UPDATE: {
        const struct mail_transaction_keyword_update *rec = tdata;
        unsigned int seqset_offset;

        seqset_offset = sizeof(*rec) + rec->name_size;
        if ((seqset_offset % 4) != 0)
            seqset_offset += 4 - (seqset_offset % 4);

        buffer_create_const_data(&uid_buf,
                     CONST_PTR_OFFSET(tdata, seqset_offset),
                     thdr->size - seqset_offset);
        array_create_from_buffer(&uids, &uid_buf, sizeof(uint32_t)*2);
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_RESET:
        buffer_create_const_data(&uid_buf, tdata, thdr->size);
        array_create_from_buffer(&uids, &uid_buf,
            sizeof(struct mail_transaction_keyword_reset));
        break;
    default:
        return;
    }

    /* update modseqs */
    count = array_count(&uids);
    for (i = 0; i < count; i++) {
        rec = array_idx(&uids, i);
        if (mail_index_lookup_seq_range(ctx->view, rec->seq1, rec->seq2,
                        &seq1, &seq2))
            mail_index_modseq_update_to_highest(ctx, seq1, seq2);
    }
}

static void mail_index_modseq_sync_init(struct mail_index_modseq_sync *ctx)
{
    struct mail_index_map *map = ctx->view->map;
    const struct mail_index_ext *ext;
    const struct mail_index_modseq_header *hdr;
    const struct mail_transaction_header *thdr;
    const void *tdata;
    uint32_t ext_map_idx;
    uint32_t end_seq;
    uoff_t end_offset;
    uint64_t cur_modseq;
    bool reset;
    int ret;

    if (!mail_index_map_get_ext_idx(map, ctx->view->index->modseq_ext_id,
                    &ext_map_idx))
        i_unreached();
    ext = array_idx(&map->extensions, ext_map_idx);

    /* get the current highest_modseq. don't change any modseq below it. */
    hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
    ctx->highest_modseq = hdr->highest_modseq;

    /* Scan logs for updates between ext_hdr.log_* .. view position.
       There are two reasons why there could be any:

       1) We just enabled modseqs and we're filling the initial values.
       2) A non-modseq-aware Dovecot version added new messages and wrote
          dovecot.index file. */
    mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
                           &end_seq, &end_offset);
    if (end_seq <= hdr->log_seq ||
        (end_seq == hdr->log_seq && end_offset <= hdr->log_offset)) {
        /* modseqs are up to date */
        return;
    }

    ctx->log_view = mail_transaction_log_view_open(ctx->view->index->log);
    ret = mail_transaction_log_view_set(ctx->log_view,
                        I_MAX(1, hdr->log_seq),
                        hdr->log_offset,
                        end_seq, end_offset, &reset);
    if (ret <= 0) {
        /* missing files / error - try with only the last file */
        ret = mail_transaction_log_view_set(ctx->log_view, end_seq, 0,
                            end_seq, end_offset,
                            &reset);
        /* since we don't know if we skipped some changes, set all
           modseqs to beginning of the latest file. */
        cur_modseq = mail_transaction_log_view_get_prev_modseq(
                                ctx->log_view);
        if (cur_modseq < hdr->highest_modseq) {
            /* should happen only when setting initial modseqs.
               we may already have returned highest_modseq as
               some messages' modseq value. don't shrink it. */
            cur_modseq = hdr->highest_modseq;
        }
        mail_index_modseq_update(ctx, cur_modseq, TRUE, 1,
                     map->hdr.messages_count);
    } else {
        /* we have all the logs. replace zero modseqs with the current
           highest modseq (we may have already returned it for them). */
        mail_index_modseq_update(ctx, hdr->highest_modseq, FALSE, 1,
                     map->hdr.messages_count);
    }
    if (ret > 0) {
        while (mail_transaction_log_view_next(ctx->log_view,
                              &thdr, &tdata) > 0) {
            T_BEGIN {
                mail_index_modseq_update_old_rec(ctx, thdr,
                                 tdata);
            } T_END;
        }
    }
    mail_index_sync_write_seq_update(ctx->sync_map_ctx, 1,
                     map->hdr.messages_count);
    mail_transaction_log_view_close(&ctx->log_view);
}

struct mail_index_modseq_sync *
mail_index_modseq_sync_begin(struct mail_index_sync_map_ctx *sync_map_ctx)
{
    struct mail_index_modseq_sync *ctx;

    ctx = i_new(struct mail_index_modseq_sync, 1);
    ctx->sync_map_ctx = sync_map_ctx;
    ctx->view = sync_map_ctx->view;
    ctx->mmap = mail_index_map_modseq(ctx->view);
    if (ctx->mmap != NULL) {
        mail_index_modseq_sync_init(ctx);
        ctx->log_view = ctx->view->log_view;
    }
    return ctx;
}

static void mail_index_modseq_update_header(struct mail_index_view *view,
                        uint64_t highest_modseq)
{
    struct mail_index_map *map = view->map;
    const struct mail_index_ext *ext;
    const struct mail_index_modseq_header *old_modseq_hdr;
    struct mail_index_modseq_header new_modseq_hdr;
    uint32_t ext_map_idx, log_seq;
    uoff_t log_offset;

    if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id,
                    &ext_map_idx))
        return;

    mail_transaction_log_view_get_prev_pos(view->log_view,
                           &log_seq, &log_offset);

    ext = array_idx(&map->extensions, ext_map_idx);
    old_modseq_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);

    if (old_modseq_hdr->log_seq < log_seq ||
        (old_modseq_hdr->log_seq == log_seq &&
         old_modseq_hdr->log_offset < log_offset)) {
        memset(&new_modseq_hdr, 0, sizeof(new_modseq_hdr));
        new_modseq_hdr.highest_modseq = highest_modseq;
        new_modseq_hdr.log_seq = log_seq;
        new_modseq_hdr.log_offset = log_offset;

        buffer_write(map->hdr_copy_buf, ext->hdr_offset,
                 &new_modseq_hdr, sizeof(new_modseq_hdr));
        map->hdr_base = map->hdr_copy_buf->data;
        map->write_ext_header = TRUE;
    }
}

void mail_index_modseq_sync_end(struct mail_index_modseq_sync **_ctx)
{
    struct mail_index_modseq_sync *ctx = *_ctx;

    *_ctx = NULL;
    if (ctx->mmap != NULL) {
        i_assert(ctx->mmap == ctx->view->map->rec_map->modseq);
        mail_index_modseq_update_header(ctx->view, ctx->highest_modseq);
    }
    i_free(ctx);
}

void mail_index_modseq_sync_map_replaced(struct mail_index_modseq_sync *ctx)
{
    ctx->mmap = mail_index_map_modseq(ctx->view);
}

void mail_index_modseq_hdr_update(struct mail_index_modseq_sync *ctx)
{
    if (ctx->mmap == NULL) {
        ctx->mmap = mail_index_map_modseq(ctx->view);
        i_assert(ctx->mmap != NULL);
        mail_index_modseq_sync_init(ctx);
        ctx->log_view = ctx->view->log_view;
    }
}

void mail_index_modseq_append(struct mail_index_modseq_sync *ctx, uint32_t seq)
{
    mail_index_modseq_update_to_highest(ctx, seq, seq);
}

void mail_index_modseq_expunge(struct mail_index_modseq_sync *ctx,
                   uint32_t seq1, uint32_t seq2)
{
    struct metadata_modseqs *metadata;
    uint64_t modseq;

    if (ctx->mmap == NULL)
        return;

    seq1--;
    array_foreach_modifiable(&ctx->mmap->metadata_modseqs, metadata) {
        if (array_is_created(&metadata->modseqs))
            array_delete(&metadata->modseqs, seq1, seq2-seq1);
    }

    modseq = mail_transaction_log_view_get_prev_modseq(ctx->log_view);
    if (ctx->highest_modseq < modseq)
        ctx->highest_modseq = modseq;
}

static void
modseqs_update(ARRAY_TYPE(modseqs) *array, uint32_t seq1, uint32_t seq2,
           uint64_t value)
{
    for (; seq1 <= seq2; seq1++)
        array_idx_set(array, seq1-1, &value);
}

static void
modseqs_idx_update(struct mail_index_modseq_sync *ctx, unsigned int idx,
           uint32_t seq1, uint32_t seq2)
{
    struct metadata_modseqs *metadata;

    if (!ctx->view->index->modseqs_enabled) {
        /* we want to keep permanent modseqs updated, but don't bother
           updating in-memory per-flag updates */
        return;
    }

    metadata = array_idx_modifiable(&ctx->mmap->metadata_modseqs, idx);
    if (!array_is_created(&metadata->modseqs))
        i_array_init(&metadata->modseqs, seq2 + 16);
    modseqs_update(&metadata->modseqs, seq1, seq2, ctx->highest_modseq);
}

void mail_index_modseq_update_flags(struct mail_index_modseq_sync *ctx,
                    enum mail_flags flags_mask,
                    uint32_t seq1, uint32_t seq2)
{
    unsigned int i;

    if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
        return;

    for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) {
        if ((flags_mask & (1 << i)) != 0)
            modseqs_idx_update(ctx, i, seq1, seq2);
    }
}

void mail_index_modseq_update_keyword(struct mail_index_modseq_sync *ctx,
                      unsigned int keyword_idx,
                      uint32_t seq1, uint32_t seq2)
{
    if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
        return;

    modseqs_idx_update(ctx, METADATA_MODSEQ_IDX_KEYWORD_START + keyword_idx,
               seq1, seq2);
}

void mail_index_modseq_reset_keywords(struct mail_index_modseq_sync *ctx,
                      uint32_t seq1, uint32_t seq2)
{
    unsigned int i, count;

    if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
        return;

    count = array_count(&ctx->mmap->metadata_modseqs);
    for (i = METADATA_MODSEQ_IDX_KEYWORD_START; i < count; i++)
        modseqs_idx_update(ctx, i, seq1, seq2);
}

void mail_index_modseq_update_highest(struct mail_index_modseq_sync *ctx,
                      uint64_t highest_modseq)
{
    if (ctx->highest_modseq < highest_modseq)
        ctx->highest_modseq = highest_modseq;
}

struct mail_index_map_modseq *
mail_index_map_modseq_clone(const struct mail_index_map_modseq *mmap)
{
    struct mail_index_map_modseq *new_mmap;
    const struct metadata_modseqs *src_metadata;
    struct metadata_modseqs *dest_metadata;
    unsigned int i, count;

    src_metadata = array_get(&mmap->metadata_modseqs, &count);

    new_mmap = i_new(struct mail_index_map_modseq, 1);
    i_array_init(&new_mmap->metadata_modseqs, count + 16);

    for (i = 0; i < count; i++) {
        dest_metadata = array_append_space(&new_mmap->metadata_modseqs);
        if (array_is_created(&src_metadata[i].modseqs)) {
            i_array_init(&dest_metadata->modseqs,
                     array_count(&src_metadata[i].modseqs));
            array_append_array(&dest_metadata->modseqs,
                       &src_metadata[i].modseqs);
        }
    }
    return new_mmap;
}

void mail_index_map_modseq_free(struct mail_index_map_modseq **_mmap)
{
    struct mail_index_map_modseq *mmap = *_mmap;
    struct metadata_modseqs *metadata;

    *_mmap = NULL;

    array_foreach_modifiable(&mmap->metadata_modseqs, metadata) {
        if (array_is_created(&metadata->modseqs))
            array_free(&metadata->modseqs);
    }
    array_free(&mmap->metadata_modseqs);
    i_free(mmap);
}

bool mail_index_modseq_get_next_log_offset(struct mail_index_view *view,
                       uint64_t modseq, uint32_t *log_seq_r,
                       uoff_t *log_offset_r)
{
    struct mail_transaction_log_file *file, *prev_file = NULL;

    for (file = view->index->log->files; file != NULL; file = file->next) {
        if (modseq < file->hdr.initial_modseq)
            break;
        prev_file = file;
    }

    if (prev_file == NULL) {
        /* the log file has been deleted already */
        return FALSE;
    }

    *log_seq_r = prev_file->hdr.file_seq;
    return mail_transaction_log_file_get_modseq_next_offset(
                    prev_file, modseq, log_offset_r) == 0;
}