src/lib-index/mail-index-sync-keywords.c

	mail-index-sync-keywords.c revision 5f5870385cff47efd2f58e7892f251cf13761528
2454dfa32c93c20a8522c6ed42fe057baaac9f9aStephan Bosch/* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi#include "lib.h"
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi#include "array.h"
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi#include "buffer.h"
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi#include "mail-index-modseq.h"
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi#include "mail-index-view-private.h"
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi#include "mail-index-sync-private.h"
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi#include "mail-transaction-log.h"
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomistatic bool
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomikeyword_lookup(struct mail_index_sync_map_ctx *ctx,
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi           const char *keyword_name, unsigned int *idx_r)
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi{
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    struct mail_index_map *map = ctx->view->map;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    const unsigned int *idx_map;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    unsigned int i, count, keyword_idx;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    if (array_is_created(&map->keyword_idx_map) &&
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi        mail_index_keyword_lookup(ctx->view->index, keyword_name,
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi                      &keyword_idx)) {
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi        /* FIXME: slow. maybe create index -> file mapping as well */
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi        idx_map = array_get(&map->keyword_idx_map, &count);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi        for (i = 0; i < count; i++) {
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi            if (idx_map[i] == keyword_idx) {
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi                *idx_r = i;
bcab95c672604588f1645e4a52e43f52ee567593Timo Sirainen                return TRUE;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi            }
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi        }
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    }
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    return FALSE;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi}
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomistatic buffer_t *
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomikeywords_get_header_buf(struct mail_index_map *map,
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi            const struct mail_index_ext *ext,
bcab95c672604588f1645e4a52e43f52ee567593Timo Sirainen            unsigned int new_count, unsigned int *keywords_count_r,
bcab95c672604588f1645e4a52e43f52ee567593Timo Sirainen            size_t *rec_offset_r, size_t *name_offset_root_r,
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi            size_t *name_offset_r)
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi{
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    buffer_t *buf;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    const struct mail_index_keyword_header *kw_hdr;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    const struct mail_index_keyword_header_rec *kw_rec;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    const char *name;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    struct mail_index_keyword_header new_kw_hdr;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    uint32_t offset;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    kw_rec = (const void *)(kw_hdr + 1);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    name = (const char *)(kw_rec + kw_hdr->keywords_count);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    if (kw_hdr->keywords_count == 0)
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi        return NULL;
bcab95c672604588f1645e4a52e43f52ee567593Timo Sirainen
bcab95c672604588f1645e4a52e43f52ee567593Timo Sirainen    i_assert((size_t)(name - (const char *)kw_hdr) < ext->hdr_size);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    new_kw_hdr = *kw_hdr;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    new_kw_hdr.keywords_count += new_count;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    *keywords_count_r = new_kw_hdr.keywords_count;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    offset = kw_rec[kw_hdr->keywords_count-1].name_offset;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    offset += strlen(name + offset) + 1;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    buf = buffer_create_dynamic(pool_datastack_create(), 512);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    buffer_append(buf, &new_kw_hdr, sizeof(new_kw_hdr));
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    buffer_append(buf, kw_rec, sizeof(*kw_rec) * kw_hdr->keywords_count);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    *rec_offset_r = buf->used;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    buffer_write(buf, buf->used + sizeof(*kw_rec) * new_count,
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi             name, offset);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    *name_offset_root_r = buf->used;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    *name_offset_r = offset;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    return buf;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi}
2935863763d8655c2c23e52591f2cbfff54811a6Aki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomistatic void keywords_ext_register(struct mail_index_sync_map_ctx *ctx,
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi                  uint32_t ext_map_idx, uint32_t reset_id,
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi                  uint32_t hdr_size, uint32_t keywords_count)
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi{
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    buffer_t ext_intro_buf;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    struct mail_transaction_ext_intro *u;
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    unsigned char ext_intro_data[sizeof(*u) +
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi                     sizeof(MAIL_INDEX_EXT_KEYWORDS)-1];
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    i_assert(keywords_count > 0);
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi    buffer_create_data(&ext_intro_buf, ext_intro_data,
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi               sizeof(ext_intro_data));
662bb64be818407c6719a69780411f7ca8d6c96fAki Tuomi
    u = buffer_append_space_unsafe(&ext_intro_buf, sizeof(*u));
    u->ext_id = ext_map_idx;
    u->reset_id = reset_id;
    u->hdr_size = hdr_size;
    u->record_size = (keywords_count + CHAR_BIT - 1) / CHAR_BIT;
    if ((u->record_size % 4) != 0) {
        /* since we aren't properly aligned anyway,
           reserve one extra byte for future */
        u->record_size++;
    }
    u->record_align = 1;

    if (ext_map_idx == (uint32_t)-1) {
        u->name_size = strlen(MAIL_INDEX_EXT_KEYWORDS);
        buffer_append(&ext_intro_buf, MAIL_INDEX_EXT_KEYWORDS,
                  u->name_size);
    }

    ctx->internal_update = TRUE;
    if (mail_index_sync_ext_intro(ctx, u) < 0)
        i_panic("Keyword extension growing failed");
    ctx->internal_update = FALSE;
}

static void
keywords_header_add(struct mail_index_sync_map_ctx *ctx,
            const char *keyword_name, unsigned int *keyword_idx_r)
{
    struct mail_index_map *map;
        const struct mail_index_ext *ext = NULL;
    struct mail_index_keyword_header *kw_hdr;
    struct mail_index_keyword_header_rec kw_rec;
    uint32_t ext_map_idx;
    buffer_t *buf = NULL;
    size_t keyword_len, rec_offset, name_offset, name_offset_root;
    unsigned int keywords_count;

    /* if we crash in the middle of writing the header, the
       keywords are more or less corrupted. avoid that by
       making sure the header is updated atomically. */
    map = mail_index_sync_get_atomic_map(ctx);

    if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS,
                       &ext_map_idx))
        ext_map_idx = (uint32_t)-1;
    else {
        /* update existing header */
        ext = array_idx(&map->extensions, ext_map_idx);
        buf = keywords_get_header_buf(map, ext, 1, &keywords_count,
                          &rec_offset, &name_offset_root,
                          &name_offset);
    }

    if (buf == NULL) {
        /* create new / replace broken header */
        const unsigned int initial_keywords_count = 1;

        buf = buffer_create_dynamic(pool_datastack_create(), 512);
        kw_hdr = buffer_append_space_unsafe(buf, sizeof(*kw_hdr));
        kw_hdr->keywords_count = initial_keywords_count;

                keywords_count = kw_hdr->keywords_count;
        rec_offset = buf->used;
        name_offset_root = rec_offset +
            initial_keywords_count * sizeof(kw_rec);
        name_offset = 0;
    }

    /* add the keyword */
    memset(&kw_rec, 0, sizeof(kw_rec));
    kw_rec.name_offset = name_offset;

    keyword_len = strlen(keyword_name) + 1;
    buffer_write(buf, rec_offset, &kw_rec, sizeof(kw_rec));
    buffer_write(buf, name_offset_root, keyword_name, keyword_len);

    rec_offset += sizeof(kw_rec);
    kw_rec.name_offset += keyword_len;
    name_offset_root += keyword_len;

    if ((buf->used % 4) != 0)
        buffer_append_zero(buf, 4 - (buf->used % 4));

    if (ext == NULL || buf->used > ext->hdr_size ||
        (uint32_t)ext->record_size * CHAR_BIT < keywords_count) {
        /* if we need to grow the buffer, add some padding */
        buffer_append_zero(buf, 128);
        keywords_ext_register(ctx, ext_map_idx,
                      ext == NULL ? 0 : ext->reset_id,
                      buf->used, keywords_count);

        /* map may have changed */
        map = ctx->view->map;

        if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS,
                           &ext_map_idx))
            i_unreached();
        ext = array_idx(&map->extensions, ext_map_idx);

        i_assert(ext->hdr_size == buf->used);
    }

    buffer_copy(map->hdr_copy_buf, ext->hdr_offset, buf, 0, buf->used);
    map->hdr_base = map->hdr_copy_buf->data;

    if (mail_index_map_parse_keywords(map) < 0)
        i_panic("Keyword update corrupted keywords header");

    *keyword_idx_r = keywords_count - 1;
    i_assert(*keyword_idx_r / CHAR_BIT < ext->record_size);
}

static int
keywords_update_records(struct mail_index_sync_map_ctx *ctx,
            const struct mail_index_ext *ext,
            unsigned int keyword_idx, enum modify_type type,
            uint32_t uid1, uint32_t uid2)
{
    struct mail_index_view *view = ctx->view;
    struct mail_index_record *rec;
    unsigned char *data, data_mask;
    unsigned int data_offset;
    uint32_t seq1, seq2;

    i_assert(keyword_idx != (unsigned int)-1);

    if (!mail_index_lookup_seq_range(view, uid1, uid2, &seq1, &seq2))
        return 1;

    mail_index_sync_write_seq_update(ctx, seq1, seq2);
    mail_index_modseq_update_keyword(ctx->modseq_ctx, keyword_idx,
                      seq1, seq2);

    data_offset = keyword_idx / CHAR_BIT;
    data_mask = 1 << (keyword_idx % CHAR_BIT);

    i_assert(data_offset < ext->record_size);
    data_offset += ext->record_offset;

    i_assert(data_offset >= sizeof(struct mail_index_record));

    switch (type) {
    case MODIFY_ADD:
        for (seq1--; seq1 < seq2; seq1++) {
            rec = MAIL_INDEX_MAP_IDX(view->map, seq1);
            data = PTR_OFFSET(rec, data_offset);
            *data |= data_mask;
        }
        break;
    case MODIFY_REMOVE:
        data_mask = ~data_mask;
        for (seq1--; seq1 < seq2; seq1++) {
            rec = MAIL_INDEX_MAP_IDX(view->map, seq1);
            data = PTR_OFFSET(rec, data_offset);
            *data &= data_mask;
        }
        break;
    default:
        i_unreached();
    }
    return 1;
}

int mail_index_sync_keywords(struct mail_index_sync_map_ctx *ctx,
                 const struct mail_transaction_header *hdr,
                 const struct mail_transaction_keyword_update *rec)
{
    struct mail_index_view *view = ctx->view;
    const char *keyword_name;
    const struct mail_index_ext *ext;
    const uint32_t *uid, *end;
    uint32_t seqset_offset, ext_map_idx;
    unsigned int keyword_idx;
    int ret;

    i_assert(rec->name_size > 0);

    seqset_offset = sizeof(*rec) + rec->name_size;
    if ((seqset_offset % 4) != 0)
        seqset_offset += 4 - (seqset_offset % 4);
    i_assert(seqset_offset < hdr->size);

    uid = CONST_PTR_OFFSET(rec, seqset_offset);
    end = CONST_PTR_OFFSET(rec, hdr->size);

    keyword_name = t_strndup(rec + 1, rec->name_size);
    if (!keyword_lookup(ctx, keyword_name, &keyword_idx))
        keywords_header_add(ctx, keyword_name, &keyword_idx);

    /* if the keyword wasn't found, the "keywords" extension was created.
       if it was found, the record size should already be correct, but
       in case it isn't just fix it ourself. */
    if (!mail_index_map_lookup_ext(view->map, MAIL_INDEX_EXT_KEYWORDS,
                       &ext_map_idx))
        i_unreached();

    ext = array_idx(&view->map->extensions, ext_map_idx);
    if (keyword_idx / CHAR_BIT >= ext->record_size) {
        if (rec->modify_type == MODIFY_REMOVE) {
            /* nothing to do */
            return 1;
        }

        /* grow the record size */
        keywords_ext_register(ctx, ext_map_idx, ext->reset_id,
                      ext->hdr_size,
                      array_count(&view->map->keyword_idx_map));
        if (!mail_index_map_lookup_ext(view->map,
                           MAIL_INDEX_EXT_KEYWORDS,
                           &ext_map_idx))
            i_unreached();
        ext = array_idx(&view->map->extensions, ext_map_idx);
    }

    while (uid+2 <= end) {
        ret = keywords_update_records(ctx, ext, keyword_idx,
                          rec->modify_type,
                          uid[0], uid[1]);
        if (ret <= 0)
            return ret;

        uid += 2;
    }

    return 1;
}

int
mail_index_sync_keywords_reset(struct mail_index_sync_map_ctx *ctx,
                   const struct mail_transaction_header *hdr,
                   const struct mail_transaction_keyword_reset *r)
{
    struct mail_index_map *map = ctx->view->map;
    struct mail_index_record *rec;
    const struct mail_index_ext *ext;
    const struct mail_transaction_keyword_reset *end;
    uint32_t ext_map_idx, seq1, seq2;

    if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS,
                       &ext_map_idx)) {
        /* nothing to do */
        return 1;
    }

    ext = array_idx(&map->extensions, ext_map_idx);
    end = CONST_PTR_OFFSET(r, hdr->size);
    for (; r != end; r++) {
        if (!mail_index_lookup_seq_range(ctx->view, r->uid1, r->uid2,
                         &seq1, &seq2))
            continue;

        mail_index_sync_write_seq_update(ctx, seq1, seq2);
        mail_index_modseq_reset_keywords(ctx->modseq_ctx, seq1, seq2);
        for (seq1--; seq1 < seq2; seq1++) {
            rec = MAIL_INDEX_MAP_IDX(map, seq1);
            memset(PTR_OFFSET(rec, ext->record_offset),
                   0, ext->record_size);
        }
    }
    return 1;
}