src/lib-index/mail-index-sync-update.c

	mail-index-sync-update.c revision 215daa7d5bf9a849849586490fd6fdd28a724dd8
8e47d6ce64ae5a3392b4d92867b2a83bd0dc9bbeEugen Kuksa/* Copyright (C) 2004 Timo Sirainen */
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase#include "lib.h"
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase#include "buffer.h"
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase#include "file-set-size.h"
a847d9812b328c048773e705606b10875a929034Eugen Kuksa#include "mmap-util.h"
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase#include "mail-index-view-private.h"
a847d9812b328c048773e705606b10875a929034Eugen Kuksa#include "mail-index-sync-private.h"
a847d9812b328c048773e705606b10875a929034Eugen Kuksa#include "mail-transaction-log.h"
a847d9812b328c048773e705606b10875a929034Eugen Kuksa
a847d9812b328c048773e705606b10875a929034Eugen Kuksastruct mail_index_update_ctx {
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    struct mail_index *index;
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    struct mail_index_header hdr;
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    struct mail_transaction_log_view *log_view;
a847d9812b328c048773e705606b10875a929034Eugen Kuksa};
a847d9812b328c048773e705606b10875a929034Eugen Kuksa
a847d9812b328c048773e705606b10875a929034Eugen Kuksavoid mail_index_header_update_counts(struct mail_index_header *hdr,
8e47d6ce64ae5a3392b4d92867b2a83bd0dc9bbeEugen Kuksa                     uint8_t old_flags, uint8_t new_flags)
a847d9812b328c048773e705606b10875a929034Eugen Kuksa{
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        /* different seen-flag */
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        if ((old_flags & MAIL_SEEN) == 0)
a847d9812b328c048773e705606b10875a929034Eugen Kuksa            hdr->seen_messages_count++;
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        else
a847d9812b328c048773e705606b10875a929034Eugen Kuksa            hdr->seen_messages_count--;
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    }
a847d9812b328c048773e705606b10875a929034Eugen Kuksa
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        /* different deleted-flag */
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        if ((old_flags & MAIL_DELETED) == 0)
a847d9812b328c048773e705606b10875a929034Eugen Kuksa            hdr->deleted_messages_count++;
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        else
a847d9812b328c048773e705606b10875a929034Eugen Kuksa            hdr->deleted_messages_count--;
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    }
a847d9812b328c048773e705606b10875a929034Eugen Kuksa}
a847d9812b328c048773e705606b10875a929034Eugen Kuksa
a847d9812b328c048773e705606b10875a929034Eugen Kuksavoid mail_index_header_update_lowwaters(struct mail_index_header *hdr,
a847d9812b328c048773e705606b10875a929034Eugen Kuksa                    const struct mail_index_record *rec)
a847d9812b328c048773e705606b10875a929034Eugen Kuksa{
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    if ((rec->flags & MAIL_RECENT) != 0 &&
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        rec->uid < hdr->first_recent_uid_lowwater)
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        hdr->first_recent_uid_lowwater = rec->uid;
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    if ((rec->flags & MAIL_SEEN) == 0 &&
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        rec->uid < hdr->first_unseen_uid_lowwater)
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase        hdr->first_unseen_uid_lowwater = rec->uid;
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase    if ((rec->flags & MAIL_DELETED) != 0 &&
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase        rec->uid < hdr->first_deleted_uid_lowwater)
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase        hdr->first_deleted_uid_lowwater = rec->uid;
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase}
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehasestatic void mail_index_sync_update_expunges(struct mail_index_update_ctx *ctx,
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase                        uint32_t seq1, uint32_t seq2)
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase{
c4cb6bccad9d72c8b284075f716fc9095b3f3be2Tim Reddehase    struct mail_index_record *rec;
c4cb6bccad9d72c8b284075f716fc9095b3f3be2Tim Reddehase
c4cb6bccad9d72c8b284075f716fc9095b3f3be2Tim Reddehase    rec = &ctx->index->map->records[seq1-1];
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase    for (; seq1 <= seq2; seq1++, rec++)
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase        mail_index_header_update_counts(&ctx->hdr, rec->flags, 0);
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase}
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehasestatic void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx,
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase                     struct mail_index_sync_rec *syncrec)
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase{
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase    struct mail_index_record *rec, *end;
d51cee8257d50d1f2d7c0818d968b32096f6b795Tim Reddehase    uint8_t flag_mask, old_flags;
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase    keywords_mask_t keyword_mask;
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase    int i, update_keywords;
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase    update_keywords = FALSE;
0abeb43875687e94f2d551053ad09eebeff1f7c9Tim Reddehase    for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        if (syncrec->add_keywords[i] != 0)
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase            update_keywords = TRUE;
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase        if (syncrec->remove_keywords[i] != 0)
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase            update_keywords = TRUE;
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase        keyword_mask[i] = ~syncrec->remove_keywords[i];
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase    }
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase    flag_mask = ~syncrec->remove_flags;
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase    rec = &ctx->index->map->records[syncrec->seq1-1];
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase    end = rec + (syncrec->seq2 - syncrec->seq1) + 1;
81a6387a4ab56a24194ecbabd6609c6bcca568b7Tim Reddehase    for (; rec != end; rec++) {
8e47d6ce64ae5a3392b4d92867b2a83bd0dc9bbeEugen Kuksa        old_flags = rec->flags;
511ba3e3d7deb4a13a03ce522a99a0395acc61b7Tim Reddehase        rec->flags = (rec->flags & flag_mask) | syncrec->add_flags;
419f986abb76f6fce54b71e17f52a1deaa06dbd6Tim Reddehase        if (update_keywords) {
2794bc43ef90861870dcc134505d87b291dfe5c2Tim Reddehase            for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
a847d9812b328c048773e705606b10875a929034Eugen Kuksa                rec->keywords[i] =
a847d9812b328c048773e705606b10875a929034Eugen Kuksa                    (rec->keywords[i] & keyword_mask[i]) |
a847d9812b328c048773e705606b10875a929034Eugen Kuksa                    syncrec->add_keywords[i];
a847d9812b328c048773e705606b10875a929034Eugen Kuksa            }
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        }
a847d9812b328c048773e705606b10875a929034Eugen Kuksa
a847d9812b328c048773e705606b10875a929034Eugen Kuksa        mail_index_header_update_counts(&ctx->hdr,
a847d9812b328c048773e705606b10875a929034Eugen Kuksa                        old_flags, rec->flags);
a847d9812b328c048773e705606b10875a929034Eugen Kuksa                mail_index_header_update_lowwaters(&ctx->hdr, rec);
a847d9812b328c048773e705606b10875a929034Eugen Kuksa    }
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase}
419f986abb76f6fce54b71e17f52a1deaa06dbd6Tim Reddehase
419f986abb76f6fce54b71e17f52a1deaa06dbd6Tim Reddehasestatic int mail_index_grow(struct mail_index *index, unsigned int count)
419f986abb76f6fce54b71e17f52a1deaa06dbd6Tim Reddehase{
2794bc43ef90861870dcc134505d87b291dfe5c2Tim Reddehase    struct mail_index_map *map = index->map;
419f986abb76f6fce54b71e17f52a1deaa06dbd6Tim Reddehase    unsigned int records_count;
419f986abb76f6fce54b71e17f52a1deaa06dbd6Tim Reddehase    size_t size;
419f986abb76f6fce54b71e17f52a1deaa06dbd6Tim Reddehase
419f986abb76f6fce54b71e17f52a1deaa06dbd6Tim Reddehase    if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        (void)buffer_append_space_unsafe(map->buffer,
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase            count * sizeof(struct mail_index_record));
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        map->records = buffer_get_modifyable_data(map->buffer, NULL);
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        return 0;
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase    }
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase    size = map->hdr->header_size +
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        (map->records_count + count) * sizeof(struct mail_index_record);
80021fb6bde14c39046950cdbaaadf767d173f94Tim Reddehase    if (size <= map->mmap_size)
80021fb6bde14c39046950cdbaaadf767d173f94Tim Reddehase        return 0;
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase
80021fb6bde14c39046950cdbaaadf767d173f94Tim Reddehase    /* when we grow fast, do it exponentially */
80021fb6bde14c39046950cdbaaadf767d173f94Tim Reddehase    if (count < index->last_grow_count)
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        count = index->last_grow_count;
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase    if (count < MAIL_INDEX_MAX_POWER_GROW)
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        count = nearest_power(count);
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase    index->last_grow_count = count;
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase    size = map->hdr->header_size +
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        (map->records_count + count) * sizeof(struct mail_index_record);
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase    if (file_set_size(index->fd, (off_t)size) < 0)
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        return mail_index_set_syscall_error(index, "file_set_size()");
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase    records_count = map->records_count;
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase    if (mail_index_map(index, TRUE) <= 0)
d9a3935fa80ce492e782d17ec22825d1708dea97Tim Reddehase        return -1;

    i_assert(map->mmap_size >= size);
    map->records_count = records_count;
    return 0;
}

static int mail_index_sync_appends(struct mail_index_update_ctx *ctx,
                   const struct mail_index_record *appends,
                   unsigned int count)
{
    struct mail_index_map *map = ctx->index->map;
    unsigned int i;
    uint32_t next_uid;

    if (mail_index_grow(ctx->index, count) < 0)
        return -1;

    next_uid = ctx->hdr.next_uid;
    for (i = 0; i < count; i++) {
        mail_index_header_update_counts(&ctx->hdr, 0, appends[i].flags);
                mail_index_header_update_lowwaters(&ctx->hdr, &appends[i]);

        if (appends[i].uid < next_uid) {
            mail_transaction_log_view_set_corrupted(ctx->log_view,
                "Append with UID %u, but next_uid = %u",
                appends[i].uid, next_uid);
            return -1;
        }
        next_uid = appends[i].uid+1;
    }
    ctx->hdr.next_uid = next_uid;

    memcpy(map->records + map->records_count, appends,
           count * sizeof(*appends));
    map->records_count += count;
    return 0;
}

int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
                 uint32_t sync_stamp, uint64_t sync_size)
{
    struct mail_index *index = sync_ctx->index;
    struct mail_index_map *map;
        struct mail_index_update_ctx ctx;
    struct mail_index_sync_rec rec;
    const struct mail_index_record *appends;
    unsigned int append_count;
    uint32_t count, file_seq, src_idx, dest_idx, dirty_flag;
    uoff_t file_offset;
    unsigned int lock_id;
    int ret, changed;

    /* rewind */
    sync_ctx->update_idx = sync_ctx->expunge_idx = 0;
    sync_ctx->sync_appends =
        buffer_get_used_size(sync_ctx->appends_buf) != 0;

    changed = mail_index_sync_have_more(sync_ctx);

    memset(&ctx, 0, sizeof(ctx));
    ctx.index = index;
    ctx.hdr = *index->hdr;
    ctx.log_view = sync_ctx->view->log_view;

    dirty_flag = sync_ctx->have_dirty ? MAIL_INDEX_HDR_FLAG_HAVE_DIRTY : 0;
    if ((ctx.hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != dirty_flag) {
        ctx.hdr.flags ^= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
        changed = TRUE;
    }

    /* see if we need to update sync headers */
    if (ctx.hdr.sync_stamp != sync_stamp && sync_stamp != 0) {
        ctx.hdr.sync_stamp = sync_stamp;
        changed = TRUE;
    }
    if (ctx.hdr.sync_size != sync_size && sync_size != 0) {
        ctx.hdr.sync_size = sync_size;
        changed = TRUE;
    }

    if (!changed) {
        /* nothing to sync */
        return 0;
    }

    if (mail_index_lock_exclusive(index, &lock_id) < 0)
        return -1;

    map = index->map;
    if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
        map->write_to_disk = TRUE;

    src_idx = dest_idx = 0;
    append_count = 0; appends = NULL;
    while (mail_index_sync_next(sync_ctx, &rec) > 0) {
        switch (rec.type) {
        case MAIL_INDEX_SYNC_TYPE_APPEND:
            i_assert(appends == NULL);
            append_count = rec.seq2 - rec.seq1 + 1;
            appends = rec.appends;
            break;
        case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
            if (src_idx == 0) {
                /* expunges have to be atomic. so we'll have
                   to copy the mapping, do the changes there
                   and then finally replace the whole index
                   file. to avoid extra disk I/O we copy the
                   index into memory rather than to temporary
                   file */
                map = mail_index_map_to_memory(map);
                mail_index_unmap(index, index->map);
                index->map = map;
                index->hdr = map->hdr;
                map->write_to_disk = TRUE;

                dest_idx = rec.seq1-1;
            } else {
                count = (rec.seq1-1) - src_idx;
                memmove(map->records + dest_idx,
                    map->records + src_idx,
                    count * sizeof(*map->records));
                dest_idx += count;
            }

            mail_index_sync_update_expunges(&ctx, rec.seq1,
                            rec.seq2);
            src_idx = rec.seq2;
            break;
        case MAIL_INDEX_SYNC_TYPE_FLAGS:
            mail_index_sync_update_flags(&ctx, &rec);
            break;
        }
    }

    if (src_idx != 0) {
        count = map->records_count - src_idx;
        memmove(map->records + dest_idx,
            map->records + src_idx,
            count * sizeof(*map->records));
        dest_idx += count;

        map->records_count = dest_idx;
    }

    ret = 0;
    if (append_count > 0)
        ret = mail_index_sync_appends(&ctx, appends, append_count);

    mail_transaction_log_get_head(index->log, &file_seq, &file_offset);

    ctx.hdr.messages_count = map->records_count;
    ctx.hdr.log_file_seq = file_seq;
    ctx.hdr.log_file_offset = file_offset;

    if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
        map->mmap_used_size = index->hdr->header_size +
            map->records_count * sizeof(struct mail_index_record);

        memcpy(map->mmap_base, &ctx.hdr, sizeof(ctx.hdr));
        if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
            mail_index_set_syscall_error(index, "msync()");
            ret = -1;
        }
    } else {
        map->hdr_copy = ctx.hdr;
        map->hdr = &map->hdr_copy;
    }

    mail_index_unlock(index, lock_id);
    return ret;
}