src/lib-index/mail-transaction-log-view.c

	mail-transaction-log-view.c revision 1cf72a848805fcf014b01c9d3665b6a157846a21
2454dfa32c93c20a8522c6ed42fe057baaac9f9aStephan Bosch/* Copyright (C) 2003-2004 Timo Sirainen */
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen#include "lib.h"
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen#include "array.h"
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen#include "mail-index-private.h"
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen#include "mail-transaction-log-private.h"
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainenstruct mail_transaction_log_view {
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    struct mail_transaction_log *log;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen        struct mail_transaction_log_view *next;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    uint32_t min_file_seq, max_file_seq;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    uoff_t min_file_offset, max_file_offset;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    struct mail_transaction_header tmp_hdr;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    /* a list of log files we've referenced. we have to keep this list
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen       explicitly because more files may be added into the linked list
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen       at any time. */
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    ARRAY_DEFINE(file_refs, struct mail_transaction_log_file *);
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen        struct mail_transaction_log_file *cur, *head, *tail;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    uoff_t cur_offset;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    uint32_t prev_file_seq;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    uoff_t prev_file_offset;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    unsigned int broken:1;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen};
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainenstruct mail_transaction_log_view *
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainenmail_transaction_log_view_open(struct mail_transaction_log *log)
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen{
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    struct mail_transaction_log_view *view;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    view = i_new(struct mail_transaction_log_view, 1);
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    view->log = log;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    view->broken = TRUE;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    view->head = view->tail = view->log->head;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    view->head->refcount++;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    i_array_init(&view->file_refs, 8);
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    array_append(&view->file_refs, &view->head, 1);
0d1b8b6bec79746c5d89d57dd8c1688946bd9237Josef 'Jeff' Sipek
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    view->next = log->views;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    log->views = view;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    return view;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen}
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainenstatic void
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainenmail_transaction_log_view_unref_all(struct mail_transaction_log_view *view)
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen{
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    struct mail_transaction_log_file *const *files;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    unsigned int i, count;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    files = array_get(&view->file_refs, &count);
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    for (i = 0; i < count; i++)
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen        files[i]->refcount--;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    array_clear(&view->file_refs);
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen}
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainenvoid mail_transaction_log_view_close(struct mail_transaction_log_view **_view)
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen{
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen        struct mail_transaction_log_view *view = *_view;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    struct mail_transaction_log_view **p;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    *_view = NULL;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    for (p = &view->log->views; *p != NULL; p = &(*p)->next) {
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen        if (*p == view) {
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen            *p = view->next;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen            break;
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen        }
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    }
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    mail_transaction_log_view_unref_all(view);
ed1e909434714837f583c9d180c2a16a1c2675c2Timo Sirainen    mail_transaction_logs_clean(view->log);

    array_free(&view->file_refs);
    i_free(view);
}

void mail_transaction_log_views_close(struct mail_transaction_log *log)
{
    struct mail_transaction_log_view *view;

    for (view = log->views; view != NULL; view = view->next)
        view->log = NULL;
}

int
mail_transaction_log_view_set(struct mail_transaction_log_view *view,
                  uint32_t min_file_seq, uoff_t min_file_offset,
                  uint32_t max_file_seq, uoff_t max_file_offset)
{
    struct mail_transaction_log_file *file, *const *files;
    uoff_t start_offset, end_offset;
    unsigned int i;
    uint32_t seq;
    int ret;

    i_assert(view->log != NULL);
    i_assert(min_file_seq <= max_file_seq);

    if (view->log == NULL) {
        /* transaction log is closed already. this log view shouldn't
           be used anymore. */
        return -1;
    }

    if (min_file_seq == 0) {
        /* index file doesn't exist yet. this transaction log should
           start from the beginning */
        if (view->log->files->hdr.prev_file_seq != 0) {
            /* but it doesn't */
            return 0;
        }

        min_file_seq = view->log->files->hdr.file_seq;
        min_file_offset = 0;

        if (max_file_seq == 0) {
            max_file_seq = min_file_seq;
            max_file_offset = min_file_offset;
        }
    }

    if (min_file_seq == view->log->files->hdr.prev_file_seq &&
        min_file_offset == view->log->files->hdr.prev_file_offset) {
        /* we can skip this */
        min_file_seq = view->log->files->hdr.file_seq;
        min_file_offset = 0;

        if (min_file_seq > max_file_seq) {
            /* empty view */
            max_file_seq = min_file_seq;
            max_file_offset = min_file_offset;
        }
    }

    /* find the oldest log file first. */
    ret = mail_transaction_log_find_file(view->log, min_file_seq, &file);
    if (ret <= 0)
        return ret;

    if (min_file_offset == 0) {
        /* beginning of the file */
        min_file_offset = file->hdr.hdr_size;
        if (max_file_offset == 0 && min_file_seq == max_file_seq) {
            /* we don't actually want to show anything */
            max_file_offset = min_file_offset;
        }
    }
    i_assert(min_file_offset >= file->hdr.hdr_size);

    if (min_file_seq == max_file_seq && min_file_offset > max_file_offset) {
        /* log file offset is probably corrupted in the index file. */
        mail_transaction_log_view_set_corrupted(view,
            "file_seq=%u, min_file_offset (%"PRIuUOFF_T
            ") > max_file_offset (%"PRIuUOFF_T")",
            min_file_seq, min_file_offset, max_file_offset);
        return -1;
    }

    view->tail = file;
    view->head = file;

    for (seq = min_file_seq+1; seq <= max_file_seq; seq++) {
        file = file->next;
        if (file == NULL || file->hdr.file_seq != seq) {
            /* see if we could find the missing file */
            ret = mail_transaction_log_find_file(view->log,
                                 seq, &file);
            if (ret <= 0) {
                if (ret < 0)
                    return -1;

                /* not found / corrupted */
                file = NULL;
            }
        }

        if (file == NULL || file->hdr.file_seq != seq) {
            if (file == NULL && max_file_seq == (uint32_t)-1) {
                /* we just wanted to sync everything */
                i_assert(max_file_offset == (uoff_t)-1);
                max_file_seq = seq-1;
                break;
            }

            /* missing files in the middle */
            return 0;
        }
        view->head = file;
    }

    /* we have all of them. update refcounts. */
    mail_transaction_log_view_unref_all(view);

    /* Reference all used files. */
    for (file = view->tail;; file = file->next) {
        array_append(&view->file_refs, &file, 1);
        file->refcount++;

        if (file == view->head)
            break;
    }

    /* Map the files only after we've found them all. Otherwise if we map
       one file and then another file just happens to get rotated, we could
       include both files in the view but skip the last transactions from
       the first file.

       We're mapping the files in reverse order so that _log_file_map()
       can verify that prev_file_offset matches how far it actually managed
       to sync the file. */
    files = array_idx(&view->file_refs, 0);
    for (i = array_count(&view->file_refs); i > 0; i--) {
        file = files[i-1];
        start_offset = file->hdr.file_seq == min_file_seq ?
            min_file_offset : file->hdr.hdr_size;
        end_offset = file->hdr.file_seq == max_file_seq ?
            max_file_offset : (uoff_t)-1;
        ret = mail_transaction_log_file_map(file, start_offset,
                            end_offset);
        if (ret <= 0)
            return ret;
    }

    i_assert(max_file_seq == (uint32_t)-1 ||
         max_file_seq == view->head->hdr.file_seq);
    i_assert(max_file_offset == (uoff_t)-1 ||
         max_file_offset <= view->head->sync_offset);

    view->prev_file_seq = 0;
    view->prev_file_offset = 0;

    view->cur = view->tail;
    view->cur_offset = min_file_offset;

    view->min_file_seq = min_file_seq;
    view->min_file_offset = min_file_offset;
    view->max_file_seq = max_file_seq;
    view->max_file_offset = I_MIN(max_file_offset, view->head->sync_offset);
    view->broken = FALSE;

    i_assert(view->cur_offset <= view->cur->sync_offset);
    i_assert(view->cur->hdr.file_seq == min_file_seq);
    return 1;
}

void
mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view,
                       uint32_t *file_seq_r,
                       uoff_t *file_offset_r)
{
    *file_seq_r = view->prev_file_seq;
    *file_offset_r = view->prev_file_offset;
}

static bool
mail_transaction_log_view_get_last(struct mail_transaction_log_view *view,
                   struct mail_transaction_log_file **last_r,
                   uoff_t *last_offset_r)
{
    struct mail_transaction_log_file *cur = view->cur;
    uoff_t cur_offset = view->cur_offset;
    bool last = FALSE;

    if (cur == NULL) {
        *last_r = NULL;
        return TRUE;
    }

    for (;;) {
        if (cur->hdr.file_seq == view->max_file_seq) {
            /* last file */
            if (cur_offset == view->max_file_offset ||
                cur_offset == cur->sync_offset) {
                /* we're all finished */
                last = TRUE;
            }
        } else if (cur_offset == cur->sync_offset) {
            /* end of file, go to next one */
            if (cur->next == NULL) {
                last = TRUE;
            } else {
                cur = cur->next;
                cur_offset = cur->hdr.hdr_size;
                continue;
            }
        }

        /* not EOF */
        break;
    }

    *last_r = cur;
    *last_offset_r = cur_offset;
    return last;
}

bool mail_transaction_log_view_is_last(struct mail_transaction_log_view *view)
{
    struct mail_transaction_log_file *cur;
    uoff_t cur_offset;

    return mail_transaction_log_view_get_last(view, &cur, &cur_offset);
}

void
mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view,
                    const char *fmt, ...)
{
    va_list va;

    view->broken = TRUE;

    va_start(va, fmt);
    t_push();
    mail_transaction_log_file_set_corrupted(view->log->head, "%s",
                        t_strdup_vprintf(fmt, va));
    t_pop();
    va_end(va);
}

bool
mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view)
{
    return view->broken;
}

static bool
log_view_is_record_valid(struct mail_transaction_log_file *file,
             const struct mail_transaction_header *hdr,
             const void *data)
{
    enum mail_transaction_type rec_type;
    ARRAY_TYPE(seq_range) uids = ARRAY_INIT;
    buffer_t *uid_buf = NULL;
    uint32_t rec_size;
    bool ret = TRUE;

    rec_type = hdr->type & MAIL_TRANSACTION_TYPE_MASK;
    rec_size = mail_index_offset_to_uint32(hdr->size) - sizeof(*hdr);

    /* we want to be extra careful with expunges */
    if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
        if (rec_type != (MAIL_TRANSACTION_EXPUNGE |
                 MAIL_TRANSACTION_EXPUNGE_PROT)) {
            mail_transaction_log_file_set_corrupted(file,
                "expunge record missing protection mask");
            return FALSE;
        }
    }

    if (rec_size == 0) {
        mail_transaction_log_file_set_corrupted(file,
            "Empty record contents (type=0x%x)", rec_type);
        return FALSE;
    }

    /* records that are exported by syncing and view syncing will be
       checked here so that we don't have to implement the same validation
       multiple times. other records are checked internally by
       mail_index_sync_record(). */
    t_push();
    switch (rec_type) {
    case MAIL_TRANSACTION_APPEND:
        if ((rec_size % sizeof(struct mail_index_record)) != 0) {
            mail_transaction_log_file_set_corrupted(file,
                "Invalid append record size");
            ret = FALSE;
        }
        break;
    case MAIL_TRANSACTION_EXPUNGE:
        uid_buf = buffer_create_const_data(pool_datastack_create(),
                           data, rec_size);
        array_create_from_buffer(&uids, uid_buf,
            sizeof(struct mail_transaction_expunge));
        break;
    case MAIL_TRANSACTION_FLAG_UPDATE:
        uid_buf = buffer_create_const_data(pool_datastack_create(),
                           data, rec_size);
        array_create_from_buffer(&uids, uid_buf,
            sizeof(struct mail_transaction_flag_update));
        break;
    case MAIL_TRANSACTION_KEYWORD_UPDATE: {
        const struct mail_transaction_keyword_update *rec = data;
        unsigned int seqset_offset;

        seqset_offset = sizeof(*rec) + rec->name_size;
        if ((seqset_offset % 4) != 0)
            seqset_offset += 4 - (seqset_offset % 4);

        if (seqset_offset > rec_size) {
            mail_transaction_log_file_set_corrupted(file,
                "Invalid keyword update record size");
            ret = FALSE;
            break;
        }

        uid_buf = buffer_create_const_data(pool_datastack_create(),
                    CONST_PTR_OFFSET(data, seqset_offset),
                    rec_size - seqset_offset);
        array_create_from_buffer(&uids, uid_buf, sizeof(uint32_t)*2);
        break;
    }
    case MAIL_TRANSACTION_KEYWORD_RESET:
        uid_buf = buffer_create_const_data(pool_datastack_create(),
                           data, rec_size);
        array_create_from_buffer(&uids, uid_buf,
            sizeof(struct mail_transaction_keyword_reset));
        break;
    default:
        break;
    }

    if (array_is_created(&uids)) {
        const struct seq_range *rec, *prev = NULL;
        unsigned int i, count = array_count(&uids);

        if ((uid_buf->used % uids.arr.element_size) != 0) {
            mail_transaction_log_file_set_corrupted(file,
                "Invalid record size (type=0x%x)", rec_type);
            ret = FALSE;
            count = 0;
        } else if (count == 0) {
            mail_transaction_log_file_set_corrupted(file,
                "No UID ranges (type=0x%x)", rec_type);
            ret = FALSE;
        }

        for (i = 0; i < count; i++, prev = rec) {
            rec = array_idx(&uids, i);
            if (rec->seq1 > rec->seq2 || rec->seq1 == 0) {
                mail_transaction_log_file_set_corrupted(file,
                    "Invalid UID range "
                    "(%u .. %u, type=0x%x)",
                    rec->seq1, rec->seq2, rec_type);
                ret = FALSE;
                break;
            }
            if (prev != NULL && rec->seq1 <= prev->seq2) {
                mail_transaction_log_file_set_corrupted(file,
                    "Non-sorted UID ranges (type=0x%x)",
                    rec_type);
                ret = FALSE;
                break;
            }
        }
    }
    t_pop();
    return ret;
}

static int
log_view_get_next(struct mail_transaction_log_view *view,
          const struct mail_transaction_header **hdr_r,
          const void **data_r)
{
    const struct mail_transaction_header *hdr;
    struct mail_transaction_log_file *file;
    const void *data;
    enum mail_transaction_type rec_type;
    uint32_t full_size;
    size_t file_size;

    if (view->cur == NULL)
        return 0;

    /* prev_file_offset should point to beginning of previous log record.
       when we reach EOF, it should be left there, not to beginning of the
       next file that's not included inside the view. */
    if (mail_transaction_log_view_get_last(view, &view->cur,
                           &view->cur_offset)) {
        /* if the last file was the beginning of a file, we want to
           move prev pointers there */
        view->prev_file_seq = view->cur->hdr.file_seq;
        view->prev_file_offset = view->cur_offset;
        view->cur = NULL;
        return 0;
    }

    view->prev_file_seq = view->cur->hdr.file_seq;
    view->prev_file_offset = view->cur_offset;

    file = view->cur;

    data = buffer_get_data(file->buffer, &file_size);
    file_size += file->buffer_offset;

    if (view->cur_offset + sizeof(*hdr) > file_size) {
        mail_transaction_log_file_set_corrupted(file,
            "offset points outside file "
            "(%"PRIuUOFF_T" + %"PRIuSIZE_T" > %"PRIuSIZE_T")",
            view->cur_offset, sizeof(*hdr), file_size);
        return -1;
    }

    i_assert(view->cur_offset >= file->buffer_offset);
    hdr = CONST_PTR_OFFSET(data, view->cur_offset - file->buffer_offset);
    data = CONST_PTR_OFFSET(hdr, sizeof(*hdr));

    rec_type = hdr->type & MAIL_TRANSACTION_TYPE_MASK;
    full_size = mail_index_offset_to_uint32(hdr->size);
    if (full_size < sizeof(*hdr)) {
        mail_transaction_log_file_set_corrupted(file,
            "record size too small (type=0x%x, "
            "offset=%"PRIuUOFF_T", size=%u)",
            rec_type, view->cur_offset, full_size);
        return -1;
    }

    if (file_size - view->cur_offset < full_size) {
        mail_transaction_log_file_set_corrupted(file,
            "record size too large (type=0x%x, "
            "offset=%"PRIuUOFF_T", size=%u, end=%"PRIuSIZE_T")",
            rec_type, view->cur_offset, full_size, file_size);
        return -1;
    }

    if (!log_view_is_record_valid(file, hdr, data))
        return -1;

    *hdr_r = hdr;
    *data_r = data;
    view->cur_offset += full_size;
    return 1;
}

int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
                   const struct mail_transaction_header **hdr_r,
                   const void **data_r)
{
    const struct mail_transaction_header *hdr;
    const void *data;
    int ret = 0;

    if (view->broken)
        return -1;

    ret = log_view_get_next(view, &hdr, &data);
    if (ret <= 0) {
        if (ret < 0)
            view->cur_offset = view->cur->sync_offset;
        return ret;
    }

    /* drop expunge protection */
    if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
        (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT))
        view->tmp_hdr.type = hdr->type & ~MAIL_TRANSACTION_EXPUNGE_PROT;
    else
        view->tmp_hdr.type = hdr->type;

    /* return record's size */
    view->tmp_hdr.size = mail_index_offset_to_uint32(hdr->size);
    i_assert(view->tmp_hdr.size > sizeof(*hdr));
    view->tmp_hdr.size -= sizeof(*hdr);

    *hdr_r = &view->tmp_hdr;
    *data_r = data;
    return 1;
}

void mail_transaction_log_view_seek(struct mail_transaction_log_view *view,
                    uint32_t seq, uoff_t offset)
{
    struct mail_transaction_log_file *file;

    i_assert(seq >= view->min_file_seq && seq <= view->max_file_seq);
    i_assert(seq != view->min_file_seq || offset >= view->min_file_offset);
    i_assert(seq != view->max_file_seq || offset < view->max_file_offset);

    if (view->cur == NULL || seq != view->cur->hdr.file_seq) {
        for (file = view->tail; file != NULL; file = file->next) {
            if (file->hdr.file_seq == seq)
                break;
        }
        i_assert(file != NULL);

        view->cur = file;
    }

    view->cur_offset = offset;
}