src/lib-index/mail-transaction-log-view.c

	mail-transaction-log-view.c revision ccffbed92cb02c24fd717808a84138240bf1885b
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "lib.h"
be5c76fabc7439fd33bc799bc3ab3f570799977bTimo Sirainen#include "array.h"
be5c76fabc7439fd33bc799bc3ab3f570799977bTimo Sirainen#include "mail-index-private.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "mail-transaction-log-private.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "mail-transaction-util.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
1e21e6be70994b1aa9e52ca0e2f51afefca6d0dfTimo Sirainenstruct mail_transaction_log_view {
04052d7cacaa866a3f00afb4e104fa46c04c1dd7Timo Sirainen    struct mail_transaction_log *log;
1e21e6be70994b1aa9e52ca0e2f51afefca6d0dfTimo Sirainen        struct mail_transaction_log_view *next;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen    uint32_t min_file_seq, max_file_seq;
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen    uoff_t min_file_offset, max_file_offset;
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen    enum mail_transaction_type type_mask;
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen    struct mail_transaction_header tmp_hdr;
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen
6e5a4cdf7ef123589e2409e0012b1024c97957d5Aki Tuomi    /* a list of log files we've referenced. we have to keep this list
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen       explicitly because more files may be added into the linked list
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen       at any time. */
b82474d60c15409eda71c55971710fd3b12b8a0fTimo Sirainen    ARRAY_DEFINE(file_refs, struct mail_transaction_log_file *);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen        struct mail_transaction_log_file *cur, *head, *tail;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    uoff_t cur_offset;
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen    uint32_t prev_file_seq;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    uoff_t prev_file_offset;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    unsigned int broken:1;
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen};
484e12acec34f16e5a8adc001e23ae48f1dda8c7Timo Sirainen
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainenstruct mail_transaction_log_view *
22535a9e685e29214082878e37a267157044618eTimo Sirainenmail_transaction_log_view_open(struct mail_transaction_log *log)
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen{
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    struct mail_transaction_log_view *view;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen    view = i_new(struct mail_transaction_log_view, 1);
f3d506e525a720f214020ca0f989a1966b30edaeTimo Sirainen    view->log = log;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    view->broken = TRUE;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    view->head = view->tail = view->log->head;
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen    view->head->refcount++;
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen    i_array_init(&view->file_refs, 8);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    array_append(&view->file_refs, &view->head, 1);
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen    view->next = log->views;
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen    log->views = view;
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen    return view;
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen}
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen
adea1e1e46ccb4ae107767fd930e3d1fb4f1d11dTimo Sirainenstatic void
adea1e1e46ccb4ae107767fd930e3d1fb4f1d11dTimo Sirainenmail_transaction_log_view_unref_all(struct mail_transaction_log_view *view)
adea1e1e46ccb4ae107767fd930e3d1fb4f1d11dTimo Sirainen{
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    struct mail_transaction_log_file *const *files;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    unsigned int i, count;
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen
f3d506e525a720f214020ca0f989a1966b30edaeTimo Sirainen    files = array_get(&view->file_refs, &count);
74674a53a72dab535c61f455b2246ef2797844eaTimo Sirainen    for (i = 0; i < count; i++)
74674a53a72dab535c61f455b2246ef2797844eaTimo Sirainen        files[i]->refcount--;
74674a53a72dab535c61f455b2246ef2797844eaTimo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    array_clear(&view->file_refs);
1ae87afde32c1ac73909dfacfd59641b470a3e93Martti Rannanjärvi}
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen
10f6f2224c897fc543973efd2f46b86a3ab1148dAki Tuomivoid mail_transaction_log_view_close(struct mail_transaction_log_view **_view)
10f6f2224c897fc543973efd2f46b86a3ab1148dAki Tuomi{
10f6f2224c897fc543973efd2f46b86a3ab1148dAki Tuomi        struct mail_transaction_log_view *view = *_view;
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    struct mail_transaction_log_view **p;
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen    *_view = NULL;
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen    for (p = &view->log->views; *p != NULL; p = &(*p)->next) {
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen        if (*p == view) {
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen            *p = view->next;
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen            break;
e48d89622047bd8bbd0475b881ca9377d592f535Timo Sirainen        }
2d8f66596f445dd8b399b7032c3f0e9202015b63Timo Sirainen    }
2d8f66596f445dd8b399b7032c3f0e9202015b63Timo Sirainen
2d8f66596f445dd8b399b7032c3f0e9202015b63Timo Sirainen    mail_transaction_log_view_unref_all(view);
04052d7cacaa866a3f00afb4e104fa46c04c1dd7Timo Sirainen    mail_transaction_logs_clean(view->log);
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    array_free(&view->file_refs);
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    i_free(view);
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen}
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenvoid mail_transaction_log_views_close(struct mail_transaction_log *log)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen{
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    struct mail_transaction_log_view *view;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    for (view = log->views; view != NULL; view = view->next)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen        view->log = NULL;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen}
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenint
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainenmail_transaction_log_view_set(struct mail_transaction_log_view *view,
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen                  uint32_t min_file_seq, uoff_t min_file_offset,
dc9bfb7dc057964238e181d3d8b08751527bb08aTimo Sirainen                  uint32_t max_file_seq, uoff_t max_file_offset,
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen                  enum mail_transaction_type type_mask)
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen{
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    struct mail_transaction_log_file *file, *first;
e48d89622047bd8bbd0475b881ca9377d592f535Timo Sirainen    uint32_t seq;
04052d7cacaa866a3f00afb4e104fa46c04c1dd7Timo Sirainen    uoff_t end_offset;
f3d506e525a720f214020ca0f989a1966b30edaeTimo Sirainen    int ret;
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    i_assert(view->log != NULL);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    i_assert(min_file_seq <= max_file_seq);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    if (view->log == NULL) {
9625595c47c665f5aee57ebfcb1fcbe9ad1bf3a0Martti Rannanjärvi        /* transaction log is closed already. this log view shouldn't
be5c76fabc7439fd33bc799bc3ab3f570799977bTimo Sirainen           be used anymore. */
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        return -1;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    }
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
1e21e6be70994b1aa9e52ca0e2f51afefca6d0dfTimo Sirainen    if (min_file_seq == 0) {
1e21e6be70994b1aa9e52ca0e2f51afefca6d0dfTimo Sirainen        /* new index, transaction file not synced yet */
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen        min_file_seq = 1;
        min_file_offset = 0;

        if (max_file_seq == 0) {
            max_file_seq = min_file_seq;
            max_file_offset = min_file_offset;
        }
    }

    if (min_file_seq == view->log->files->hdr.prev_file_seq &&
        min_file_offset == view->log->files->hdr.prev_file_offset) {
        /* we can skip this */
        min_file_seq = view->log->files->hdr.file_seq;
        min_file_offset = 0;

        if (min_file_seq > max_file_seq) {
            /* empty view */
            max_file_seq = min_file_seq;
            max_file_offset = min_file_offset;
        }
    }

    /* find the oldest log file first. */
    ret = mail_transaction_log_file_find(view->log, min_file_seq, &file);
    if (ret <= 0)
        return ret;

    if (min_file_offset == 0) {
        /* this could happen if internal transactions haven't yet been
           committed but external are. just assume we're at the
           beginning. */
        min_file_offset = file->hdr.hdr_size;
        if (max_file_offset == 0 && min_file_seq == max_file_seq)
            max_file_offset = min_file_offset;
    }
    i_assert(min_file_offset >= file->hdr.hdr_size);

    if (min_file_seq == max_file_seq && min_file_offset > max_file_offset) {
        /* log file offset is probably corrupted in the index file. */
        mail_transaction_log_view_set_corrupted(view,
            "file_seq=%u, min_file_offset (%"PRIuUOFF_T
            ") > max_file_offset (%"PRIuUOFF_T")",
            min_file_seq, min_file_offset, max_file_offset);
        return -1;
    }

    end_offset = min_file_seq == max_file_seq ?
        max_file_offset : (uoff_t)-1;
    ret = mail_transaction_log_file_map(file, min_file_offset, end_offset);
    if (ret <= 0)
        return ret;
    first = file;

    for (seq = min_file_seq+1; seq <= max_file_seq; seq++) {
        file = file->next;
        if (file == NULL || file->hdr.file_seq != seq) {
            /* see if we could find the missing file */
            ret = mail_transaction_log_file_find(view->log,
                                 seq, &file);
            if (ret <= 0) {
                if (ret < 0)
                    return -1;

                /* not found / corrupted */
                file = NULL;
            }
        }

        if (file == NULL || file->hdr.file_seq != seq) {
            if (file == NULL && max_file_seq == (uint32_t)-1) {
                /* we just wanted to sync everything */
                i_assert(max_file_offset == (uoff_t)-1);
                max_file_seq = seq-1;
                break;
            }

            /* missing files in the middle */
            return 0;
        }

        end_offset = file->hdr.file_seq == max_file_seq ?
            max_file_offset : (uoff_t)-1;
        ret = mail_transaction_log_file_map(file, file->hdr.hdr_size,
                            end_offset);
        if (ret <= 0)
            return ret;
    }

    i_assert(max_file_offset == (uoff_t)-1 ||
         max_file_offset <= file->sync_offset);

    /* we have all of them. update refcounts. */
    mail_transaction_log_view_unref_all(view);

    view->tail = first;
    view->head = view->log->head;

    /* reference all used files */
    for (file = view->tail; file != NULL; file = file->next) {
        array_append(&view->file_refs, &file, 1);
        file->refcount++;
    }

    view->prev_file_seq = 0;
    view->prev_file_offset = 0;

    view->cur = first;
    view->cur_offset = min_file_offset;

    view->min_file_seq = min_file_seq;
    view->min_file_offset = min_file_offset;
    view->max_file_seq = max_file_seq;
    view->max_file_offset = max_file_offset;
    view->type_mask = type_mask;
    view->broken = FALSE;

    i_assert(view->cur_offset <= view->cur->sync_offset);
    i_assert(view->cur->hdr.file_seq == min_file_seq);
    return 1;
}

void
mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view,
                       uint32_t *file_seq_r,
                       uoff_t *file_offset_r)
{
    *file_seq_r = view->prev_file_seq;
    *file_offset_r = view->prev_file_offset;
}

void
mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view,
                    const char *fmt, ...)
{
    va_list va;

    view->broken = TRUE;

    va_start(va, fmt);
    t_push();
    mail_transaction_log_file_set_corrupted(view->log->head, "%s",
                        t_strdup_vprintf(fmt, va));
    t_pop();
    va_end(va);
}

bool
mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view)
{
    return view->broken;
}

static int
log_view_get_next(struct mail_transaction_log_view *view,
          const struct mail_transaction_header **hdr_r,
          const void **data_r)
{
    const struct mail_transaction_header *hdr;
    struct mail_transaction_log_file *file;
    const struct mail_transaction_type_map *type_rec;
    const void *data;
    unsigned int record_size;
    enum mail_transaction_type hdr_type;
    uint32_t hdr_size;
    size_t file_size;

    if (view->cur == NULL)
        return 0;

    /* prev_file_offset should point to beginning of previous log record.
       when we reach EOF, it should be left there, not to beginning of the
       next file. */
    view->prev_file_seq = view->cur->hdr.file_seq;
    view->prev_file_offset = view->cur_offset;

    if (view->cur->hdr.file_seq == view->max_file_seq) {
        /* last file */
        if (view->cur_offset == view->max_file_offset ||
            view->cur_offset == view->cur->sync_offset) {
            /* we're all finished */
            view->cur = NULL;
            return 0;
        }
    } else if (view->cur_offset == view->cur->sync_offset) {
        /* end of file, go to next one */
        view->cur = view->cur->next;
        if (view->cur == NULL)
            return 0;

        view->cur_offset = view->cur->hdr.hdr_size;
        return log_view_get_next(view, hdr_r, data_r);
    }
    file = view->cur;

    data = buffer_get_data(file->buffer, &file_size);
    file_size += file->buffer_offset;

    if (view->cur_offset + sizeof(*hdr) > file_size) {
        mail_transaction_log_file_set_corrupted(file,
            "offset points outside file "
            "(%"PRIuUOFF_T" + %"PRIuSIZE_T" > %"PRIuSIZE_T")",
            view->cur_offset, sizeof(*hdr), file_size);
        return -1;
    }

    i_assert(view->cur_offset >= file->buffer_offset);
    hdr = CONST_PTR_OFFSET(data, view->cur_offset - file->buffer_offset);
    data = CONST_PTR_OFFSET(hdr, sizeof(*hdr));

    hdr_type = hdr->type & MAIL_TRANSACTION_TYPE_MASK;
    hdr_size = mail_index_offset_to_uint32(hdr->size);
    if (hdr_size < sizeof(*hdr)) {
        type_rec = NULL;
        record_size = 0;
    } else {
        type_rec = mail_transaction_type_lookup(hdr->type);
        if (type_rec != NULL)
            record_size = type_rec->record_size;
        else {
            mail_transaction_log_file_set_corrupted(file,
                "unknown record type 0x%x", hdr_type);
            return -1;
        }
    }

    if (hdr_size < sizeof(*hdr) + record_size) {
        mail_transaction_log_file_set_corrupted(file,
            "record size too small (type=0x%x, "
            "offset=%"PRIuUOFF_T", size=%u)",
            hdr_type, view->cur_offset, hdr_size);
        return -1;
    }

    if (record_size != 0 &&
        (hdr_size - sizeof(*hdr)) % record_size != 0) {
        mail_transaction_log_file_set_corrupted(file,
            "record size wrong (type 0x%x, "
            "offset=%"PRIuUOFF_T", size=%"PRIuSIZE_T" %% %u != 0)",
            hdr_type, view->cur_offset, (hdr_size - sizeof(*hdr)),
            record_size);
        return -1;
    }

    if (file_size - view->cur_offset < hdr_size) {
        mail_transaction_log_file_set_corrupted(file,
            "record size too large (type=0x%x, "
            "offset=%"PRIuUOFF_T", size=%u, end=%"PRIuSIZE_T")",
            hdr_type, view->cur_offset, hdr_size, file_size);
        return -1;
    }

    if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
        if (hdr_type != (MAIL_TRANSACTION_EXPUNGE |
                 MAIL_TRANSACTION_EXPUNGE_PROT)) {
            mail_transaction_log_file_set_corrupted(file,
                "found expunge without protection mask");
            return -1;
        }
    } else if (hdr_type != type_rec->type) {
        mail_transaction_log_file_set_corrupted(file,
            "extra bits in header type: 0x%x", hdr_type);
        return -1;
    } else if (hdr_type == MAIL_TRANSACTION_EXT_INTRO) {
        const struct mail_transaction_ext_intro *intro;
        uint32_t i;

        for (i = 0; i < hdr_size; ) {
            if (i + sizeof(*intro) > hdr_size) {
                /* should be just extra padding */
                break;
            }

            intro = CONST_PTR_OFFSET(data, i);
            if (intro->name_size >
                hdr_size - sizeof(*hdr) - sizeof(*intro)) {
                mail_transaction_log_file_set_corrupted(file,
                    "extension intro: name_size too large");
                return -1;
            }

            i += sizeof(*intro) + intro->name_size;
        }
    }

    *hdr_r = hdr;
    *data_r = data;
    view->cur_offset += hdr_size;
    return 1;
}

int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
                   const struct mail_transaction_header **hdr_r,
                   const void **data_r, bool *skipped_r)
{
    const struct mail_transaction_header *hdr;
    const void *data;
    int ret = 0;

    if (skipped_r != NULL)
        *skipped_r = FALSE;
    if (view->broken)
        return -1;

    while ((ret = log_view_get_next(view, &hdr, &data)) > 0) {
        if ((view->type_mask & hdr->type) != 0) {
            /* looks like this is within our mask, but expunge
               protection may mess up the check. */
            if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0 ||
                (view->type_mask & MAIL_TRANSACTION_EXPUNGE) != 0)
                break;
        }

        /* we don't want this record */
        if (skipped_r != NULL)
            *skipped_r = TRUE;

        /* FIXME: hide flag/cache updates for appends if
           append isn't in mask */
    }

    if (ret < 0) {
        view->cur_offset = view->cur->sync_offset;
        return -1;
    }
    if (ret == 0)
        return 0;

    view->tmp_hdr = *hdr;
    view->tmp_hdr.size =
        mail_index_offset_to_uint32(view->tmp_hdr.size) - sizeof(*hdr);
    i_assert(view->tmp_hdr.size != 0);

    if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
        /* hide expunge protection */
        view->tmp_hdr.type &= ~MAIL_TRANSACTION_EXPUNGE_PROT;
    }

    *hdr_r = &view->tmp_hdr;
    *data_r = data;
    return 1;
}

void mail_transaction_log_view_seek(struct mail_transaction_log_view *view,
                    uint32_t seq, uoff_t offset)
{
    struct mail_transaction_log_file *file;

    i_assert(seq >= view->min_file_seq && seq <= view->max_file_seq);
    i_assert(seq != view->min_file_seq || offset >= view->min_file_offset);
    i_assert(seq != view->max_file_seq || offset < view->max_file_offset);

    if (view->cur == NULL || seq != view->cur->hdr.file_seq) {
        for (file = view->tail; file != NULL; file = file->next) {
            if (file->hdr.file_seq == seq)
                break;
        }
        i_assert(file != NULL);

        view->cur = file;
    }

    view->cur_offset = offset;
}