index/dbox-common/dbox-file-fix.c

	dbox-file-fix.c revision d1ba8ecbb936ace90179d2292952546708d68f71
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi/* Copyright (c) 2009-2017 Dovecot authors, see the included COPYING file */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#include "lib.h"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#include "hex-dec.h"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#include "istream.h"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#include "ostream.h"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#include "message-size.h"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#include "dbox-storage.h"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#include "dbox-file.h"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#include <stdio.h>
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi#define DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX ".broken"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomistatic int
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomidbox_file_match_pre_magic(struct istream *input,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi              uoff_t *pre_offset, size_t *need_bytes)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi{
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    const struct dbox_message_header *hdr;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    const unsigned char *data;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    size_t size;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    uoff_t offset = input->v_offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    bool have_lf = FALSE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    data = i_stream_get_data(input, &size);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (data[0] == '\n') {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        data++; size--; offset++;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        have_lf = TRUE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    i_assert(data[0] == DBOX_MAGIC_PRE[0]);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (size < sizeof(*hdr)) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        *need_bytes = sizeof(*hdr) + (have_lf ? 1 : 0);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        return -1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    hdr = (const void *)data;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (memcmp(hdr->magic_pre, DBOX_MAGIC_PRE, strlen(DBOX_MAGIC_PRE)) != 0)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        return 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (hdr->type != DBOX_MESSAGE_TYPE_NORMAL)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        return 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (hdr->space1 != ' ' || hdr->space2 != ' ')
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        return 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (hex2dec(hdr->message_size_hex, sizeof(hdr->message_size_hex)) == 0 &&
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        memcmp(hdr->message_size_hex, "0000000000000000", sizeof(hdr->message_size_hex)) != 0)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        return 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    *pre_offset = offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    return 1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi}
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomistatic bool memchr_nocontrol(const unsigned char *data, char chr,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                 unsigned int len, const unsigned char **pos_r)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi{
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    unsigned int i;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    for (i = 0; i < len; i++) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (data[i] == chr) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            *pos_r = data+i;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            return TRUE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (data[i] < ' ')
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            return FALSE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    *pos_r = NULL;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    return TRUE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi}
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomistatic int
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomidbox_file_match_post_magic(struct istream *input, bool input_full,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi               size_t *need_bytes)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi{
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    const unsigned char *data, *p;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    size_t i, size;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    bool allow_control;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    data = i_stream_get_data(input, &size);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (size < strlen(DBOX_MAGIC_POST)) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        *need_bytes = strlen(DBOX_MAGIC_POST);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        return -1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (memcmp(data, DBOX_MAGIC_POST, strlen(DBOX_MAGIC_POST)) != 0)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        return 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    /* see if the metadata block looks valid */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    for (i = strlen(DBOX_MAGIC_POST); i < size; ) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        switch (data[i]) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case '\n':
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            return 1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_GUID:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_POP3_UIDL:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_ORIG_MAILBOX:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_OLDV1_KEYWORDS:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            /* these could contain anything */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            allow_control = TRUE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            break;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_POP3_ORDER:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_RECEIVED_TIME:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_PHYSICAL_SIZE:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_VIRTUAL_SIZE:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_EXT_REF:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_OLDV1_EXPUNGED:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_OLDV1_FLAGS:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_OLDV1_SAVE_TIME:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        case DBOX_METADATA_OLDV1_SPACE:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            /* no control chars */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            allow_control = FALSE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            break;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        default:
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            if (data[i] < 'A' || data[i] > 'Z')
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                return 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            /* unknown */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            allow_control = TRUE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            break;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (allow_control) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            p = memchr(data+i, '\n', size-i);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        } else {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            if (!memchr_nocontrol(data+i, '\n', size-i, &p))
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                return 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (p == NULL) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            /* LF not found - try to find the end-of-metadata LF */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            if (input_full) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                /* can't look any further - assume it's ok */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                return 1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            *need_bytes = size+1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            return -1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        i = p - data+1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    *need_bytes = size+1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    return -1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi}
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomistatic int
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomidbox_file_find_next_magic(struct dbox_file *file, uoff_t *offset_r, bool *pre_r)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi{
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    /* We're scanning message bodies here, trying to find the beginning of
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi       the next message. Although our magic strings are very unlikely to
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi       be found in regular emails, they are much more likely when emails
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi       are stored compressed.. So try to be sure we find the correct
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi       magic markers. */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    struct istream *input = file->input;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    uoff_t orig_offset, pre_offset, post_offset, prev_offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    const unsigned char *data, *magic;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    size_t size, need_bytes, prev_need_bytes;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    int ret, match;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    *pre_r = FALSE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    orig_offset = prev_offset = input->v_offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    need_bytes = strlen(DBOX_MAGIC_POST); prev_need_bytes = 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    while ((ret = i_stream_read_bytes(input, &data, &size, need_bytes)) > 0 ||
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi           ret == -2) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        /* search for the beginning of a potential pre/post magic */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        i_assert(size > 1);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        i_assert(prev_offset != input->v_offset ||
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi             need_bytes > prev_need_bytes);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        prev_offset = input->v_offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        prev_need_bytes = need_bytes;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        magic = memchr(data, DBOX_MAGIC_PRE[0], size);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (magic == NULL) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            i_stream_skip(input, size-1);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            need_bytes = strlen(DBOX_MAGIC_POST);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            continue;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (magic == data && input->v_offset == orig_offset) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            /* beginning of the file */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        } else if (magic != data && magic[-1] == '\n') {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            /* PRE/POST block? leave \n */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            i_stream_skip(input, magic-data-1);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        } else {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            i_stream_skip(input, magic-data+1);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            need_bytes = strlen(DBOX_MAGIC_POST);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            continue;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        pre_offset = (uoff_t)-1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        match = dbox_file_match_pre_magic(input, &pre_offset, &need_bytes);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (match < 0) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            /* more data needed */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            if (ret == -2) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                i_stream_skip(input, 2);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                need_bytes = strlen(DBOX_MAGIC_POST);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            continue;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (match > 0)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            *pre_r = TRUE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        match = dbox_file_match_post_magic(input, ret == -2, &need_bytes);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (match < 0) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            /* more data needed */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            if (ret == -2) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                i_stream_skip(input, 2);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                need_bytes = strlen(DBOX_MAGIC_POST);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            continue;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (match > 0) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            post_offset = input->v_offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            if (pre_offset == (uoff_t)-1 ||
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                post_offset < pre_offset) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                pre_offset = post_offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi                *pre_r = FALSE;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (pre_offset != (uoff_t)-1) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            *offset_r = pre_offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            ret = 1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            break;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        i_stream_skip(input, size-1);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (ret <= 0) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        i_assert(ret == -1);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (input->stream_errno != 0)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            dbox_file_set_syscall_error(file, "read()");
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        else {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            ret = 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            *offset_r = input->v_offset;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    i_stream_seek(input, orig_offset);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    return ret <= 0 ? ret : 1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi}
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomistatic int
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomistream_copy(struct dbox_file *file, struct ostream *output,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        const char *out_path, uoff_t count)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi{
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    struct istream *input;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    int ret = 0;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    input = i_stream_create_limit(file->input, count);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    (void)o_stream_send_istream(output, input);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (input->stream_errno != 0) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        mail_storage_set_critical(&file->storage->storage,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            "read(%s) failed: %s", file->cur_path,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            i_stream_get_error(input));
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        ret = -1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    } else if (o_stream_flush(output) < 0) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        mail_storage_set_critical(&file->storage->storage,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            "write(%s) failed: %s", out_path,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            o_stream_get_error(output));
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        ret = -1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    } else if (input->v_offset != count) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        mail_storage_set_critical(&file->storage->storage,
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            "o_stream_send_istream(%s) copied only %"
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            PRIuUOFF_T" of %"PRIuUOFF_T" bytes",
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            out_path, input->v_offset, count);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        ret = -1;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    }
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    i_stream_unref(&input);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    return ret;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi}
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomistatic void dbox_file_skip_broken_header(struct dbox_file *file)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi{
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    const size_t magic_len = strlen(DBOX_MAGIC_PRE);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    const unsigned char *data;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    size_t i, size;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    /* if there's LF close to our position, assume that the header ends
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi       there. */
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    data = i_stream_get_data(file->input, &size);
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    if (size > file->msg_header_size + 16)
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        size = file->msg_header_size + 16;
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi    for (i = 0; i < size; i++) {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi        if (data[i] == '\n') {
316cbe323513a0f20d1cf519fe9405e231d633e2Aki Tuomi            i_stream_skip(file->input, i);
            return;
        }
    }

    /* skip at least the magic bytes if possible */
    if (size > magic_len && memcmp(data, DBOX_MAGIC_PRE, magic_len) == 0)
        i_stream_skip(file->input, magic_len);
}

static void
dbox_file_copy_metadata(struct dbox_file *file, struct ostream *output,
            bool *have_guid_r)
{
    const char *line;
    uoff_t prev_offset = file->input->v_offset;

    *have_guid_r = FALSE;
    while ((line = i_stream_read_next_line(file->input)) != NULL) {
        if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
            /* end of metadata */
            return;
        }
        if (*line < 32) {
            /* broken - possibly a new pre-magic block */
            i_stream_seek(file->input, prev_offset);
            return;
        }
        if (*line == DBOX_METADATA_VIRTUAL_SIZE) {
            /* it may be wrong - recreate it */
            continue;
        }
        if (*line == DBOX_METADATA_GUID)
            *have_guid_r = TRUE;
        o_stream_nsend_str(output, line);
        o_stream_nsend_str(output, "\n");
    }
}

static int
dbox_file_fix_write_stream(struct dbox_file *file, uoff_t start_offset,
               const char *temp_path, struct ostream *output)
{
    struct dbox_message_header msg_hdr;
    uoff_t offset, msg_size, hdr_offset, body_offset;
    bool pre, write_header, have_guid;
    struct message_size body;
    bool has_nuls;
    struct istream *body_input;
    guid_128_t guid_128;
    int ret;

    i_stream_seek(file->input, 0);
    if (start_offset > 0) {
        /* copy the valid data */
        if (stream_copy(file, output, temp_path, start_offset) < 0)
            return -1;
    } else {
        /* the file header is broken. recreate it */
        if (dbox_file_header_write(file, output) < 0) {
            dbox_file_set_syscall_error(file, "write()");
            return -1;
        }
    }

    while ((ret = dbox_file_find_next_magic(file, &offset, &pre)) > 0) {
        msg_size = offset - file->input->v_offset;
        if (msg_size < 256 && pre) {
            /* probably some garbage or some broken headers.
               we most likely don't miss anything by skipping
               over this data. */
            i_stream_skip(file->input, msg_size);
            hdr_offset = file->input->v_offset;
            ret = dbox_file_read_mail_header(file, &msg_size);
            if (ret <= 0) {
                if (ret < 0)
                    return -1;
                dbox_file_skip_broken_header(file);
                body_offset = file->input->v_offset;
                msg_size = (uoff_t)-1;
            } else {
                i_stream_skip(file->input,
                          file->msg_header_size);
                body_offset = file->input->v_offset;
                i_stream_skip(file->input, msg_size);
            }

            ret = dbox_file_find_next_magic(file, &offset, &pre);
            if (ret <= 0)
                break;

            if (!pre && msg_size == offset - body_offset) {
                /* msg header ok, copy it */
                i_stream_seek(file->input, hdr_offset);
                if (stream_copy(file, output, temp_path,
                        file->msg_header_size) < 0)
                    return -1;
                write_header = FALSE;
            } else {
                /* msg header is broken. write our own. */
                i_stream_seek(file->input, body_offset);
                if (msg_size != (uoff_t)-1) {
                    /* previous magic find might have
                       skipped too much. seek back and
                       make sure */
                    ret = dbox_file_find_next_magic(file, &offset, &pre);
                    if (ret <= 0)
                        break;
                }

                write_header = TRUE;
                msg_size = offset - body_offset;
            }
        } else {
            /* treat this data as a separate message. */
            write_header = TRUE;
            body_offset = file->input->v_offset;
        }
        /* write msg header */
        if (write_header) {
            dbox_msg_header_fill(&msg_hdr, msg_size);
            o_stream_nsend(output, &msg_hdr, sizeof(msg_hdr));
        }
        /* write msg body */
        i_assert(file->input->v_offset == body_offset);
        if (stream_copy(file, output, temp_path, msg_size) < 0)
            return -1;
        i_assert(file->input->v_offset == offset);

        /* get message body size */
        i_stream_seek(file->input, body_offset);
        body_input = i_stream_create_limit(file->input, msg_size);
        ret = message_get_body_size(body_input, &body, &has_nuls);
        i_stream_unref(&body_input);
        if (ret < 0) {
            mail_storage_set_critical(&file->storage->storage,
                "read(%s) failed: %s", file->cur_path,
                i_stream_get_error(body_input));
            return -1;
        }

        /* write msg metadata. */
        i_assert(file->input->v_offset == offset);
        ret = dbox_file_metadata_skip_header(file);
        if (ret < 0)
            return -1;
        o_stream_nsend_str(output, DBOX_MAGIC_POST);
        if (ret == 0)
            have_guid = FALSE;
        else
            dbox_file_copy_metadata(file, output, &have_guid);
        if (!have_guid) {
            guid_128_generate(guid_128);
            o_stream_nsend_str(output,
                t_strdup_printf("%c%s\n", DBOX_METADATA_GUID,
                guid_128_to_string(guid_128)));
        }
        o_stream_nsend_str(output,
            t_strdup_printf("%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE,
                    (unsigned long long)body.virtual_size));
        o_stream_nsend_str(output, "\n");
        if (output->stream_errno != 0)
            break;
    }
    if (o_stream_flush(output) < 0) {
        mail_storage_set_critical(&file->storage->storage,
            "write(%s) failed: %s", temp_path, o_stream_get_error(output));
        ret = -1;
    }
    return ret;
}

int dbox_file_fix(struct dbox_file *file, uoff_t start_offset)
{
    struct ostream *output;
    const char *dir, *p, *temp_path, *broken_path;
    bool deleted, have_messages;
    int fd, ret;

    i_assert(dbox_file_is_open(file));

    p = strrchr(file->cur_path, '/');
    i_assert(p != NULL);
    dir = t_strdup_until(file->cur_path, p);

    temp_path = t_strdup_printf("%s/%s", dir, dbox_generate_tmp_filename());
    fd = file->storage->v.file_create_fd(file, temp_path, FALSE);
    if (fd == -1)
        return -1;

    output = o_stream_create_fd_file(fd, 0, FALSE);
    o_stream_cork(output);
    ret = dbox_file_fix_write_stream(file, start_offset, temp_path, output);
    if (ret < 0)
        o_stream_ignore_last_errors(output);
    have_messages = output->offset > file->file_header_size;
    o_stream_unref(&output);
    if (close(fd) < 0) {
        mail_storage_set_critical(&file->storage->storage,
                      "close(%s) failed: %m", temp_path);
        ret = -1;
    }
    if (ret < 0) {
        if (unlink(temp_path) < 0) {
            mail_storage_set_critical(&file->storage->storage,
                "unlink(%s) failed: %m", temp_path);
        }
        return -1;
    }
    /* keep a copy of the original file in case someone wants to look
       at it */
    broken_path = t_strconcat(file->cur_path,
                  DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX, NULL);
    if (link(file->cur_path, broken_path) < 0) {
        mail_storage_set_critical(&file->storage->storage,
                      "link(%s, %s) failed: %m",
                      file->cur_path, broken_path);
    } else {
        i_warning("dbox: Copy of the broken file saved to %s",
              broken_path);
    }
    if (!have_messages) {
        /* the resulting file has no messages. just delete the file. */
        dbox_file_close(file);
        i_unlink(temp_path);
        i_unlink(file->cur_path);
        return 0;
    }
    if (rename(temp_path, file->cur_path) < 0) {
        mail_storage_set_critical(&file->storage->storage,
                      "rename(%s, %s) failed: %m",
                      temp_path, file->cur_path);
        return -1;
    }

    /* file was successfully recreated - reopen it */
    dbox_file_close(file);
    if (dbox_file_open(file, &deleted) <= 0) {
        mail_storage_set_critical(&file->storage->storage,
            "dbox_file_fix(%s): reopening file failed",
            file->cur_path);
        return -1;
    }
    return 1;
}