src/lib-mail/istream-header-filter.c

	istream-header-filter.c revision 71ecc97593a25c2444316b74e5882e6ce0a82c0c
a8c5a86d183db25a57bf193c06b41e092ec2e151Timo Sirainen/* Copyright (c) 2003-2016 Dovecot authors, see the included COPYING file */
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen#include "lib.h"
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen#include "array.h"
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen#include "message-parser.h"
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen#include "istream-private.h"
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen#include "istream-header-filter.h"
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenstruct header_filter_istream {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    struct istream_private istream;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    pool_t pool;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    struct message_header_parser_ctx *hdr_ctx;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    const char **headers;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int headers_count;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    header_filter_callback *callback;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    void *context;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    buffer_t *hdr_buf;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    struct message_size header_size;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    uoff_t skip_count;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    uoff_t last_lf_offset;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int cur_line, parsed_lines;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    ARRAY(unsigned int) match_change_lines;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen    unsigned int header_read:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int seen_eoh:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int header_parsed:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int headers_edited:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int exclude:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int crlf:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int crlf_preserve:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int hide_body:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int add_missing_eoh:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int end_body_with_lf:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int last_lf_added:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int last_orig_crlf:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int last_added_newline:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int eoh_not_matched:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int callbacks_called:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    unsigned int prev_matched:1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen};
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenheader_filter_callback *null_header_filter_callback = NULL;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenstatic ssize_t i_stream_header_filter_read(struct istream_private *stream);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenstatic void i_stream_header_filter_destroy(struct iostream_private *stream)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen{
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    struct header_filter_istream *mstream =
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        (struct header_filter_istream *)stream;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    if (mstream->hdr_ctx != NULL)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        message_parse_header_deinit(&mstream->hdr_ctx);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    if (array_is_created(&mstream->match_change_lines))
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        array_free(&mstream->match_change_lines);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    pool_unref(&mstream->pool);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen}
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenstatic ssize_t
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenread_mixed(struct header_filter_istream *mstream, size_t body_highwater_size)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen{
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    const unsigned char *data;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    size_t pos;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    ssize_t ret;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    if (mstream->hide_body) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        mstream->istream.istream.eof = TRUE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        return -1;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    data = i_stream_get_data(mstream->istream.parent, &pos);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    if (pos <= body_highwater_size) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        i_assert(pos == body_highwater_size ||
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen             (mstream->end_body_with_lf &&
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen              pos+1 == body_highwater_size));
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        ret = i_stream_read(mstream->istream.parent);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        mstream->istream.istream.stream_errno =
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            mstream->istream.parent->stream_errno;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        mstream->istream.istream.eof = mstream->istream.parent->eof;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        if (ret <= 0) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            data = mstream->hdr_buf->data;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            pos = mstream->hdr_buf->used;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            i_assert(pos > 0);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            if (mstream->end_body_with_lf && data[pos-1] != '\n' &&
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                ret == -1 && mstream->istream.istream.eof) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                /* add missing trailing LF to body */
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                if (mstream->crlf)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                    buffer_append_c(mstream->hdr_buf, '\r');
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                buffer_append_c(mstream->hdr_buf, '\n');
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                mstream->istream.buffer = mstream->hdr_buf->data;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                mstream->istream.pos = mstream->hdr_buf->used;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                return mstream->hdr_buf->used - pos;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            return ret;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        data = i_stream_get_data(mstream->istream.parent, &pos);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    buffer_append(mstream->hdr_buf, data + body_highwater_size,
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen              pos - body_highwater_size);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    mstream->istream.buffer = buffer_get_data(mstream->hdr_buf, &pos);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    ret = (ssize_t)(pos - mstream->istream.pos - mstream->istream.skip);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    i_assert(ret > 0);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    mstream->istream.pos = pos;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    return ret;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen}
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenstatic int cmp_uint(const unsigned int *i1, const unsigned int *i2)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen{
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    return *i1 < *i2 ? -1 :
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        (*i1 > *i2 ? 1 : 0);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen}
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenstatic bool match_line_changed(struct header_filter_istream *mstream)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen{
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    if (!array_is_created(&mstream->match_change_lines))
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        return FALSE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    return array_bsearch(&mstream->match_change_lines, &mstream->cur_line,
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                 cmp_uint) != NULL;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen}
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenstatic void add_eol(struct header_filter_istream *mstream, bool orig_crlf)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen{
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    if (mstream->crlf || (orig_crlf && mstream->crlf_preserve))
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        buffer_append(mstream->hdr_buf, "\r\n", 2);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    else
c971768955f826fb965d8ffbb13dac93c9bbead8Phil Carmody        buffer_append_c(mstream->hdr_buf, '\n');
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    mstream->last_orig_crlf = orig_crlf;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    mstream->last_added_newline = TRUE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen}
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainenstatic ssize_t hdr_stream_update_pos(struct header_filter_istream *mstream)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen{
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    ssize_t ret;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    size_t pos;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    mstream->istream.buffer = buffer_get_data(mstream->hdr_buf, &pos);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    ret = (ssize_t)(pos - mstream->istream.pos - mstream->istream.skip);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    i_assert(ret >= 0);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    mstream->istream.pos = pos;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    return ret;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen}
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainenstatic ssize_t read_header(struct header_filter_istream *mstream)
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen{
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen    struct message_header_line *hdr;
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen    uoff_t highwater_offset;
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen    ssize_t ret, ret2;
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen    int hdr_ret;
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen    if (mstream->hdr_ctx == NULL) {
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen        mstream->hdr_ctx =
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            message_parse_header_init(mstream->istream.parent,
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                          NULL, 0);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    /* remove skipped data from hdr_buf */
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    buffer_copy(mstream->hdr_buf, 0,
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            mstream->hdr_buf, mstream->istream.skip, (size_t)-1);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        mstream->istream.pos -= mstream->istream.skip;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    mstream->istream.skip = 0;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    buffer_set_used_size(mstream->hdr_buf, mstream->istream.pos);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    if (mstream->header_read) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        i_assert(mstream->istream.skip == 0);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        highwater_offset = mstream->istream.istream.v_offset +
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            mstream->istream.pos;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        if (highwater_offset >= mstream->header_size.virtual_size) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            /* we want to return mixed headers and body */
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            size_t body_highwater_size = highwater_offset -
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                mstream->header_size.virtual_size;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            return read_mixed(mstream, body_highwater_size);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    if (mstream->hdr_buf->used >= mstream->istream.max_buffer_size)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        return -2;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen    while ((hdr_ret = message_parse_header_next(mstream->hdr_ctx,
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                            &hdr)) > 0) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        bool matched;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        if (!hdr->continued)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            mstream->cur_line++;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        if (hdr->eoh) {
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen            mstream->seen_eoh = TRUE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            matched = FALSE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            if (mstream->header_parsed && !mstream->headers_edited) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                if (mstream->eoh_not_matched)
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                    matched = !matched;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            } else if (mstream->callback != NULL) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                mstream->callback(mstream, hdr, &matched,
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                          mstream->context);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                mstream->callbacks_called = TRUE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            }
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen            if (matched) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                mstream->seen_eoh = FALSE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                mstream->eoh_not_matched = TRUE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                continue;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            add_eol(mstream, hdr->crlf_newline);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            continue;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        if (hdr->continued) {
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen            /* Header line continued - use only the first line's
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen               matched-result. Otherwise multiline headers might
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen               end up being only partially picked, which wouldn't
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen               be very good. However, allow callbacks to modify
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen               the headers in any way they want. */
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            matched = mstream->prev_matched;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        } else if (mstream->headers_count == 0) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            /* no include/exclude headers - default matching */
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen            matched = FALSE;
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen        } else {
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen            matched = i_bsearch(hdr->name, mstream->headers,
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen                        mstream->headers_count,
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen                        sizeof(*mstream->headers),
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen                        bsearch_strcasecmp) != NULL;
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen        }
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        if (mstream->callback == NULL) {
ddd1adf19cacc6186fbc713d255e1e82086d7751Timo Sirainen            /* nothing gets excluded */
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen        } else if (!mstream->header_parsed || mstream->headers_edited) {
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            /* first time in this line or we have actually modified
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen               the header so we always want to call the callbacks */
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            bool orig_matched = matched;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            mstream->parsed_lines = mstream->cur_line;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            mstream->callback(mstream, hdr, &matched,
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen                      mstream->context);
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            mstream->callbacks_called = TRUE;
37bd98570b30513255a19cc52de16594bc4256e0Timo Sirainen            if (matched != orig_matched &&
                !hdr->continued && !mstream->headers_edited) {
                if (!array_is_created(&mstream->match_change_lines))
                    i_array_init(&mstream->match_change_lines, 8);
                array_append(&mstream->match_change_lines,
                         &mstream->cur_line, 1);
            }
        } else if (!hdr->continued) {
            /* second time in this line. was it excluded by the
               callback the first time? */
            if (match_line_changed(mstream))
                matched = !matched;
        }
        mstream->prev_matched = matched;

        if (matched == mstream->exclude) {
            /* ignore */
        } else {
            if (!hdr->continued) {
                buffer_append(mstream->hdr_buf,
                          hdr->name, hdr->name_len);
                buffer_append(mstream->hdr_buf,
                          hdr->middle, hdr->middle_len);
            }
            buffer_append(mstream->hdr_buf,
                      hdr->value, hdr->value_len);
            if (!hdr->no_newline)
                add_eol(mstream, hdr->crlf_newline);

            if (mstream->skip_count >= mstream->hdr_buf->used) {
                /* we need more */
                mstream->skip_count -= mstream->hdr_buf->used;
                buffer_set_used_size(mstream->hdr_buf, 0);
            } else {
                if (mstream->skip_count > 0) {
                    mstream->istream.skip =
                        mstream->skip_count;
                    mstream->skip_count = 0;
                }
                break;
            }
        }
        if (mstream->hdr_buf->used >= mstream->istream.max_buffer_size)
            break;
    }
    if (mstream->hdr_buf->used > 0) {
        const unsigned char *data = mstream->hdr_buf->data;
        mstream->last_added_newline =
            data[mstream->hdr_buf->used-1] == '\n';
    }

    if (hdr_ret < 0) {
        if (mstream->istream.parent->stream_errno != 0) {
            mstream->istream.istream.stream_errno =
                mstream->istream.parent->stream_errno;
            mstream->istream.istream.eof =
                mstream->istream.parent->eof;
            return -1;
        }
        if (!mstream->seen_eoh && mstream->add_missing_eoh) {
            mstream->seen_eoh = TRUE;
            if (!mstream->last_added_newline)
                add_eol(mstream, mstream->last_orig_crlf);
            add_eol(mstream, mstream->last_orig_crlf);
        }
    }

    /* don't copy eof here because we're only returning headers here.
       the body will be returned in separate read() call. */
    ret = hdr_stream_update_pos(mstream);

    if (hdr_ret == 0) {
        /* need more data to finish parsing headers. we may have some
           data already available though. */
        return ret;
    }

    if (hdr == NULL) {
        /* finished */
        message_parse_header_deinit(&mstream->hdr_ctx);
        mstream->hdr_ctx = NULL;

        if ((!mstream->header_parsed || mstream->headers_edited ||
             mstream->callbacks_called) &&
            mstream->callback != NULL) {
            bool matched = FALSE;
            mstream->callback(mstream, NULL,
                      &matched, mstream->context);
            /* check if the callback added more headers.
               this is allowed only if EOH wasn't added yet. */
            ret2 = hdr_stream_update_pos(mstream);
            if (!mstream->seen_eoh)
                ret += ret2;
            else {
                i_assert(ret2 == 0);
            }
        }
        mstream->header_parsed = TRUE;
        mstream->header_read = TRUE;
        mstream->callbacks_called = FALSE;

        mstream->header_size.physical_size =
            mstream->istream.parent->v_offset;
        mstream->header_size.virtual_size =
            mstream->istream.istream.v_offset +
            mstream->istream.pos;
    }

    if (ret == 0) {
        /* we're at the end of headers. */
        i_assert(hdr == NULL);
        i_assert(mstream->istream.istream.v_offset +
             mstream->istream.pos ==
             mstream->header_size.virtual_size);

        return i_stream_header_filter_read(&mstream->istream);
    }

    return ret;
}

static ssize_t
handle_end_body_with_lf(struct header_filter_istream *mstream, ssize_t ret)
{
    struct istream_private *stream = &mstream->istream;
    const unsigned char *data;
    size_t size, last_offset;
    bool last_lf;

    data = i_stream_get_data(stream->parent, &size);
    last_offset = stream->parent->v_offset + size-1;

    if (mstream->last_lf_offset == last_offset)
        last_lf = TRUE;
    else if (size > 0)
        last_lf = data[size-1] == '\n';
    else
        last_lf = FALSE;

    if (ret == -1 && stream->parent->eof && !last_lf) {
        /* missing LF, need to add it */
        i_assert(!mstream->last_lf_added);
        i_assert(size == 0 || data[size-1] != '\n');

        buffer_set_used_size(mstream->hdr_buf, 0);
        buffer_append(mstream->hdr_buf, data, size);
        if (mstream->crlf)
            buffer_append_c(mstream->hdr_buf, '\r');
        buffer_append_c(mstream->hdr_buf, '\n');
        mstream->last_lf_offset = last_offset;
        mstream->last_lf_added = TRUE;

        stream->skip = 0;
        stream->pos = mstream->hdr_buf->used;
        stream->buffer = mstream->hdr_buf->data;
        return mstream->crlf ? 2 : 1;
    } else {
        mstream->last_lf_offset = last_lf ? last_offset : (uoff_t)-1;
    }
    return ret;
}

static ssize_t i_stream_header_filter_read(struct istream_private *stream)
{
    struct header_filter_istream *mstream =
        (struct header_filter_istream *)stream;
    uoff_t v_offset;
    ssize_t ret;

    if (mstream->last_lf_added) {
        stream->istream.eof = TRUE;
        return -1;
    }

    if (!mstream->header_read ||
        stream->istream.v_offset < mstream->header_size.virtual_size) {
        ret = read_header(mstream);
        if (ret != -2 || stream->pos != stream->skip)
            return ret;
    }

    if (mstream->hide_body) {
        stream->istream.eof = TRUE;
        return -1;
    }

    v_offset = stream->parent_start_offset + stream->istream.v_offset -
        mstream->header_size.virtual_size +
        mstream->header_size.physical_size;
    i_stream_seek(stream->parent, v_offset);
    ret = i_stream_read_copy_from_parent(&stream->istream);
    if (mstream->end_body_with_lf)
        ret = handle_end_body_with_lf(mstream, ret);
    return ret;
}

static void
i_stream_header_filter_seek_to_header(struct header_filter_istream *mstream,
                      uoff_t v_offset)
{
    i_stream_seek(mstream->istream.parent,
              mstream->istream.parent_start_offset);
    mstream->istream.parent_expected_offset =
        mstream->istream.parent_start_offset;
    mstream->istream.access_counter =
        mstream->istream.parent->real_stream->access_counter;

    if (mstream->hdr_ctx != NULL)
        message_parse_header_deinit(&mstream->hdr_ctx);
    mstream->skip_count = v_offset;
    mstream->cur_line = 0;
    mstream->prev_matched = FALSE;
    mstream->header_read = FALSE;
    mstream->seen_eoh = FALSE;
}

static int skip_header(struct header_filter_istream *mstream)
{
    size_t pos;

    if (mstream->header_read)
        return 0;

    if (mstream->istream.access_counter !=
        mstream->istream.parent->real_stream->access_counter) {
        /* need to re-parse headers */
        i_stream_header_filter_seek_to_header(mstream, 0);
    }

    while (!mstream->header_read &&
           i_stream_read(&mstream->istream.istream) != -1) {
        pos = i_stream_get_data_size(&mstream->istream.istream);
        i_stream_skip(&mstream->istream.istream, pos);
    }
    return mstream->istream.istream.stream_errno != 0 ? -1 : 0;
}

static void
stream_reset_to(struct header_filter_istream *mstream, uoff_t v_offset)
{
    mstream->istream.istream.v_offset = v_offset;
    mstream->istream.skip = mstream->istream.pos = 0;
    mstream->istream.buffer = NULL;
    buffer_set_used_size(mstream->hdr_buf, 0);
}

static void i_stream_header_filter_seek(struct istream_private *stream,
                    uoff_t v_offset, bool mark ATTR_UNUSED)
{
    struct header_filter_istream *mstream =
        (struct header_filter_istream *)stream;

    if (stream->istream.v_offset == v_offset) {
        /* just reset the input buffer */
        stream_reset_to(mstream, v_offset);
        i_stream_seek(mstream->istream.parent,
                  mstream->istream.parent_expected_offset);
        return;
    }
    /* if last_lf_added=TRUE, we're currently at EOF. So reset it only if
       we're seeking backwards, otherwise we would just add a duplicate */
    mstream->last_lf_added = FALSE;

    if (v_offset == 0) {
        /* seeking to beginning of headers. */
        stream_reset_to(mstream, 0);
        i_stream_header_filter_seek_to_header(mstream, 0);
        return;
    }

    /* if we haven't parsed the whole header yet, we don't know if we
       want to seek inside header or body. so make sure we've parsed the
       header. */
    if (skip_header(mstream) < 0)
        return;
    stream_reset_to(mstream, v_offset);

    if (v_offset < mstream->header_size.virtual_size) {
        /* seek into headers. we'll have to re-parse them, use
           skip_count to set the wanted position */
        i_stream_header_filter_seek_to_header(mstream, v_offset);
    } else {
        /* body */
        v_offset += mstream->header_size.physical_size -
            mstream->header_size.virtual_size;
        i_stream_seek(stream->parent,
                  stream->parent_start_offset + v_offset);
    }
}

static void ATTR_NORETURN
i_stream_header_filter_sync(struct istream_private *stream ATTR_UNUSED)
{
    i_panic("istream-header-filter sync() not implemented");
}

static int
i_stream_header_filter_stat(struct istream_private *stream, bool exact)
{
    struct header_filter_istream *mstream =
        (struct header_filter_istream *)stream;
    const struct stat *st;
    uoff_t old_offset;

    if (i_stream_stat(stream->parent, exact, &st) < 0) {
        stream->istream.stream_errno = stream->parent->stream_errno;
        return -1;
    }
    stream->statbuf = *st;
    if (stream->statbuf.st_size == -1 || !exact)
        return 0;

    /* fix the filtered header size */
    old_offset = stream->istream.v_offset;
    if (skip_header(mstream) < 0)
        return -1;

    if (mstream->hide_body) {
        /* no body */
        stream->statbuf.st_size = mstream->header_size.physical_size;
    } else if (!mstream->end_body_with_lf) {
        /* no last-LF */
    } else if (mstream->last_lf_added) {
        /* yes, we have added LF */
        stream->statbuf.st_size += mstream->crlf ? 2 : 1;
    } else if (mstream->last_lf_offset != (uoff_t)-1) {
        /* no, we didn't need to add LF */
    } else {
        /* check if we need to add LF */
        i_stream_seek(stream->parent, st->st_size - 1);
        (void)i_stream_read(stream->parent);
        if (stream->parent->stream_errno != 0) {
            stream->istream.stream_errno =
                stream->parent->stream_errno;
            return -1;
        }
        i_assert(stream->parent->eof);
        ssize_t ret = handle_end_body_with_lf(mstream, -1);
        if (ret > 0)
            stream->statbuf.st_size += ret;
    }

    stream->statbuf.st_size -=
        (off_t)mstream->header_size.physical_size -
        (off_t)mstream->header_size.virtual_size;
    i_stream_seek(&stream->istream, old_offset);
    return 0;
}

#undef i_stream_create_header_filter
struct istream *
i_stream_create_header_filter(struct istream *input,
                              enum header_filter_flags flags,
                  const char *const *headers,
                  unsigned int headers_count,
                  header_filter_callback *callback, void *context)
{
    struct header_filter_istream *mstream;
    unsigned int i, j;
    int ret;

    i_assert((flags & (HEADER_FILTER_INCLUDE|HEADER_FILTER_EXCLUDE)) != 0);

    mstream = i_new(struct header_filter_istream, 1);
    mstream->pool = pool_alloconly_create(MEMPOOL_GROWING
                          "header filter stream", 4096);
    mstream->istream.max_buffer_size = input->real_stream->max_buffer_size;

    mstream->headers = headers_count == 0 ? NULL :
        p_new(mstream->pool, const char *, headers_count);
    for (i = j = 0; i < headers_count; i++)  {
        ret = j == 0 ? -1 :
            strcasecmp(mstream->headers[j-1], headers[i]);
        if (ret == 0) {
            /* drop duplicate */
            continue;
        }
        i_assert(ret < 0);
        mstream->headers[j++] = p_strdup(mstream->pool, headers[i]);
    }
    mstream->headers_count = j;
    mstream->hdr_buf = buffer_create_dynamic(mstream->pool, 1024);

    mstream->callback = callback;
    mstream->context = context;
    mstream->exclude = (flags & HEADER_FILTER_EXCLUDE) != 0;
    if ((flags & HEADER_FILTER_CRLF_PRESERVE) != 0)
        mstream->crlf_preserve = TRUE;
    else if ((flags & HEADER_FILTER_NO_CR) != 0)
        mstream->crlf = FALSE;
    else
        mstream->crlf = TRUE;
    mstream->hide_body = (flags & HEADER_FILTER_HIDE_BODY) != 0;
    mstream->add_missing_eoh = (flags & HEADER_FILTER_ADD_MISSING_EOH) != 0;
    mstream->end_body_with_lf =
        (flags & HEADER_FILTER_END_BODY_WITH_LF) != 0;
    mstream->last_lf_offset = (uoff_t)-1;

    mstream->istream.iostream.destroy = i_stream_header_filter_destroy;
    mstream->istream.read = i_stream_header_filter_read;
    mstream->istream.seek = i_stream_header_filter_seek;
    mstream->istream.sync = i_stream_header_filter_sync;
    mstream->istream.stat = i_stream_header_filter_stat;

    mstream->istream.istream.readable_fd = FALSE;
    mstream->istream.istream.blocking = input->blocking;
    mstream->istream.istream.seekable = input->seekable;

    return i_stream_create(&mstream->istream, input, -1);
}

void i_stream_header_filter_add(struct header_filter_istream *input,
                const void *data, size_t size)
{
    buffer_append(input->hdr_buf, data, size);
    input->headers_edited = TRUE;
}