src/lib-mail/message-header-parser.c

	message-header-parser.c revision 15362cdf9df29fef8795e865957e17ec027a9ebf
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (c) 2002-2008 Dovecot authors, see the included COPYING file */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "lib.h"
49e358eebea107aad9919dcc4bd88cee8519ba2eTimo Sirainen#include "buffer.h"
49e358eebea107aad9919dcc4bd88cee8519ba2eTimo Sirainen#include "istream.h"
49e358eebea107aad9919dcc4bd88cee8519ba2eTimo Sirainen#include "str.h"
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen#include "message-size.h"
dd62b77c932d1b518f2a3e4bf80e36542becc256Timo Sirainen#include "message-header-parser.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainenstruct message_header_parser_ctx {
03f5c621d06d6b6d77a145196c9633a7aa64dc78Timo Sirainen    struct message_header_line line;
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    struct istream *input;
411d6baa37f31d90730e90c4a28c43e1974bbe58Timo Sirainen    struct message_size *hdr_size;
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen    string_t *name;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    buffer_t *value_buf;
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen    size_t skip;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    enum message_header_parser_flags flags;
2526d52441ef368215ab6bf04fd0356d3b09d235Timo Sirainen    unsigned int skip_line:1;
2526d52441ef368215ab6bf04fd0356d3b09d235Timo Sirainen    unsigned int has_nuls:1;
fe363b433b8038a69b55169da9dca27892ad7d18Timo Sirainen};
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenstruct message_header_parser_ctx *
fe363b433b8038a69b55169da9dca27892ad7d18Timo Sirainenmessage_parse_header_init(struct istream *input, struct message_size *hdr_size,
fe363b433b8038a69b55169da9dca27892ad7d18Timo Sirainen              enum message_header_parser_flags flags)
dd62b77c932d1b518f2a3e4bf80e36542becc256Timo Sirainen{
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    struct message_header_parser_ctx *ctx;
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen    ctx = i_new(struct message_header_parser_ctx, 1);
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen    ctx->input = input;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    ctx->hdr_size = hdr_size;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    ctx->name = str_new(default_pool, 128);
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    ctx->flags = flags;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    if (hdr_size != NULL)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        memset(hdr_size, 0, sizeof(*hdr_size));
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    return ctx;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen}
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainenvoid message_parse_header_deinit(struct message_header_parser_ctx **_ctx)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen{
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    struct message_header_parser_ctx *ctx = *_ctx;
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainen
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainen    i_stream_skip(ctx->input, ctx->skip);
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainen    if (ctx->value_buf != NULL)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        buffer_free(&ctx->value_buf);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    str_free(&ctx->name);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    i_free(ctx);
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    *_ctx = NULL;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen}
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainenint message_parse_header_next(struct message_header_parser_ctx *ctx,
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen                  struct message_header_line **hdr_r)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen{
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen        struct message_header_line *line = &ctx->line;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    const unsigned char *msg;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    size_t i, size, startpos, colon_pos, parse_size;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    int ret;
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen    bool continued, continues, last_no_newline, last_crlf;
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen    bool no_newline, crlf_newline;
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen    *hdr_r = NULL;
602a0434db30d8e3292d1c161a803d96a879a74fTimo Sirainen    if (line->eoh)
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainen        return -1;
07e4875d250e7a7157cd99132aafc773cf3cdf83Timo Sirainen
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen    if (ctx->skip > 0) {
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen        i_stream_skip(ctx->input, ctx->skip);
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen        ctx->skip = 0;
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen    }
7d207b1e77a7b5e3fda640e353acfc86d261fedfTimo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen    if (line->continues) {
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen        if (line->use_full_value && !line->continued) {
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen            /* save the first line */
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen            if (ctx->value_buf != NULL)
89e195dfb5c4b0efd9b9f459771a4467674e5b1fTimo Sirainen                buffer_set_used_size(ctx->value_buf, 0);
08fb191d6148feb3ed14e2d6c625cd248dd1c1d4Timo Sirainen            else {
08fb191d6148feb3ed14e2d6c625cd248dd1c1d4Timo Sirainen                ctx->value_buf =
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen                    buffer_create_dynamic(default_pool,
89e195dfb5c4b0efd9b9f459771a4467674e5b1fTimo Sirainen                                  4096);
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen            }
89e195dfb5c4b0efd9b9f459771a4467674e5b1fTimo Sirainen            buffer_append(ctx->value_buf,
7e1f68ad71d3485f1882142837b01f7a98ca8467Timo Sirainen                      line->value, line->value_len);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        }
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen        colon_pos = 0;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    } else {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        /* new header line */
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen        line->name_offset = ctx->input->v_offset;
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen        colon_pos = UINT_MAX;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    }
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
65cb456a072219fa35b55695d476b0bf51e2d735Timo Sirainen    no_newline = FALSE;
65cb456a072219fa35b55695d476b0bf51e2d735Timo Sirainen    crlf_newline = FALSE;
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen    continued = line->continues;
c06f4017027263cf3a08becc551f5126409e2a83Timo Sirainen    continues = FALSE;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    for (startpos = 0;;) {
        ret = i_stream_read_data(ctx->input, &msg, &size, startpos+1);

        if (ret >= 0) {
            /* we want to know one byte in advance to find out
               if it's multiline header */
            parse_size = size == 0 ? 0 : size-1;
        } else {
            parse_size = size;
        }

        if (ret <= 0 && startpos == parse_size) {
            if (ret == -1) {
                if (startpos > 0) {
                    /* header ended unexpectedly. */
                    no_newline = TRUE;
                    ctx->skip = startpos;
                    break;
                }
                /* error / EOF with no bytes */
                return -1;
            }
            if (ret == 0 && !ctx->input->eof) {
                /* stream is nonblocking - need more data */
                return 0;
            }

            if (msg[0] == '\n' ||
                (msg[0] == '\r' && size > 1 && msg[1] == '\n')) {
                /* end of headers - this mostly happens just
                   with mbox where headers are read separately
                   from body */
                size = 0;
                if (ctx->hdr_size != NULL)
                    ctx->hdr_size->lines++;
                if (msg[0] == '\r') {
                    ctx->skip = 2;
                    crlf_newline = TRUE;
                } else {
                    ctx->skip = 1;
                    if (ctx->hdr_size != NULL)
                        ctx->hdr_size->virtual_size++;
                }
                break;
            }

            /* a) line is larger than input buffer
               b) header ended unexpectedly */
            if (colon_pos == UINT_MAX && ret == -2 && !continued) {
                /* header name is huge. just skip it. */
                if (msg[size-1] == '\r')
                    size--;

                if (ctx->hdr_size != NULL) {
                    ctx->hdr_size->physical_size += size;
                    ctx->hdr_size->virtual_size += size;
                }
                i_stream_skip(ctx->input, size);
                ctx->skip_line = TRUE;
                startpos = 0;
                continue;
            }

            if (ret == -2) {
                /* go back to last LWSP if found. */
                size_t min_pos = !continued ? colon_pos : 0;
                for (i = size-1; i > min_pos; i--) {
                    if (IS_LWSP(msg[i])) {
                        size = i;
                        break;
                    }
                }

                continues = TRUE;
            }
            no_newline = TRUE;
            ctx->skip = size;
            break;
        }

        /* find ':' */
        if (colon_pos == UINT_MAX) {
            for (i = startpos; i < parse_size; i++) {
                if (msg[i] > ':')
                    continue;

                if (msg[i] == ':') {
                    colon_pos = i;
                    line->full_value_offset =
                        ctx->input->v_offset + i + 1;
                    break;
                }
                if (msg[i] == '\n') {
                    /* end of headers, or error */
                    break;
                }

                if (msg[i] == '\0')
                    ctx->has_nuls = TRUE;
            }
        } else {
            i = startpos;
        }

        /* find '\n' */
        for (; i < parse_size; i++) {
            if (msg[i] <= '\n') {
                if (msg[i] == '\n')
                    break;
                if (msg[i] == '\0')
                    ctx->has_nuls = TRUE;
            }
        }

        if (i < parse_size) {
            /* got a line */
            if (ctx->skip_line) {
                /* skipping a huge line */
                if (ctx->hdr_size != NULL) {
                    ctx->hdr_size->physical_size += i;
                    ctx->hdr_size->virtual_size += i;
                }

                if (i == 0 || msg[i-1] != '\r') {
                    /* missing CR */
                    if (ctx->hdr_size != NULL)
                        ctx->hdr_size->virtual_size++;
                } else {
                    crlf_newline = TRUE;
                }
                i_stream_skip(ctx->input, i);
                startpos = 0;
                ctx->skip_line = FALSE;
                continue;
            }
            continues = i+1 < size && IS_LWSP(msg[i+1]);

            if (ctx->hdr_size != NULL)
                ctx->hdr_size->lines++;
            if (i == 0 || msg[i-1] != '\r') {
                /* missing CR */
                if (ctx->hdr_size != NULL)
                    ctx->hdr_size->virtual_size++;
                size = i;
            } else {
                size = i-1;
                crlf_newline = TRUE;
            }

            ctx->skip = i+1;
            break;
        }

        startpos = i;
    }

    last_crlf = line->crlf_newline &&
        (ctx->flags & MESSAGE_HEADER_PARSER_FLAG_DROP_CR) == 0;
    last_no_newline = line->no_newline ||
        (ctx->flags & MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE) != 0;

    line->continues = continues;
    line->continued = continued;
    line->crlf_newline = crlf_newline;
    line->no_newline = no_newline;
    if (size == 0) {
        /* end of headers */
        line->eoh = TRUE;
        line->name_len = line->value_len = line->full_value_len = 0;
        line->name = ""; line->value = line->full_value = NULL;
        line->middle = NULL; line->middle_len = 0;
    } else if (line->continued) {
        line->value = msg;
        line->value_len = size;
    } else if (colon_pos == UINT_MAX) {
        /* missing ':', assume the whole line is name */
        line->value = NULL;
        line->value_len = 0;

        str_truncate(ctx->name, 0);
        str_append_n(ctx->name, msg, size);
        line->name = str_c(ctx->name);
        line->name_len = str_len(ctx->name);

        line->middle = NULL;
        line->middle_len = 0;
    } else {
        size_t pos;

        line->value = msg + colon_pos+1;
        line->value_len = size - colon_pos - 1;
        if (ctx->flags & MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP) {
            /* get value. skip all LWSP after ':'. Note that
               RFC2822 doesn't say we should, but history behind
               it..

               Exception to this is if the value consists only of
               LWSP, then skip only the one LWSP after ':'. */
            for (pos = 0; pos < line->value_len; pos++) {
                if (!IS_LWSP(line->value[0]))
                    break;
            }

            if (pos == line->value_len) {
                /* everything was LWSP */
                if (line->value_len > 0 &&
                    IS_LWSP(line->value[0]))
                    pos = 1;
            }
        } else {
            pos = line->value_len > 0 &&
                IS_LWSP(line->value[0]) ? 1 : 0;
        }

        line->value += pos;
        line->value_len -= pos;
        line->full_value_offset += pos;

        /* get name, skip LWSP before ':' */
        while (colon_pos > 0 && IS_LWSP(msg[colon_pos-1]))
            colon_pos--;

        str_truncate(ctx->name, 0);
        str_append_n(ctx->name, msg, colon_pos);
        str_append_c(ctx->name, '\0');

        /* keep middle stored also in ctx->name so it's available
           with use_full_value */
        line->middle = msg + colon_pos;
        line->middle_len = (size_t)(line->value - line->middle);
        str_append_n(ctx->name, line->middle, line->middle_len);

        line->name = str_c(ctx->name);
        line->name_len = colon_pos;
        line->middle = str_data(ctx->name) + line->name_len + 1;
    }

    if (!line->continued) {
        /* first header line, set full_value = value */
        line->full_value = line->value;
        line->full_value_len = line->value_len;
    } else if (line->use_full_value) {
        /* continue saving the full value. */
        if (last_no_newline) {
            /* line is longer than fit into our buffer, so we
               were forced to break it into multiple
               message_header_lines */
        } else {
            if (last_crlf)
                buffer_append_c(ctx->value_buf, '\r');
            buffer_append_c(ctx->value_buf, '\n');
        }
        if ((ctx->flags & MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE) &&
            line->value_len > 0 && line->value[0] != ' ' &&
            IS_LWSP(line->value[0])) {
            buffer_append_c(ctx->value_buf, ' ');
            buffer_append(ctx->value_buf,
                      line->value + 1, line->value_len - 1);
        } else {
            buffer_append(ctx->value_buf,
                      line->value, line->value_len);
        }
        line->full_value = buffer_get_data(ctx->value_buf,
                           &line->full_value_len);
    } else {
        /* we didn't want full_value, and this is a continued line. */
        line->full_value = NULL;
        line->full_value_len = 0;
    }

    /* always reset it */
    line->use_full_value = FALSE;

    if (ctx->hdr_size != NULL) {
        ctx->hdr_size->physical_size += ctx->skip;
        ctx->hdr_size->virtual_size += ctx->skip;
    }

    *hdr_r = line;
    return 1;
}

bool message_parse_header_has_nuls(struct message_header_parser_ctx *ctx)
{
    return ctx->has_nuls;
}

#undef message_parse_header
void message_parse_header(struct istream *input, struct message_size *hdr_size,
              enum message_header_parser_flags flags,
              message_header_callback_t *callback, void *context)
{
    struct message_header_parser_ctx *hdr_ctx;
    struct message_header_line *hdr;
    int ret;

    hdr_ctx = message_parse_header_init(input, hdr_size, flags);
    while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0)
        callback(hdr, context);
    i_assert(ret != 0);
    message_parse_header_deinit(&hdr_ctx);

    /* call after the final skipping */
    callback(NULL, context);
}