bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen#include "lib.h"
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen#include "istream.h"
16f816d3f3c32ae3351834253f52ddd0212bcbf3Timo Sirainen#include "array.h"
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen#include "buffer.h"
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen#include "str.h"
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen#include "message-date.h"
57e3b63a75335f45cf6cf9cd89317e2e6cec249dStephan Bosch#include "message-part-data.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "message-parser.h"
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen#include "message-header-decode.h"
e54512a5189192fe72d1e2c53927c98c5ac920b4Timo Sirainen#include "istream-tee.h"
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include "istream-header-filter.h"
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen#include "imap-envelope.h"
645f258ea29afaf09b673fc65d1bd788dfec8db8Timo Sirainen#include "imap-bodystructure.h"
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen#include "index-storage.h"
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen#include "index-mail.h"
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainenstatic const enum message_header_parser_flags hdr_parser_flags =
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP |
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen MESSAGE_HEADER_PARSER_FLAG_DROP_CR;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainenstatic const enum message_parser_flags msg_parser_flags =
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen MESSAGE_PARSER_FLAG_SKIP_BODY_BLOCK;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainenstatic int header_line_cmp(const struct index_mail_line *l1,
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen const struct index_mail_line *l2)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen int diff;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen diff = (int)l1->field_idx - (int)l2->field_idx;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return diff != 0 ? diff :
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen (int)l1->line_num - (int)l2->line_num;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen}
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic void index_mail_parse_header_finish(struct index_mail *mail)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen struct mail *_mail = &mail->mail.mail;
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen const struct index_mail_line *lines;
b88c43d09a288e99d439c78de4cc613212ea924cTimo Sirainen const unsigned char *header;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen const uint8_t *match;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_t *buf;
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen unsigned int i, j, count, match_idx, match_count;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen bool noncontiguous;
8fb1e3e2349c9940732b5bb77a2a4053b8f72a4fTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* sort it first so fields are grouped together and ordered by
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen line number */
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen array_sort(&mail->header_lines, header_line_cmp);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen lines = array_get(&mail->header_lines, &count);
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen match = array_get(&mail->header_match, &match_count);
b88c43d09a288e99d439c78de4cc613212ea924cTimo Sirainen header = mail->header_data->data;
c147bff818798a979d93537f72f5c1f68f5d5ba8Aki Tuomi buf = t_buffer_create(256);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* go through all the header lines we found */
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen for (i = match_idx = 0; i < count; i = j) {
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* matches and header lines are both sorted, all matches
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen until lines[i] weren't found */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen while (match_idx < lines[i].field_idx &&
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen match_idx < match_count) {
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen if (HEADER_MATCH_USABLE(mail, match[match_idx]) &&
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen mail_cache_field_can_add(_mail->transaction->cache_trans,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen _mail->seq, match_idx)) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* this header doesn't exist. remember that. */
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen i_assert((match[match_idx] &
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen HEADER_MATCH_FLAG_FOUND) == 0);
5cda0bfea032000c4a51134c748d9efe6614870bTimo Sirainen index_mail_cache_add_idx(mail, match_idx,
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen "", 0);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen match_idx++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen if (match_idx < match_count) {
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* save index to first header line */
8b12e7b44abca3bd51a1c46e19ca504f3b55e723Timo Sirainen i_assert(match_idx == lines[i].field_idx);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen j = i + 1;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen array_idx_set(&mail->header_match_lines, match_idx, &j);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen match_idx++;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen }
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen if (!mail_cache_field_can_add(_mail->transaction->cache_trans,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen _mail->seq, lines[i].field_idx)) {
8b12e7b44abca3bd51a1c46e19ca504f3b55e723Timo Sirainen /* header is already cached. skip over all the
8b12e7b44abca3bd51a1c46e19ca504f3b55e723Timo Sirainen header lines. */
8b12e7b44abca3bd51a1c46e19ca504f3b55e723Timo Sirainen for (j = i+1; j < count; j++) {
8b12e7b44abca3bd51a1c46e19ca504f3b55e723Timo Sirainen if (lines[j].field_idx != lines[i].field_idx)
8b12e7b44abca3bd51a1c46e19ca504f3b55e723Timo Sirainen break;
8b12e7b44abca3bd51a1c46e19ca504f3b55e723Timo Sirainen }
872b8fd8a8db97dc54067b7ab25bda96ec0aac0dTimo Sirainen continue;
872b8fd8a8db97dc54067b7ab25bda96ec0aac0dTimo Sirainen }
872b8fd8a8db97dc54067b7ab25bda96ec0aac0dTimo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* buffer contains: { uint32_t line_num[], 0, header texts }
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen noncontiguous is just a small optimization.. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_set_used_size(buf, 0);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_append(buf, &lines[i].line_num,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen sizeof(lines[i].line_num));
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen noncontiguous = FALSE;
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen for (j = i+1; j < count; j++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (lines[j].field_idx != lines[i].field_idx)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen break;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (lines[j].start_pos != lines[j-1].end_pos)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen noncontiguous = TRUE;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_append(buf, &lines[j].line_num,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen sizeof(lines[j].line_num));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen buffer_append_zero(buf, sizeof(uint32_t));
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (noncontiguous) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (; i < j; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_append(buf, header + lines[i].start_pos,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen lines[i].end_pos -
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen lines[i].start_pos);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen i--;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_append(buf, header + lines[i].start_pos,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen lines[j-1].end_pos - lines[i].start_pos);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
5cda0bfea032000c4a51134c748d9efe6614870bTimo Sirainen index_mail_cache_add_idx(mail, lines[i].field_idx,
b88c43d09a288e99d439c78de4cc613212ea924cTimo Sirainen buf->data, buf->used);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen for (; match_idx < match_count; match_idx++) {
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen if (HEADER_MATCH_USABLE(mail, match[match_idx]) &&
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen mail_cache_field_can_add(_mail->transaction->cache_trans,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen _mail->seq, match_idx)) {
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen /* this header doesn't exist. remember that. */
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen i_assert((match[match_idx] &
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen HEADER_MATCH_FLAG_FOUND) == 0);
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen index_mail_cache_add_idx(mail, match_idx, "", 0);
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen }
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen }
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen
8ae72ad7d0c69e972cfa65d1e2ce4e3e9a8b765cTimo Sirainen mail->data.dont_cache_field_idx = UINT_MAX;
70612e07102b75a8511aa7f9de60771176b18de0Timo Sirainen mail->data.header_parser_initialized = FALSE;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen}
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainenstatic unsigned int
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainenget_header_field_idx(struct mailbox *box, const char *field,
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen enum mail_cache_decision_type decision)
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainen{
5df33e9ee65eec194105b338c55dedbf8422f695Timo Sirainen struct mail_cache_field header_field;
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&header_field);
5df33e9ee65eec194105b338c55dedbf8422f695Timo Sirainen header_field.type = MAIL_CACHE_FIELD_HEADER;
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen header_field.decision = decision;
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen T_BEGIN {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen header_field.name = t_strconcat("hdr.", field, NULL);
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen mail_cache_register_fields(box->cache, &header_field, 1);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen } T_END;
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainen return header_field.idx;
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainen}
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainen
77af0bd168cf3e3ddc3ae68abc82bfad7e9b5ff4Timo Sirainenbool index_mail_want_parse_headers(struct index_mail *mail)
77af0bd168cf3e3ddc3ae68abc82bfad7e9b5ff4Timo Sirainen{
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen if (mail->data.wanted_headers != NULL ||
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen mail->data.save_bodystructure_header)
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen return TRUE;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen if ((mail->data.cache_fetch_fields & MAIL_FETCH_DATE) != 0 &&
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen !mail->data.sent_date_parsed)
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen return TRUE;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen return FALSE;
77af0bd168cf3e3ddc3ae68abc82bfad7e9b5ff4Timo Sirainen}
77af0bd168cf3e3ddc3ae68abc82bfad7e9b5ff4Timo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainenstatic void index_mail_parse_header_register_all_wanted(struct index_mail *mail)
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen{
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen struct mail *_mail = &mail->mail.mail;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen const struct mail_cache_field *all_cache_fields;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen unsigned int i, count;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen all_cache_fields =
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen mail_cache_register_get_list(_mail->box->cache,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen pool_datastack_create(), &count);
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen for (i = 0; i < count; i++) {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen if (strncasecmp(all_cache_fields[i].name, "hdr.", 4) != 0)
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen continue;
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen if (!mail_cache_field_want_add(_mail->transaction->cache_trans,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen _mail->seq, i))
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen continue;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen array_idx_set(&mail->header_match, all_cache_fields[i].idx,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen &mail->header_match_value);
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen }
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen}
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenvoid index_mail_parse_header_init(struct index_mail *mail,
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen struct mailbox_header_lookup_ctx *headers)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen struct index_mail_data *data = &mail->data;
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen const uint8_t *match;
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen unsigned int i, field_idx, match_count;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
093d5c0cdf04c9190b6377e624206cdf85fe665aTimo Sirainen i_assert(!mail->data.header_parser_initialized);
093d5c0cdf04c9190b6377e624206cdf85fe665aTimo Sirainen
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen mail->header_seq = data->seq;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (mail->header_data == NULL) {
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen mail->header_data = buffer_create_dynamic(default_pool, 4096);
91dca97b367c54a139c268b56a0c67f564bd9197Timo Sirainen i_array_init(&mail->header_lines, 32);
91dca97b367c54a139c268b56a0c67f564bd9197Timo Sirainen i_array_init(&mail->header_match, 32);
91dca97b367c54a139c268b56a0c67f564bd9197Timo Sirainen i_array_init(&mail->header_match_lines, 32);
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen mail->header_match_value = HEADER_MATCH_SKIP_COUNT;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_set_used_size(mail->header_data, 0);
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen array_clear(&mail->header_lines);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen array_clear(&mail->header_match_lines);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen mail->header_match_value += HEADER_MATCH_SKIP_COUNT;
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen i_assert((mail->header_match_value &
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen (HEADER_MATCH_SKIP_COUNT-1)) == 0);
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen if (mail->header_match_value == 0) {
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen /* wrapped, we'll have to clear the buffer */
1952eb389b8aba39195380970f905dcebea38dfcTimo Sirainen array_clear(&mail->header_match);
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen mail->header_match_value = HEADER_MATCH_SKIP_COUNT;
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (headers != NULL) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (i = 0; i < headers->count; i++) {
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen array_idx_set(&mail->header_match, headers->idx[i],
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen &mail->header_match_value);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen if (data->wanted_headers != NULL && data->wanted_headers != headers) {
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen headers = data->wanted_headers;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (i = 0; i < headers->count; i++) {
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen array_idx_set(&mail->header_match, headers->idx[i],
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen &mail->header_match_value);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainen
45b2a27617d8475f71fdfc870690e46cd63849f2Timo Sirainen /* register also all the other headers that exist in cache file */
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen T_BEGIN {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen index_mail_parse_header_register_all_wanted(mail);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen } T_END;
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen /* if we want sent date, it doesn't mean that we also want to cache
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen Date: header. if we have Date field's index set at this point we
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen know that we want it. otherwise add it and remember that we don't
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen want it cached. */
d22301419109ed4a38351715e6760011421dadecTimo Sirainen field_idx = get_header_field_idx(mail->mail.mail.box, "Date",
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen MAIL_CACHE_DECISION_NO);
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen match = array_get(&mail->header_match, &match_count);
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen if (field_idx < match_count &&
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen match[field_idx] == mail->header_match_value) {
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen /* cache Date: header */
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen } else if ((data->cache_fetch_fields & MAIL_FETCH_DATE) != 0 ||
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen data->save_sent_date) {
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen /* parse Date: header, but don't cache it. */
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen data->dont_cache_field_idx = field_idx;
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen array_idx_set(&mail->header_match, field_idx,
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen &mail->header_match_value);
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen }
70612e07102b75a8511aa7f9de60771176b18de0Timo Sirainen mail->data.header_parser_initialized = TRUE;
70612e07102b75a8511aa7f9de60771176b18de0Timo Sirainen mail->data.parse_line_num = 0;
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&mail->data.parse_line);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen}
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic void index_mail_parse_finish_imap_envelope(struct index_mail *mail)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen struct mail *_mail = &mail->mail.mail;
469a6cf705835ade57dcb59979b6e054207ae5d7Timo Sirainen const unsigned int cache_field_envelope =
469a6cf705835ade57dcb59979b6e054207ae5d7Timo Sirainen mail->ibox->cache_fields[MAIL_CACHE_IMAP_ENVELOPE].idx;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen string_t *str;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen str = str_new(mail->mail.data_pool, 256);
b674bd911aaab7e8b1a77c106a0b5bccb603439fStephan Bosch imap_envelope_write(mail->data.envelope_data, str);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail->data.envelope = str_c(str);
644268f7848a7c4221146d0b11feb8ed5bbed233Timo Sirainen
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen if (mail_cache_field_can_add(_mail->transaction->cache_trans,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen _mail->seq, cache_field_envelope)) {
469a6cf705835ade57dcb59979b6e054207ae5d7Timo Sirainen index_mail_cache_add_idx(mail, cache_field_envelope,
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen str_data(str), str_len(str));
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen}
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
d2b94d25f842cd1b7acaf4dd7de858f7c6a821c9Timo Sirainenvoid index_mail_parse_header(struct message_part *part,
d2b94d25f842cd1b7acaf4dd7de858f7c6a821c9Timo Sirainen struct message_header_line *hdr,
d2b94d25f842cd1b7acaf4dd7de858f7c6a821c9Timo Sirainen struct index_mail *mail)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
d22301419109ed4a38351715e6760011421dadecTimo Sirainen struct mail *_mail = &mail->mail.mail;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen struct index_mail_data *data = &mail->data;
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen unsigned int field_idx, count;
38d7db318188c4ac9cdc8c6cdb936b36a5258e19Timo Sirainen uint8_t *match;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
70612e07102b75a8511aa7f9de60771176b18de0Timo Sirainen i_assert(data->header_parser_initialized);
70612e07102b75a8511aa7f9de60771176b18de0Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->parse_line_num++;
645f258ea29afaf09b673fc65d1bd788dfec8db8Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->save_bodystructure_header) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen i_assert(part != NULL);
df459621b9124dfd88d56619ac84611f30fec854Stephan Bosch message_part_data_parse_from_header(mail->mail.data_pool, part, hdr);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
645f258ea29afaf09b673fc65d1bd788dfec8db8Timo Sirainen
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen if (data->save_envelope) {
946f22af116d5af80d5bbe1710ac121aa5acef71Stephan Bosch message_part_envelope_parse_from_header(mail->mail.data_pool,
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen &data->envelope_data, hdr);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (hdr == NULL)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_mail_parse_finish_imap_envelope(mail);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen if (hdr == NULL) {
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen /* end of headers */
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen if (mail->data.save_sent_date)
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen mail->data.sent_date_parsed = TRUE;
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen T_BEGIN {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen index_mail_parse_header_finish(mail);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen } T_END;
eed20b28dd9039d21f5c2770beef2e8b19f7c2f9Timo Sirainen if (data->save_bodystructure_header) {
eed20b28dd9039d21f5c2770beef2e8b19f7c2f9Timo Sirainen i_assert(!data->save_bodystructure_body ||
eed20b28dd9039d21f5c2770beef2e8b19f7c2f9Timo Sirainen data->parser_ctx != NULL);
eed20b28dd9039d21f5c2770beef2e8b19f7c2f9Timo Sirainen data->save_bodystructure_header = FALSE;
eed20b28dd9039d21f5c2770beef2e8b19f7c2f9Timo Sirainen }
d2b94d25f842cd1b7acaf4dd7de858f7c6a821c9Timo Sirainen return;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (!hdr->continued) {
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen T_BEGIN {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen const char *cache_field_name =
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen t_strconcat("hdr.", hdr->name, NULL);
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen data->parse_line.field_idx =
d22301419109ed4a38351715e6760011421dadecTimo Sirainen mail_cache_register_lookup(_mail->box->cache,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen cache_field_name);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen } T_END;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen field_idx = data->parse_line.field_idx;
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen match = array_get_modifiable(&mail->header_match, &count);
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen if (field_idx >= count ||
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen !HEADER_MATCH_USABLE(mail, match[field_idx])) {
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen /* we don't want this header. */
d2b94d25f842cd1b7acaf4dd7de858f7c6a821c9Timo Sirainen return;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (!hdr->continued) {
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen /* beginning of a line. add the header name. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->parse_line.start_pos = str_len(mail->header_data);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->parse_line.line_num = data->parse_line_num;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen str_append(mail->header_data, hdr->name);
60576cd64e6a537413cd90104f7e862f71d48c81Timo Sirainen str_append_n(mail->header_data, hdr->middle, hdr->middle_len);
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen /* remember that we saw this header so we don't add it to
7fc0f80480063a9d4cb9e8c07b50db2a5627799eTimo Sirainen cache as nonexistent. */
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen match[field_idx] |= HEADER_MATCH_FLAG_FOUND;
203560029e3ad8687c2c759e6a81ecdb8b37ebe6Timo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen str_append_n(mail->header_data, hdr->value, hdr->value_len);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (!hdr->no_newline)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen str_append(mail->header_data, "\n");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (!hdr->continues) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->parse_line.end_pos = str_len(mail->header_data);
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen array_append(&mail->header_lines, &data->parse_line, 1);
644268f7848a7c4221146d0b11feb8ed5bbed233Timo Sirainen }
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen}
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic void
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenindex_mail_parse_part_header_cb(struct message_part *part,
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen struct message_header_line *hdr,
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen struct index_mail *mail)
80cdfe47daf6129410bafcecbe5c9faf09f2721bTimo Sirainen{
d2b94d25f842cd1b7acaf4dd7de858f7c6a821c9Timo Sirainen index_mail_parse_header(part, hdr, mail);
80cdfe47daf6129410bafcecbe5c9faf09f2721bTimo Sirainen}
80cdfe47daf6129410bafcecbe5c9faf09f2721bTimo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstatic void
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainenindex_mail_parse_header_cb(struct message_header_line *hdr,
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen struct index_mail *mail)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen index_mail_parse_header(mail->data.parts, hdr, mail);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
7a54d58280aad8a64f266c61273ea1e8dff511a3Timo Sirainenstruct istream *
7a54d58280aad8a64f266c61273ea1e8dff511a3Timo Sirainenindex_mail_cache_parse_init(struct mail *_mail, struct istream *input)
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(_mail);
e54512a5189192fe72d1e2c53927c98c5ac920b4Timo Sirainen struct istream *input2;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen
6f6f3dc5b33b09097192124bce17b7017d6bbfcfTimo Sirainen i_assert(mail->data.tee_stream == NULL);
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen i_assert(mail->data.parser_ctx == NULL);
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen
4d0d535efdfc4aad3bd48b74adfafecf58094e0aTimo Sirainen /* we're doing everything for now, figure out later if we want to
4d0d535efdfc4aad3bd48b74adfafecf58094e0aTimo Sirainen save them. */
4d0d535efdfc4aad3bd48b74adfafecf58094e0aTimo Sirainen mail->data.save_sent_date = TRUE;
4d0d535efdfc4aad3bd48b74adfafecf58094e0aTimo Sirainen mail->data.save_bodystructure_header = TRUE;
4d0d535efdfc4aad3bd48b74adfafecf58094e0aTimo Sirainen mail->data.save_bodystructure_body = TRUE;
e78c6f8ecf4927dd579df8bcfd600b96e3f3a2f0Josef 'Jeff' Sipek /* Don't unnecessarily waste time generating a snippet, since it's
e78c6f8ecf4927dd579df8bcfd600b96e3f3a2f0Josef 'Jeff' Sipek not as cheap as the others to generate. */
e78c6f8ecf4927dd579df8bcfd600b96e3f3a2f0Josef 'Jeff' Sipek if (index_mail_want_cache(mail, MAIL_CACHE_BODY_SNIPPET))
e78c6f8ecf4927dd579df8bcfd600b96e3f3a2f0Josef 'Jeff' Sipek mail->data.save_body_snippet = TRUE;
4d0d535efdfc4aad3bd48b74adfafecf58094e0aTimo Sirainen
6f6f3dc5b33b09097192124bce17b7017d6bbfcfTimo Sirainen mail->data.tee_stream = tee_i_stream_create(input);
6f6f3dc5b33b09097192124bce17b7017d6bbfcfTimo Sirainen input = tee_i_stream_create_child(mail->data.tee_stream);
6f6f3dc5b33b09097192124bce17b7017d6bbfcfTimo Sirainen input2 = tee_i_stream_create_child(mail->data.tee_stream);
e54512a5189192fe72d1e2c53927c98c5ac920b4Timo Sirainen
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen index_mail_parse_header_init(mail, NULL);
49be238e250e99af8c69321264a461d8f6ceef62Timo Sirainen mail->data.parser_input = input;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen mail->data.parser_ctx =
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen message_parser_init(mail->mail.data_pool, input,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen hdr_parser_flags, msg_parser_flags);
e54512a5189192fe72d1e2c53927c98c5ac920b4Timo Sirainen i_stream_unref(&input);
e54512a5189192fe72d1e2c53927c98c5ac920b4Timo Sirainen return input2;
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen}
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainenstatic void index_mail_init_parser(struct index_mail *mail)
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen{
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen struct index_mail_data *data = &mail->data;
992118a50af940482b6cf884a89be56d7015580aTimo Sirainen struct message_part *parts;
8b5b1f6cb19253dfd7821fcef8e9b7e95e6caf3aTimo Sirainen const char *error;
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen
49be238e250e99af8c69321264a461d8f6ceef62Timo Sirainen if (data->parser_ctx != NULL) {
49be238e250e99af8c69321264a461d8f6ceef62Timo Sirainen data->parser_input = NULL;
8b5b1f6cb19253dfd7821fcef8e9b7e95e6caf3aTimo Sirainen if (message_parser_deinit_from_parts(&data->parser_ctx, &parts, &error) < 0) {
489301ee88b2174e3171875e979e667de2c4a174Timo Sirainen index_mail_set_message_parts_corrupted(&mail->mail.mail, error);
8b5b1f6cb19253dfd7821fcef8e9b7e95e6caf3aTimo Sirainen data->parts = NULL;
8b5b1f6cb19253dfd7821fcef8e9b7e95e6caf3aTimo Sirainen }
49be238e250e99af8c69321264a461d8f6ceef62Timo Sirainen }
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen if (data->parts == NULL) {
49be238e250e99af8c69321264a461d8f6ceef62Timo Sirainen data->parser_input = data->stream;
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen data->parser_ctx = message_parser_init(mail->mail.data_pool,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen data->stream,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen hdr_parser_flags,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen msg_parser_flags);
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen } else {
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen data->parser_ctx =
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen message_parser_init_from_parts(data->parts,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen data->stream,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen hdr_parser_flags,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen msg_parser_flags);
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen }
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen}
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainenint index_mail_parse_headers(struct index_mail *mail,
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen struct mailbox_header_lookup_ctx *headers,
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen const char *reason)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen struct index_mail_data *data = &mail->data;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen struct istream *input;
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen uoff_t old_offset;
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen if (mail_get_hdr_stream_because(&mail->mail.mail, NULL, reason, &input) < 0)
a24b0595f0f7d3925d4c9ac26fa503ff87c43e43Timo Sirainen return -1;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
2fa3e62e491621ca7a25211ecc6746c5823cacdeAki Tuomi i_assert(data->stream != NULL);
2fa3e62e491621ca7a25211ecc6746c5823cacdeAki Tuomi
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen index_mail_parse_header_init(mail, headers);
645f258ea29afaf09b673fc65d1bd788dfec8db8Timo Sirainen
1045a1d4c191a14867cde0d5cea9e4ac5e36f85fTimo Sirainen if (data->parts == NULL || data->save_bodystructure_header) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* initialize bodystructure parsing in case we read the whole
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen message. */
dd0dea1fdd913a04bae16e82dd66d67571a5f6c2Timo Sirainen index_mail_init_parser(mail);
96541d31299bb40b5a6efdbf9b4cb3d4f4b4a069Timo Sirainen message_parser_parse_header(data->parser_ctx, &data->hdr_size,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen index_mail_parse_part_header_cb,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen mail);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* just read the header */
eed20b28dd9039d21f5c2770beef2e8b19f7c2f9Timo Sirainen i_assert(!data->save_bodystructure_body ||
eed20b28dd9039d21f5c2770beef2e8b19f7c2f9Timo Sirainen data->parser_ctx != NULL);
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen message_parse_header(data->stream, &data->hdr_size,
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen hdr_parser_flags,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_mail_parse_header_cb, mail);
645f258ea29afaf09b673fc65d1bd788dfec8db8Timo Sirainen }
50de46721446795c42943c572625f2f1a9abfe01Timo Sirainen if (index_mail_stream_check_failure(mail) < 0)
50de46721446795c42943c572625f2f1a9abfe01Timo Sirainen return -1;
96541d31299bb40b5a6efdbf9b4cb3d4f4b4a069Timo Sirainen data->hdr_size_set = TRUE;
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainen data->access_part &= ~PARSE_HDR;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen i_stream_seek(data->stream, old_offset);
a24b0595f0f7d3925d4c9ac26fa503ff87c43e43Timo Sirainen return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
645f258ea29afaf09b673fc65d1bd788dfec8db8Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic void
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainenimap_envelope_parse_callback(struct message_header_line *hdr,
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen struct index_mail *mail)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
946f22af116d5af80d5bbe1710ac121aa5acef71Stephan Bosch message_part_envelope_parse_from_header(mail->mail.data_pool,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen &mail->data.envelope_data, hdr);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (hdr == NULL)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen index_mail_parse_finish_imap_envelope(mail);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenint index_mail_headers_get_envelope(struct index_mail *mail)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
3889d05019a072a602f7a8c1eeb8a6f1c1362720Timo Sirainen const unsigned int cache_field_envelope =
3889d05019a072a602f7a8c1eeb8a6f1c1362720Timo Sirainen mail->ibox->cache_fields[MAIL_CACHE_IMAP_ENVELOPE].idx;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct mailbox_header_lookup_ctx *header_ctx;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct istream *stream;
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen uoff_t old_offset;
3889d05019a072a602f7a8c1eeb8a6f1c1362720Timo Sirainen string_t *str;
3889d05019a072a602f7a8c1eeb8a6f1c1362720Timo Sirainen
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen str = str_new(mail->mail.data_pool, 256);
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen if (index_mail_cache_lookup_field(mail, str,
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen cache_field_envelope) > 0) {
3889d05019a072a602f7a8c1eeb8a6f1c1362720Timo Sirainen mail->data.envelope = str_c(str);
3889d05019a072a602f7a8c1eeb8a6f1c1362720Timo Sirainen return 0;
3889d05019a072a602f7a8c1eeb8a6f1c1362720Timo Sirainen }
3889d05019a072a602f7a8c1eeb8a6f1c1362720Timo Sirainen str_free(&str);
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen old_offset = mail->data.stream == NULL ? 0 :
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen mail->data.stream->v_offset;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
36977c4a74e164f7d81eb4785f0a5d3ff436fd19Timo Sirainen mail->data.save_envelope = TRUE;
d22301419109ed4a38351715e6760011421dadecTimo Sirainen header_ctx = mailbox_header_lookup_init(mail->mail.mail.box,
042a58438dc324c6d4de18e4c8a68044d6328e66Stephan Bosch message_part_envelope_headers);
00db1d630a723113609598e28acbae4d416e0cb4Timo Sirainen if (mail_get_header_stream(&mail->mail.mail, header_ctx, &stream) < 0) {
f95b3d29bc56f139c18c880aa574a0ca72b0cffbTimo Sirainen mailbox_header_lookup_unref(&header_ctx);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return -1;
00db1d630a723113609598e28acbae4d416e0cb4Timo Sirainen }
50de46721446795c42943c572625f2f1a9abfe01Timo Sirainen mailbox_header_lookup_unref(&header_ctx);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
9db263f2b9ab771fbf9a2bff44a245c45eaef218Timo Sirainen if (mail->data.envelope == NULL && stream != NULL) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* we got the headers from cache - parse them to get the
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen envelope */
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen message_parse_header(stream, NULL, hdr_parser_flags,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen imap_envelope_parse_callback, mail);
045e41df5a72f7726aca03f6687cd79e7281fb4bTimo Sirainen if (stream->stream_errno != 0) {
d052dcfff0c96a0af17a3158e51f709edf4b93a1Timo Sirainen index_mail_stream_log_failure_for(mail, stream);
50de46721446795c42943c572625f2f1a9abfe01Timo Sirainen return -1;
045e41df5a72f7726aca03f6687cd79e7281fb4bTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail->data.save_envelope = FALSE;
645f258ea29afaf09b673fc65d1bd788dfec8db8Timo Sirainen }
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen if (mail->data.stream != NULL)
20195ef995a4eb63a282283db63f1dc0605323e0Timo Sirainen i_stream_seek(mail->data.stream, old_offset);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
645f258ea29afaf09b673fc65d1bd788dfec8db8Timo Sirainen
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainenstatic size_t get_header_size(buffer_t *buffer, size_t pos)
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen{
b88c43d09a288e99d439c78de4cc613212ea924cTimo Sirainen const unsigned char *data = buffer->data;
b88c43d09a288e99d439c78de4cc613212ea924cTimo Sirainen size_t i, size = buffer->used;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen i_assert(pos <= size);
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen for (i = pos; i < size; i++) {
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen if (data[i] == '\n') {
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen if (i+1 == size ||
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen (data[i+1] != ' ' && data[i+1] != '\t'))
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen return i - pos;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen }
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen }
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen return size - pos;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen}
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainenstatic int index_mail_header_is_parsed(struct index_mail *mail,
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen unsigned int field_idx)
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen{
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen const uint8_t *match;
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen unsigned int count;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen match = array_get(&mail->header_match, &count);
86ad841251a38aa9ffcf4db4ee2c9fd449121bcbTimo Sirainen if (field_idx < count && HEADER_MATCH_USABLE(mail, match[field_idx]))
f0339f522dc9c8e2e8a29ef9a3f937c431c6bd1bTimo Sirainen return (match[field_idx] & HEADER_MATCH_FLAG_FOUND) != 0 ? 1 : 0;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen return -1;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen}
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenstatic bool skip_header(const unsigned char **data, size_t len)
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen{
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen const unsigned char *p = *data;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen size_t i;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen for (i = 0; i < len; i++) {
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen if (p[i] == ':')
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen break;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen }
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen if (i == len)
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen return FALSE;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen for (i++; i < len; i++) {
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen if (!IS_LWSP(p[i]))
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen break;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen }
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen *data = p + i;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen return TRUE;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen}
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainenstatic const char *const *
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainenindex_mail_get_parsed_header(struct index_mail *mail, unsigned int field_idx)
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen{
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(const char *) header_values;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen const struct index_mail_line *lines;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen const unsigned char *header, *value_start, *value_end;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen const unsigned int *line_idx;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen const char *value;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen unsigned int i, lines_count, first_line_idx;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen line_idx = array_idx(&mail->header_match_lines, field_idx);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen i_assert(*line_idx != 0);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen first_line_idx = *line_idx - 1;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen p_array_init(&header_values, mail->mail.data_pool, 4);
b88c43d09a288e99d439c78de4cc613212ea924cTimo Sirainen header = mail->header_data->data;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen lines = array_get(&mail->header_lines, &lines_count);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen for (i = first_line_idx; i < lines_count; i++) {
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen if (lines[i].field_idx != lines[first_line_idx].field_idx)
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen break;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen /* skip header: and drop ending LF */
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen value_start = header + lines[i].start_pos;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen value_end = header + lines[i].end_pos;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen if (skip_header(&value_start, value_end - value_start)) {
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen if (value_start != value_end && value_end[-1] == '\n')
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen value_end--;
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen value = p_strndup(mail->mail.data_pool, value_start,
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen value_end - value_start);
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen array_append(&header_values, &value, 1);
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen }
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen }
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen array_append_zero(&header_values);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen return array_idx(&header_values, 0);
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen}
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenstatic int
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenindex_mail_get_raw_headers(struct index_mail *mail, const char *field,
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen const char *const **value_r)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen struct mail *_mail = &mail->mail.mail;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen const char *headers[2], *value;
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen struct mailbox_header_lookup_ctx *headers_ctx;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen unsigned char *data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen unsigned int field_idx;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen string_t *dest;
d8a7046624a082938501e8268ed0cdcba4826e96Timo Sirainen size_t i, len, len2;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen int ret;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(const char *) header_values;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
042e2b8447b34051f0380baafcf0073704430239Timo Sirainen i_assert(field != NULL);
042e2b8447b34051f0380baafcf0073704430239Timo Sirainen
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen field_idx = get_header_field_idx(_mail->box, field,
e217d6fce33746e198ecc21bff0bc658664c9ef4Timo Sirainen MAIL_CACHE_DECISION_TEMP);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen dest = str_new(mail->mail.data_pool, 128);
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen if (mail_cache_lookup_headers(_mail->transaction->cache_view, dest,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen _mail->seq, &field_idx, 1) <= 0) {
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen /* not in cache / error - first see if it's already parsed */
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen p_free(mail->mail.data_pool, dest);
6b44fc75c0039d1006ce4d543544552449b8e229Aki Tuomi if (mail->data.header_parser_initialized) {
ea5a14af8ae816feda08937084954e3912748181Timo Sirainen /* don't try to parse headers recursively. we're here
ea5a14af8ae816feda08937084954e3912748181Timo Sirainen because message size was wrong and istream-mail
ea5a14af8ae816feda08937084954e3912748181Timo Sirainen wants to log some cached headers. */
ea5a14af8ae816feda08937084954e3912748181Timo Sirainen i_assert(mail->mail.mail.lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE);
ea5a14af8ae816feda08937084954e3912748181Timo Sirainen mail_set_aborted(&mail->mail.mail);
ea5a14af8ae816feda08937084954e3912748181Timo Sirainen return -1;
6b44fc75c0039d1006ce4d543544552449b8e229Aki Tuomi }
5dabb8e733a8af2337eb543f782eb6c43ea462dcTimo Sirainen if (mail->header_seq != mail->data.seq ||
5dabb8e733a8af2337eb543f782eb6c43ea462dcTimo Sirainen index_mail_header_is_parsed(mail, field_idx) < 0) {
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen /* parse */
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen const char *reason = index_mail_cache_reason(_mail,
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen t_strdup_printf("header %s", field));
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen headers[0] = field; headers[1] = NULL;
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen headers_ctx = mailbox_header_lookup_init(_mail->box,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen headers);
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen ret = index_mail_parse_headers(mail, headers_ctx, reason);
f95b3d29bc56f139c18c880aa574a0ca72b0cffbTimo Sirainen mailbox_header_lookup_unref(&headers_ctx);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen if (ret < 0)
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return -1;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen }
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen if ((ret = index_mail_header_is_parsed(mail, field_idx)) <= 0) {
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen /* not found */
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen i_assert(ret != -1);
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen *value_r = p_new(mail->mail.data_pool, const char *, 1);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 0;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen *value_r = index_mail_get_parsed_header(mail, field_idx);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen _mail->transaction->stats.cache_hit_count++;
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen data = buffer_get_modifiable_data(dest, &len);
8f61542ca70f3f0dda15630447a00877b132efa8Timo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen if (len == 0) {
7fc0f80480063a9d4cb9e8c07b50db2a5627799eTimo Sirainen /* cached as nonexistent. */
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen *value_r = p_new(mail->mail.data_pool, const char *, 1);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 0;
38d7db318188c4ac9cdc8c6cdb936b36a5258e19Timo Sirainen }
38d7db318188c4ac9cdc8c6cdb936b36a5258e19Timo Sirainen
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen p_array_init(&header_values, mail->mail.data_pool, 4);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* cached. skip "header name: " parts in dest. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (i = 0; i < len; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data[i] == ':') {
e3411c496000d3e2797b43a33584dfba954e815eTimo Sirainen i++;
e3411c496000d3e2797b43a33584dfba954e815eTimo Sirainen while (i < len && IS_LWSP(data[i])) i++;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* @UNSAFE */
d8a7046624a082938501e8268ed0cdcba4826e96Timo Sirainen len2 = get_header_size(dest, i);
d8a7046624a082938501e8268ed0cdcba4826e96Timo Sirainen data[i + len2] = '\0';
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen value = (const char *)data + i;
d8a7046624a082938501e8268ed0cdcba4826e96Timo Sirainen i += len2 + 1;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen
48ac48572e6a196ac6c415180f0c8dcfad35d788Timo Sirainen array_append(&header_values, &value, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen array_append_zero(&header_values);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen *value_r = array_idx(&header_values, 0);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 0;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen}
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainenstatic int unfold_header(pool_t pool, const char **_str)
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen{
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen const char *str = *_str;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen char *new_str;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen unsigned int i, j;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen for (i = 0; str[i] != '\0'; i++) {
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen if (str[i] == '\n')
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen break;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen }
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen if (str[i] == '\0')
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen return 0;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen /* @UNSAFE */
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen new_str = p_malloc(pool, i + strlen(str+i) + 1);
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen memcpy(new_str, str, i);
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen for (j = i; str[i] != '\0'; i++) {
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen if (str[i] == '\n') {
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen new_str[j++] = ' ';
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen i++;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen if (str[i] == '\0')
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen break;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen if (str[i] != ' ' && str[i] != '\t') {
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen /* corrupted */
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen return -1;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen }
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen } else {
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen new_str[j++] = str[i];
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen }
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen }
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen new_str[j] = '\0';
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen *_str = new_str;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen return 0;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen}
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainenstatic void str_replace_nuls(string_t *str)
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen{
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen char *data = str_c_modifiable(str);
2ac5f36aa7c2e7a07ba8815d43a6d7483f62e74cTimo Sirainen size_t i, len = str_len(str);
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen for (i = 0; i < len; i++) {
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen if (data[i] == '\0')
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen data[i] = ' ';
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen }
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen}
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainenstatic int
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainenindex_mail_headers_decode(struct index_mail *mail, const char *const **_list,
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen unsigned int max_count)
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen{
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen const char *const *list = *_list;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen const char **decoded_list, *input;
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen unsigned int i, count;
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen string_t *str;
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen
d3837c0362588792db3e3148d217f31ff3172922Timo Sirainen count = str_array_length(list);
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen if (count > max_count)
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen count = max_count;
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen decoded_list = p_new(mail->mail.data_pool, const char *, count + 1);
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen str = t_str_new(512);
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen for (i = 0; i < count; i++) {
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen str_truncate(str, 0);
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen input = list[i];
b9a664472557182cff0825a2285b8fca39f72947Timo Sirainen /* unfold all lines into a single line */
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen if (unfold_header(mail->mail.data_pool, &input) < 0)
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen return -1;
b9a664472557182cff0825a2285b8fca39f72947Timo Sirainen
b9a664472557182cff0825a2285b8fca39f72947Timo Sirainen /* decode MIME encoded-words. decoding may also add new LFs. */
82d3a1d1594ed93d04d7bf999027b3e5104de6e4Timo Sirainen message_header_decode_utf8((const unsigned char *)input,
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen strlen(input), str, NULL);
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen if (strcmp(str_c(str), input) != 0) {
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen if (strlen(str_c(str)) != str_len(str)) {
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen /* replace NULs with spaces */
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen str_replace_nuls(str);
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen }
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen input = p_strdup(mail->mail.data_pool, str_c(str));
66f9709e0c7604e2282b930b6a48fe9f0dd20ab8Timo Sirainen }
f38854c96aef76e0c859df4e8f7303325b7ae8a1Timo Sirainen decoded_list[i] = input;
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen }
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen *_list = decoded_list;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen return 0;
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen}
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenint index_mail_get_headers(struct mail *_mail, const char *field,
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen bool decode_to_utf8, const char *const **value_r)
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(_mail);
073f965351846b8c97347b882c441dc336965e26Timo Sirainen bool retry = TRUE;
073f965351846b8c97347b882c441dc336965e26Timo Sirainen int ret;
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen
073f965351846b8c97347b882c441dc336965e26Timo Sirainen for (;; retry = FALSE) {
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen if (index_mail_get_raw_headers(mail, field, value_r) < 0)
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen return -1;
990d55ce3fc461eeacce3ef830b1c5dde5c3f150Josef 'Jeff' Sipek if (**value_r == NULL)
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen return 0;
990d55ce3fc461eeacce3ef830b1c5dde5c3f150Josef 'Jeff' Sipek if (!decode_to_utf8)
990d55ce3fc461eeacce3ef830b1c5dde5c3f150Josef 'Jeff' Sipek return 1;
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen T_BEGIN {
8ae72ad7d0c69e972cfa65d1e2ce4e3e9a8b765cTimo Sirainen ret = index_mail_headers_decode(mail, value_r, UINT_MAX);
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen } T_END;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen
073f965351846b8c97347b882c441dc336965e26Timo Sirainen if (ret < 0 && retry) {
7522446d6514e5593c9d4d7e4beacd328301cb23Aki Tuomi mail_set_mail_cache_corrupted(_mail, "Broken header %s",
7522446d6514e5593c9d4d7e4beacd328301cb23Aki Tuomi field);
d48ab236010e588c7b52e54db47fe9842a2e27e8Timo Sirainen } else {
d48ab236010e588c7b52e54db47fe9842a2e27e8Timo Sirainen break;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen }
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen }
073f965351846b8c97347b882c441dc336965e26Timo Sirainen if (ret < 0) {
073f965351846b8c97347b882c441dc336965e26Timo Sirainen i_panic("BUG: Broken header %s for mail UID %u "
073f965351846b8c97347b882c441dc336965e26Timo Sirainen "wasn't fixed by re-parsing the header",
073f965351846b8c97347b882c441dc336965e26Timo Sirainen field, _mail->uid);
073f965351846b8c97347b882c441dc336965e26Timo Sirainen }
073f965351846b8c97347b882c441dc336965e26Timo Sirainen return 1;
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen}
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenint index_mail_get_first_header(struct mail *_mail, const char *field,
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen bool decode_to_utf8, const char **value_r)
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(_mail);
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen const char *const *list;
073f965351846b8c97347b882c441dc336965e26Timo Sirainen bool retry = TRUE;
073f965351846b8c97347b882c441dc336965e26Timo Sirainen int ret;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen
073f965351846b8c97347b882c441dc336965e26Timo Sirainen for (;; retry = FALSE) {
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen if (index_mail_get_raw_headers(mail, field, &list) < 0)
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen return -1;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen if (!decode_to_utf8 || list[0] == NULL) {
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen ret = 0;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen break;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen }
dce5a2719df4fc64a8762d2aa94ba98dcf9cd6feTimo Sirainen
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen T_BEGIN {
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen ret = index_mail_headers_decode(mail, &list, 1);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen } T_END;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen
073f965351846b8c97347b882c441dc336965e26Timo Sirainen if (ret < 0 && retry) {
7522446d6514e5593c9d4d7e4beacd328301cb23Aki Tuomi mail_set_mail_cache_corrupted(_mail, "Broken header %s",
7522446d6514e5593c9d4d7e4beacd328301cb23Aki Tuomi field);
d48ab236010e588c7b52e54db47fe9842a2e27e8Timo Sirainen /* retry by parsing the full header */
d48ab236010e588c7b52e54db47fe9842a2e27e8Timo Sirainen } else {
d48ab236010e588c7b52e54db47fe9842a2e27e8Timo Sirainen break;
96a464e3e417557153272c964fc8a0e9bb6d6b86Timo Sirainen }
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen }
073f965351846b8c97347b882c441dc336965e26Timo Sirainen if (ret < 0) {
073f965351846b8c97347b882c441dc336965e26Timo Sirainen i_panic("BUG: Broken header %s for mail UID %u "
073f965351846b8c97347b882c441dc336965e26Timo Sirainen "wasn't fixed by re-parsing the header",
073f965351846b8c97347b882c441dc336965e26Timo Sirainen field, _mail->uid);
073f965351846b8c97347b882c441dc336965e26Timo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen *value_r = list[0];
073f965351846b8c97347b882c441dc336965e26Timo Sirainen return list[0] != NULL ? 1 : 0;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen}
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
f7ec15aefabeb0a17d1f262bc5e9a15e43dfc5adTimo Sirainenstatic void
f7ec15aefabeb0a17d1f262bc5e9a15e43dfc5adTimo Sirainenheader_cache_callback(struct header_filter_istream *input ATTR_UNUSED,
f7ec15aefabeb0a17d1f262bc5e9a15e43dfc5adTimo Sirainen struct message_header_line *hdr,
6202c9d2492d33ff28ba61a5c57c6768df9577d0Timo Sirainen bool *matched ATTR_UNUSED, struct index_mail *mail)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
d2b94d25f842cd1b7acaf4dd7de858f7c6a821c9Timo Sirainen index_mail_parse_header(NULL, hdr, mail);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen}
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenint index_mail_get_header_stream(struct mail *_mail,
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen struct mailbox_header_lookup_ctx *headers,
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen struct istream **stream_r)
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(_mail);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen struct istream *input;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen string_t *dest;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen if (mail->data.filter_stream != NULL) {
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen const unsigned char *data;
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen size_t size;
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen /* read through the previous filter_stream. this makes sure
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen that the fields are added to cache, and most importantly it
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen resets header_parser_initialized=FALSE so we don't assert
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen on it. */
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen while (i_stream_read_more(mail->data.filter_stream, &data, &size) > 0)
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen i_stream_skip(mail->data.filter_stream, size);
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen i_stream_destroy(&mail->data.filter_stream);
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen }
cb1e7cb6766cfb18900e600961fb8b45a4dffaf8Timo Sirainen
a24b0595f0f7d3925d4c9ac26fa503ff87c43e43Timo Sirainen if (mail->data.save_bodystructure_header) {
a24b0595f0f7d3925d4c9ac26fa503ff87c43e43Timo Sirainen /* we have to parse the header. */
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen const char *reason =
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen index_mail_cache_reason(_mail, "bodystructure");
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen if (index_mail_parse_headers(mail, headers, reason) < 0)
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return -1;
a24b0595f0f7d3925d4c9ac26fa503ff87c43e43Timo Sirainen }
a24b0595f0f7d3925d4c9ac26fa503ff87c43e43Timo Sirainen
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen dest = str_new(mail->mail.data_pool, 256);
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen if (mail_cache_lookup_headers(_mail->transaction->cache_view, dest,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen _mail->seq, headers->idx,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen headers->count) > 0) {
6202c9d2492d33ff28ba61a5c57c6768df9577d0Timo Sirainen str_append(dest, "\n");
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen _mail->transaction->stats.cache_hit_count++;
19a1cfc537d979c532fac71264dba0b9dabc65d9Timo Sirainen mail->data.filter_stream =
fe363b433b8038a69b55169da9dca27892ad7d18Timo Sirainen i_stream_create_from_data(str_data(dest),
19a1cfc537d979c532fac71264dba0b9dabc65d9Timo Sirainen str_len(dest));
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen *stream_r = mail->data.filter_stream;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* not in cache / error */
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen p_free(mail->mail.data_pool, dest);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen unsigned int first_not_found = UINT_MAX, not_found_count = 0;
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen for (unsigned int i = 0; i < headers->count; i++) {
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen if (mail_cache_field_exists(_mail->transaction->cache_view,
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen _mail->seq, headers->idx[i]) <= 0) {
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen if (not_found_count++ == 0)
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen first_not_found = i;
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen }
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen }
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen const char *reason;
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen if (not_found_count == 0)
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen reason = "BUG: all headers seem to exist in cache";
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen else {
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen i_assert(first_not_found != UINT_MAX);
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen reason = index_mail_cache_reason(_mail, t_strdup_printf(
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen "%u/%u headers not cached (first=%s)",
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen not_found_count, headers->count, headers->name[first_not_found]));
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen }
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen if (mail_get_hdr_stream_because(_mail, NULL, reason, &input) < 0)
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return -1;
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen index_mail_parse_header_init(mail, headers);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail->data.filter_stream =
185ed0142fbbfb86e7a98519e7c6f11ec00723cdTimo Sirainen i_stream_create_header_filter(mail->data.stream,
185ed0142fbbfb86e7a98519e7c6f11ec00723cdTimo Sirainen HEADER_FILTER_INCLUDE |
6202c9d2492d33ff28ba61a5c57c6768df9577d0Timo Sirainen HEADER_FILTER_ADD_MISSING_EOH |
185ed0142fbbfb86e7a98519e7c6f11ec00723cdTimo Sirainen HEADER_FILTER_HIDE_BODY,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen headers->name, headers->count,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen header_cache_callback, mail);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen *stream_r = mail->data.filter_stream;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}