index-mail-headers.c revision bb92096bdd934cc19def61b5fb4644af6351b62d
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen/* Copyright (C) 2003 Timo Sirainen */
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen unsigned int count;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen unsigned int *idx;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen const char **name;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic int header_line_cmp(const void *p1, const void *p2)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen const struct index_mail_line *l1 = p1, *l2 = p2;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen diff = (int)l1->field_idx - (int)l2->field_idx;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic void index_mail_parse_header_finish(struct index_mail *mail)
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen unsigned int i, j, count, match_idx, match_count;
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen lines = array_get_modifyable(&mail->header_lines, &count);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* sort it first so fields are grouped together and ordered by
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen line number */
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen qsort(lines, count, sizeof(*lines), header_line_cmp);
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen match = array_get(&mail->header_match, &match_count);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen header = buffer_get_data(mail->header_data, NULL);
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 256);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* go through all the header lines we found */
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* matches and header lines are both sorted, all matches
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen until lines[i] weren't found */
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* if match[] doesn't have header_match_value,
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen it belongs to some older header parsing and we
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen just want to ignore it. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (match[match_idx] == mail->header_match_value) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* this header doesn't exist. remember that. */
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* save index to first header line */
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen array_idx_set(&mail->header_match_lines, match_idx, &j);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* buffer contains: { uint32_t line_num[], 0, header texts }
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen noncontiguous is just a small optimization.. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (; i < j; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_append(buf, header + lines[i].start_pos,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_append(buf, header + lines[i].start_pos,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_cache_add(mail->trans->cache_trans, mail->data.seq,
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen for (; match_idx < match_count; match_idx++) {
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen if (match[match_idx] == mail->header_match_value) {
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen /* this header doesn't exist. remember that. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenvoid index_mail_parse_header_init(struct index_mail *mail,
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen mail->header_data = buffer_create_dynamic(default_pool, 4096);
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen ARRAY_CREATE(&mail->header_lines, default_pool,
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen ARRAY_CREATE(&mail->header_match, default_pool, uint8_t, 32);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen ARRAY_CREATE(&mail->header_match_lines, default_pool,
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen unsigned int, 32);
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen /* @UNSAFE: wrapped, we'll have to clear the buffer */
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen array_idx_set(&mail->header_match, headers->idx[i],
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (mail->wanted_headers != NULL && mail->wanted_headers != headers) {
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen array_idx_set(&mail->header_match, headers->idx[i],
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic void index_mail_parse_finish_imap_envelope(struct index_mail *mail)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen imap_envelope_write_part_data(mail->data.envelope_data, str);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_cache_add(mail->trans->cache_trans, mail->data.seq,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen MAIL_CACHE_ENVELOPE, str_data(str), str_len(str));
644268f7848a7c4221146d0b11feb8ed5bbed233Timo Sirainenint index_mail_parse_header(struct message_part *part,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen imap_bodystructure_parse_header(mail->data_pool, part, hdr);
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen /* end of headers */
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen /* not found */
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen mail_cache_add(mail->trans->cache_trans, data->seq,
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen if (data->save_sent_date && strcasecmp(hdr->name, "Date") == 0) {
1bdda5c0c30463160c47151537e6bb2c6c994841Timo Sirainen /* 0 == parse error */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen cache_field_name = t_strconcat("hdr.", hdr->name, NULL);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* we don't want this field */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen decision = mail_cache_field_get_decision(mail->ibox->cache,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_cache_field_exists(mail->trans->cache_view,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* already cached */
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen match = array_get_modifyable(&mail->header_match, &count);
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen if (field_idx < count && match[field_idx] == mail->header_match_value) {
38d7db318188c4ac9cdc8c6cdb936b36a5258e19Timo Sirainen /* first header */
38d7db318188c4ac9cdc8c6cdb936b36a5258e19Timo Sirainen (match[field_idx] & ~1) != mail->header_match_value)) {
38d7db318188c4ac9cdc8c6cdb936b36a5258e19Timo Sirainen /* we don't need to do anything with this header */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->parse_line.start_pos = str_len(mail->header_data);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->parse_line.line_num = data->parse_line_num;
60576cd64e6a537413cd90104f7e862f71d48c81Timo Sirainen str_append_n(mail->header_data, hdr->middle, hdr->middle_len);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen str_append_n(mail->header_data, hdr->value, hdr->value_len);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen data->parse_line.end_pos = str_len(mail->header_data);
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen array_append(&mail->header_lines, &data->parse_line, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenindex_mail_parse_header_cb(struct message_part *part,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct message_header_line *hdr, void *context)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen (void)index_mail_parse_header(part, hdr, mail);
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainenint index_mail_parse_headers(struct index_mail *mail,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen if (mail_get_stream(&mail->mail.mail, NULL, NULL) == NULL)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (data->parts == NULL && data->parser_ctx == NULL) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* initialize bodystructure parsing in case we read the whole
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen message_parser_init(mail->data_pool, data->stream);
96541d31299bb40b5a6efdbf9b4cb3d4f4b4a069Timo Sirainen message_parser_parse_header(data->parser_ctx, &data->hdr_size,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* just read the header */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen message_parse_header(data->parts, data->stream, &data->hdr_size,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenimap_envelope_parse_callback(struct message_part *part __attr_unused__,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct message_header_line *hdr, void *context)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenvoid index_mail_headers_get_envelope(struct index_mail *mail)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen header_ctx = mailbox_header_lookup_init(&mail->ibox->box,
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen stream = mail_get_header_stream(&mail->mail.mail, header_ctx);
9db263f2b9ab771fbf9a2bff44a245c45eaef218Timo Sirainen if (mail->data.envelope == NULL && stream != NULL) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* we got the headers from cache - parse them to get the
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic unsigned int
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenget_header_field_idx(struct index_mailbox *ibox, const char *field)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen cache_field_name = t_strconcat("hdr.", field, NULL);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen field_idx = mail_cache_register_lookup(ibox->cache, cache_field_name);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_cache_register_fields(ibox->cache, &header_field, 1);
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainenstatic size_t get_header_size(buffer_t *buffer, size_t pos)
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen const unsigned char *data;
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainenstatic int index_mail_header_is_parsed(struct index_mail *mail,
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen unsigned int count;
cf52b37d807553e91a2d6fb7cb2c8b4c34589e1dTimo Sirainen match = array_get(&mail->header_match, &count);
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen if (match[field_idx] == mail->header_match_value)
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen else if (match[field_idx] == mail->header_match_value + 1)
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainenstatic int skip_header(const unsigned char **data, size_t len)
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen const unsigned char *p = *data;
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen for (i = 0; i < len; i++) {
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen if (p[i] == ':')
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen for (i++; i < len; i++) {
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainenstatic const char *const *
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainenindex_mail_get_parsed_header(struct index_mail *mail, unsigned int field_idx)
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen array_t ARRAY_DEFINE(header_values, const char *);
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen const unsigned char *header, *value_start, *value_end;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen const unsigned int *line_idx;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen line_idx = array_idx(&mail->header_match_lines, field_idx);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen ARRAY_CREATE(&header_values, mail->data_pool, const char *, 4);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen header = buffer_get_data(mail->header_data, NULL);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen lines = array_get(&mail->header_lines, &lines_count);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen for (i = first_line_idx; i < lines_count; i++) {
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen if (lines[i].field_idx != lines[first_line_idx].field_idx)
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen /* skip header: and drop ending LF */
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen if (skip_header(&value_start, value_end - value_start)) {
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen if (value_start != value_end && value_end[-1] == '\n')
1e242794e7a4f653f18fbb8edfe9ccec489a3a08Timo Sirainen value = p_strndup(mail->data_pool, value_start,
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen array_append(&header_values, &value, sizeof(value));
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainenconst char *const *index_mail_get_headers(struct mail *_mail, const char *field)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct index_mail *mail = (struct index_mail *)_mail;
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen struct mailbox_header_lookup_ctx *headers_ctx;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen unsigned char *data;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen array_t ARRAY_DEFINE(header_values, const char *);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen field_idx = get_header_field_idx(mail->ibox, field);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (mail_cache_lookup_headers(mail->trans->cache_view, dest,
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen /* not in cache / error - first see if it's already parsed */
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen ret = index_mail_header_is_parsed(mail, field_idx);
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen headers_ctx = mailbox_header_lookup_init(&mail->ibox->box,
2aecf7be5834e7f6520f8deaad683a6fa1de4d61Timo Sirainen ret = index_mail_parse_headers(mail, headers_ctx);
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen ret = index_mail_header_is_parsed(mail, field_idx);
98dd8e6e81f11f1e6040ca72f4916242d246c863Timo Sirainen index_mail_get_parsed_header(mail, field_idx);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen data = buffer_get_modifyable_data(dest, &len);
38d7db318188c4ac9cdc8c6cdb936b36a5258e19Timo Sirainen /* cached as non-existing. */
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen return p_new(mail->data_pool, const char *, 1);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen ARRAY_CREATE(&header_values, mail->data_pool, const char *, 4);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* cached. skip "header name: " parts in dest. */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (i = 0; i < len; i++) {
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen /* @UNSAFE */
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen array_append(&header_values, &value, sizeof(value));
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen array_append(&header_values, &value, sizeof(value));
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainenconst char *index_mail_get_first_header(struct mail *mail, const char *field)
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen const char *const *list = index_mail_get_headers(mail, field);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic void header_cache_callback(struct message_header_line *hdr,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen (void)index_mail_parse_header(NULL, hdr, mail);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainenindex_mail_get_header_stream(struct mail *_mail,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct index_mail *mail = (struct index_mail *)_mail;
a24b0595f0f7d3925d4c9ac26fa503ff87c43e43Timo Sirainen /* we have to parse the header. */
a24b0595f0f7d3925d4c9ac26fa503ff87c43e43Timo Sirainen if (index_mail_parse_headers(mail, _headers) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (mail_cache_lookup_headers(mail->trans->cache_view, dest,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return i_stream_create_from_data(mail->data_pool,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* not in cache / error */
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen if (mail_get_stream(&mail->mail.mail, NULL, NULL) == NULL)
185ed0142fbbfb86e7a98519e7c6f11ec00723cdTimo Sirainen i_stream_create_header_filter(mail->data.stream,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenindex_header_lookup_init(struct mailbox *box, const char *const headers[])
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct index_mailbox *ibox = (struct index_mailbox *)box;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct mail_cache_field *fields, header_field = {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen const char *const *name;
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen unsigned int i, count;
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen for (count = 0, name = headers; *name != NULL; name++)
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen /* @UNSAFE: headers need to be sorted for filter stream. */
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen qsort(sorted_headers, count, sizeof(*sorted_headers),
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen /* @UNSAFE */
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen fields = t_new(struct mail_cache_field, count);
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen for (i = 0; i < count; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen header_field.name = t_strconcat("hdr.", headers[i], NULL);
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen mail_cache_register_fields(ibox->cache, fields, count);
bb92096bdd934cc19def61b5fb4644af6351b62dTimo Sirainen pool = pool_alloconly_create("index_header_lookup_ctx", 512);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen ctx = p_new(pool, struct index_header_lookup_ctx, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* @UNSAFE */
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen for (i = 0; i < count; i++) {