bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "lib.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "str.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "safe-mkstemp.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "istream.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "istream-crlf.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "istream-seekable.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "istream-base64.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "istream-qp.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "istream-header-filter.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "ostream.h"
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen#include "message-binary-part.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "message-parser.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "message-decoder.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "mail-user.h"
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen#include "index-storage.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#include "index-mail.h"
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen#define MAIL_BINARY_CACHE_EXPIRE_MSECS (60*1000)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen#define IS_CONVERTED_CTE(cte) \
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen ((cte) == MESSAGE_CTE_QP || (cte) == MESSAGE_CTE_BASE64)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenstruct binary_block {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct istream *input;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen uoff_t physical_pos;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen unsigned int body_lines_count;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bool converted, converted_hdr;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen};
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenstruct binary_ctx {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct mail *mail;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct istream *input;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen bool has_nuls, converted;
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen /* each block is its own input stream. basically each converted MIME
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen body has its own block and the parts between the MIME bodies are
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen unconverted blocks */
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct binary_block) blocks;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen uoff_t copy_start_offset;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen};
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenstatic void binary_copy_to(struct binary_ctx *ctx, uoff_t end_offset)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen{
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct binary_block *block;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct istream *linput, *cinput;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen uoff_t orig_offset, size;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_assert(end_offset >= ctx->copy_start_offset);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (end_offset == ctx->copy_start_offset)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen size = end_offset - ctx->copy_start_offset;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen orig_offset = ctx->input->v_offset;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_seek(ctx->input, ctx->copy_start_offset);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen linput = i_stream_create_limit(ctx->input, size);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen cinput = i_stream_create_crlf(linput);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_unref(&linput);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block = array_append_space(&ctx->blocks);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block->input = cinput;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_seek(ctx->input, orig_offset);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen}
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenstatic void
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenbinary_cte_filter_callback(struct header_filter_istream *input,
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct message_header_line *hdr,
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen bool *matched ATTR_UNUSED, void *context ATTR_UNUSED)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen{
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen static const char *cte_binary = "Content-Transfer-Encoding: binary\r\n";
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (hdr != NULL && hdr->eoh) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_header_filter_add(input, cte_binary,
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen strlen(cte_binary));
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen}
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenstatic int
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenadd_binary_part(struct binary_ctx *ctx, const struct message_part *part,
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen bool include_hdr)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen{
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen static const char *filter_headers[] = {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen "Content-Transfer-Encoding",
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen };
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct message_header_parser_ctx *parser;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct message_header_line *hdr;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct message_part *child;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct message_size hdr_size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct istream *linput;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct binary_block *block;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen enum message_cte cte;
b09e2828759f801fbeb06245b69972b400067527Timo Sirainen uoff_t part_end_offset;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen int ret;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* first parse the header to find c-t-e. */
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_seek(ctx->input, part->physical_pos);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen cte = MESSAGE_CTE_78BIT;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen parser = message_parse_header_init(ctx->input, &hdr_size, 0);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen while ((ret = message_parse_header_next(parser, &hdr)) > 0) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (strcasecmp(hdr->name, "Content-Transfer-Encoding") == 0)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen cte = message_decoder_parse_cte(hdr);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_assert(ret < 0);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (message_parse_header_has_nuls(parser)) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* we're not converting NULs to 0x80 when doing a binary fetch,
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen even if they're in the message header. */
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ctx->has_nuls = TRUE;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen message_parse_header_deinit(&parser);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (ctx->input->stream_errno != 0) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi mail_set_critical(ctx->mail,
d052dcfff0c96a0af17a3158e51f709edf4b93a1Timo Sirainen "read(%s) failed: %s", i_stream_get_name(ctx->input),
d052dcfff0c96a0af17a3158e51f709edf4b93a1Timo Sirainen i_stream_get_error(ctx->input));
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return -1;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (cte == MESSAGE_CTE_UNKNOWN) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen mail_storage_set_error(ctx->mail->box->storage,
56b134799a457fd55830355f4c8d746d6bb5206fTimo Sirainen MAIL_ERROR_CONVERSION,
56b134799a457fd55830355f4c8d746d6bb5206fTimo Sirainen "Unknown Content-Transfer-Encoding.");
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return -1;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_seek(ctx->input, part->physical_pos);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (!include_hdr) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* body only */
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen } else if (IS_CONVERTED_CTE(cte)) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* write header with modified content-type */
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (ctx->copy_start_offset != 0)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen binary_copy_to(ctx, part->physical_pos);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block = array_append_space(&ctx->blocks);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen block->physical_pos = part->physical_pos;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block->converted = TRUE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block->converted_hdr = TRUE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen linput = i_stream_create_limit(ctx->input, (uoff_t)-1);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block->input = i_stream_create_header_filter(linput,
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen HEADER_FILTER_EXCLUDE | HEADER_FILTER_HIDE_BODY,
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen filter_headers, N_ELEMENTS(filter_headers),
677b75f90d81eafe742896d6570a2f63ce501d05Josef 'Jeff' Sipek binary_cte_filter_callback, NULL);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_unref(&linput);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen } else {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* copy everything as-is until the end of this header */
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen binary_copy_to(ctx, part->physical_pos +
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen part->header_size.physical_size);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ctx->copy_start_offset = part->physical_pos +
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen part->header_size.physical_size;
b09e2828759f801fbeb06245b69972b400067527Timo Sirainen part_end_offset = part->physical_pos +
b09e2828759f801fbeb06245b69972b400067527Timo Sirainen part->header_size.physical_size +
b09e2828759f801fbeb06245b69972b400067527Timo Sirainen part->body_size.physical_size;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (part->children != NULL) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* multipart */
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen for (child = part->children; child != NULL; child = child->next) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (add_binary_part(ctx, child, TRUE) < 0)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return -1;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
b09e2828759f801fbeb06245b69972b400067527Timo Sirainen binary_copy_to(ctx, part_end_offset);
b09e2828759f801fbeb06245b69972b400067527Timo Sirainen ctx->copy_start_offset = part_end_offset;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return 0;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
91dad1ec260a45694c28837c2594ad2ee04beca6Timo Sirainen if (part->body_size.physical_size == 0) {
91dad1ec260a45694c28837c2594ad2ee04beca6Timo Sirainen /* no body */
91dad1ec260a45694c28837c2594ad2ee04beca6Timo Sirainen ctx->copy_start_offset = part_end_offset;
91dad1ec260a45694c28837c2594ad2ee04beca6Timo Sirainen return 0;
91dad1ec260a45694c28837c2594ad2ee04beca6Timo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* single part - write decoded data */
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block = array_append_space(&ctx->blocks);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen block->physical_pos = part->physical_pos;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_seek(ctx->input, part->physical_pos +
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen part->header_size.physical_size);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen linput = i_stream_create_limit(ctx->input, part->body_size.physical_size);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen switch (cte) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen case MESSAGE_CTE_UNKNOWN:
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_unreached();
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen case MESSAGE_CTE_78BIT:
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen case MESSAGE_CTE_BINARY:
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* no conversion necessary */
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if ((part->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ctx->has_nuls = TRUE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block->input = i_stream_create_crlf(linput);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen break;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen case MESSAGE_CTE_QP:
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block->input = i_stream_create_qp_decoder(linput);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen ctx->converted = block->converted = TRUE;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen break;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen case MESSAGE_CTE_BASE64:
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen block->input = i_stream_create_base64_decoder(linput);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen ctx->converted = block->converted = TRUE;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen break;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_stream_unref(&linput);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
b09e2828759f801fbeb06245b69972b400067527Timo Sirainen ctx->copy_start_offset = part_end_offset;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return 0;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen}
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenstatic int fd_callback(const char **path_r, void *context)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen{
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct mail *_mail = context;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen string_t *path;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen int fd;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen path = t_str_new(256);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen mail_user_set_get_temp_prefix(path, _mail->box->storage->user->set);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen fd = safe_mkstemp_hostpid(path, 0600, (uid_t)-1, (gid_t)-1);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (fd == -1) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_error("Temp file creation to %s failed: %m", str_c(path));
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return -1;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* we just want the fd, unlink it */
46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen if (i_unlink(str_c(path)) < 0) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen /* shouldn't happen.. */
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen i_close_fd(&fd);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return -1;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen *path_r = str_c(path);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return fd;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen}
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenstatic void binary_streams_free(struct binary_ctx *ctx)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen{
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct binary_block *block;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen array_foreach_modifiable(&ctx->blocks, block)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen i_stream_unref(&block->input);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen}
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenstatic void
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenbinary_parts_update(struct binary_ctx *ctx, const struct message_part *part,
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct message_binary_part **msg_bin_parts)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(ctx->mail);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct binary_block *blocks;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct message_binary_part bin_part;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen unsigned int i, count;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen uoff_t size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bool found;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen blocks = array_get_modifiable(&ctx->blocks, &count);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen for (; part != NULL; part = part->next) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen binary_parts_update(ctx, part->children, msg_bin_parts);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&bin_part);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen /* default to unchanged header */
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bin_part.binary_hdr_size = part->header_size.virtual_size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bin_part.physical_pos = part->physical_pos;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen found = FALSE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen for (i = 0; i < count; i++) {
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen if (blocks[i].physical_pos != part->physical_pos ||
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen !blocks[i].converted)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen continue;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen size = blocks[i].input->v_offset;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (blocks[i].converted_hdr)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bin_part.binary_hdr_size = size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen else
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bin_part.binary_body_size = size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen found = TRUE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (found) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bin_part.next = *msg_bin_parts;
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen *msg_bin_parts = p_new(mail->mail.data_pool,
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct message_binary_part, 1);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen **msg_bin_parts = bin_part;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen}
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenstatic void binary_parts_cache(struct binary_ctx *ctx)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(ctx->mail);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen buffer_t *buf;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
c147bff818798a979d93537f72f5c1f68f5d5ba8Aki Tuomi buf = t_buffer_create(128);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen message_binary_part_serialize(mail->data.bin_parts, buf);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen index_mail_cache_add(mail, MAIL_CACHE_BINARY_PARTS,
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen buf->data, buf->used);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen}
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenstatic struct istream **blocks_get_streams(struct binary_ctx *ctx)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen{
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct istream **streams;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen const struct binary_block *blocks;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen unsigned int i, count;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen blocks = array_get(&ctx->blocks, &count);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen streams = t_new(struct istream *, count+1);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen for (i = 0; i < count; i++) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen streams[i] = blocks[i].input;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen i_assert(streams[i]->v_offset == 0);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return streams;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen}
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainenstatic int
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainenblocks_count_lines(struct binary_ctx *ctx, struct istream *full_input)
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen{
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen struct binary_block *blocks, *cur_block;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen unsigned int block_idx, block_count;
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen uoff_t cur_block_offset, cur_block_size;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen const unsigned char *data, *p;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen size_t size, skip;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen ssize_t ret;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen blocks = array_get_modifiable(&ctx->blocks, &block_count);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen cur_block = blocks;
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen cur_block_offset = 0;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen block_idx = 0;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen /* count the number of lines each block contains */
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody while ((ret = i_stream_read_more(full_input, &data, &size)) > 0) {
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen i_assert(cur_block_offset <= cur_block->input->v_offset);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen if (cur_block->input->eof) {
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen /* this is the last input for this block. the input
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen may also contain the next block's data, which we
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen don't want to include in this block's line count. */
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen cur_block_size = cur_block->input->v_offset +
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen i_stream_get_data_size(cur_block->input);
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen i_assert(size >= cur_block_size - cur_block_offset);
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen size = cur_block_size - cur_block_offset;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen }
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen skip = size;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen while ((p = memchr(data, '\n', size)) != NULL) {
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen size -= p-data+1;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen data = p+1;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen cur_block->body_lines_count++;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen }
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen i_stream_skip(full_input, skip);
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen cur_block_offset += skip;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen if (cur_block->input->eof) {
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen /* go to the next block */
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen if (++block_idx == block_count) {
1bc075e2e4ed422f9590c95c3ae223422b97ce6aTimo Sirainen i_assert(i_stream_read_eof(full_input));
4439a1780afc06fe644ead3df6f0b3f77f9cb867Timo Sirainen ret = -1;
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen break;
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen }
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen cur_block++;
64e75f0c95e491debf4b3bc7091d0335d0f63646Timo Sirainen cur_block_offset = 0;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen }
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen }
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen i_assert(ret == -1);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen if (full_input->stream_errno != 0)
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen return -1;
e67ccfba723334c3e3cbeebff0627b8072afe0c7Timo Sirainen i_assert(block_count == 0 || !i_stream_have_bytes_left(cur_block->input));
e67ccfba723334c3e3cbeebff0627b8072afe0c7Timo Sirainen i_assert(block_count == 0 || block_idx+1 == block_count);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen return 0;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen}
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenstatic int
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenindex_mail_read_binary_to_cache(struct mail *_mail,
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen const struct message_part *part,
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen bool include_hdr, const char *reason,
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen bool *binary_r, bool *converted_r)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(_mail);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct mail_binary_cache *cache = &_mail->box->storage->binary_cache;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen struct binary_ctx ctx;
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi struct istream *is;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&ctx);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen ctx.mail = _mail;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen t_array_init(&ctx.blocks, 8);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen mail_storage_free_binary_cache(_mail->box->storage);
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen if (mail_get_stream_because(_mail, NULL, NULL, reason, &ctx.input) < 0)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return -1;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (add_binary_part(&ctx, part, include_hdr) < 0) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen binary_streams_free(&ctx);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return -1;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
e67ccfba723334c3e3cbeebff0627b8072afe0c7Timo Sirainen if (array_count(&ctx.blocks) != 0) {
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi is = i_streams_merge(blocks_get_streams(&ctx),
e67ccfba723334c3e3cbeebff0627b8072afe0c7Timo Sirainen IO_BLOCK_SIZE,
e67ccfba723334c3e3cbeebff0627b8072afe0c7Timo Sirainen fd_callback, _mail);
e67ccfba723334c3e3cbeebff0627b8072afe0c7Timo Sirainen } else {
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi is = i_stream_create_from_data("", 0);
e67ccfba723334c3e3cbeebff0627b8072afe0c7Timo Sirainen }
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi i_stream_set_name(is, t_strdup_printf(
07725eacefe66333b9c7e4c7bfa423f5fa5aa74bTimo Sirainen "<binary stream of mailbox %s UID %u>",
07725eacefe66333b9c7e4c7bfa423f5fa5aa74bTimo Sirainen _mail->box->vname, _mail->uid));
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi if (blocks_count_lines(&ctx, is) < 0) {
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi if (is->stream_errno == EINVAL) {
56b134799a457fd55830355f4c8d746d6bb5206fTimo Sirainen /* MIME part contains invalid data */
56b134799a457fd55830355f4c8d746d6bb5206fTimo Sirainen mail_storage_set_error(_mail->box->storage,
56b134799a457fd55830355f4c8d746d6bb5206fTimo Sirainen MAIL_ERROR_INVALIDDATA,
56b134799a457fd55830355f4c8d746d6bb5206fTimo Sirainen "Invalid data in MIME part");
56b134799a457fd55830355f4c8d746d6bb5206fTimo Sirainen } else {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi mail_set_critical(_mail, "read(%s) failed: %s",
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi i_stream_get_name(is),
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi i_stream_get_error(is));
56b134799a457fd55830355f4c8d746d6bb5206fTimo Sirainen }
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi i_stream_unref(&is);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen binary_streams_free(&ctx);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return -1;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi if (_mail->uid > 0) {
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi cache->to = timeout_add(MAIL_BINARY_CACHE_EXPIRE_MSECS,
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi mail_storage_free_binary_cache,
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi _mail->box->storage);
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi cache->box = _mail->box;
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi cache->uid = _mail->uid;
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi cache->orig_physical_pos = part->physical_pos;
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi cache->include_hdr = include_hdr;
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi cache->input = is;
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi }
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi i_assert(!i_stream_have_bytes_left(is));
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi cache->size = is->v_offset;
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi i_stream_seek(is, 0);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (part->parent == NULL && include_hdr &&
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen mail->data.bin_parts == NULL) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen binary_parts_update(&ctx, part, &mail->data.bin_parts);
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi if (_mail->uid > 0)
ae5330824a7ca85edd005109a353c3ab6b23b6b8Aki Tuomi binary_parts_cache(&ctx);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen binary_streams_free(&ctx);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen *binary_r = ctx.converted ? TRUE : ctx.has_nuls;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen *converted_r = ctx.converted;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return 0;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen}
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenstatic bool get_cached_binary_parts(struct index_mail *mail)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen{
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen const unsigned int field_idx =
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen mail->ibox->cache_fields[MAIL_CACHE_BINARY_PARTS].idx;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen buffer_t *part_buf;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen int ret;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (mail->data.bin_parts != NULL)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return TRUE;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
c147bff818798a979d93537f72f5c1f68f5d5ba8Aki Tuomi part_buf = t_buffer_create(128);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ret = index_mail_cache_lookup_field(mail, part_buf, field_idx);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen if (ret <= 0)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return FALSE;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen if (message_binary_part_deserialize(mail->mail.data_pool,
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen part_buf->data, part_buf->used,
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen &mail->data.bin_parts) < 0) {
7522446d6514e5593c9d4d7e4beacd328301cb23Aki Tuomi mail_set_mail_cache_corrupted(&mail->mail.mail,
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen "Corrupted cached binary.parts data");
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return FALSE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return TRUE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen}
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenstatic struct message_part *
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenmsg_part_find(struct message_part *parts, uoff_t physical_pos)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen{
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct message_part *part, *child;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen for (part = parts; part != NULL; part = part->next) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (part->physical_pos == physical_pos)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return part;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen child = msg_part_find(part->children, physical_pos);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (child != NULL)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return child;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return NULL;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen}
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenstatic int
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenindex_mail_get_binary_size(struct mail *_mail,
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen const struct message_part *part, bool include_hdr,
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen uoff_t *size_r, unsigned int *lines_r)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(_mail);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct message_part *all_parts, *msg_part;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen const struct message_binary_part *bin_part, *root_bin_part;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen uoff_t size, end_offset;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen unsigned int lines;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bool binary, converted;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (mail_get_parts(_mail, &all_parts) < 0)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return -1;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen /* first lookup from cache */
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (!get_cached_binary_parts(mail)) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen /* not found. parse the whole message */
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (index_mail_read_binary_to_cache(_mail, all_parts, TRUE,
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen "binary.size", &binary, &converted) < 0)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return -1;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen size = part->header_size.virtual_size +
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen part->body_size.virtual_size;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen /* note that we assume here that binary translation doesn't change the
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen headers' line counts. this isn't true if the original message
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen contained duplicate Content-Transfer-Encoding lines, but since
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen that's invalid anyway we don't bother trying to handle it. */
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen lines = part->header_size.lines + part->body_size.lines;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen end_offset = part->physical_pos + size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bin_part = mail->data.bin_parts; root_bin_part = NULL;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen for (; bin_part != NULL; bin_part = bin_part->next) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen msg_part = msg_part_find(all_parts, bin_part->physical_pos);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (msg_part == NULL) {
8b5b1f6cb19253dfd7821fcef8e9b7e95e6caf3aTimo Sirainen /* either binary.parts or mime.parts is broken */
0206dc57f2c04da69599dea5816235cfeb2b897aMartti Rannanjärvi mail_set_cache_corrupted(_mail, MAIL_FETCH_MESSAGE_PARTS, t_strdup_printf(
8b5b1f6cb19253dfd7821fcef8e9b7e95e6caf3aTimo Sirainen "BINARY part at offset %"PRIuUOFF_T" not found from MIME parts",
8b5b1f6cb19253dfd7821fcef8e9b7e95e6caf3aTimo Sirainen bin_part->physical_pos));
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return -1;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (msg_part->physical_pos >= part->physical_pos &&
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen msg_part->physical_pos < end_offset) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (msg_part->physical_pos == part->physical_pos)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen root_bin_part = bin_part;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen size -= msg_part->header_size.virtual_size +
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen msg_part->body_size.virtual_size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen size += bin_part->binary_hdr_size +
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bin_part->binary_body_size;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen lines -= msg_part->body_size.lines;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen lines += bin_part->binary_body_lines_count;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (!include_hdr) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (root_bin_part != NULL)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen size -= root_bin_part->binary_hdr_size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen else
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen size -= part->header_size.virtual_size;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen lines -= part->header_size.lines;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen *size_r = size;
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen *lines_r = lines;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return 0;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen}
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainenint index_mail_get_binary_stream(struct mail *_mail,
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen const struct message_part *part,
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bool include_hdr, uoff_t *size_r,
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen unsigned int *lines_r, bool *binary_r,
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen struct istream **stream_r)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen{
87490012895b4f371635ded00add04c9107dcfefJosef 'Jeff' Sipek struct index_mail *mail = INDEX_MAIL(_mail);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct mail_binary_cache *cache = &_mail->box->storage->binary_cache;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen struct istream *input;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen bool binary, converted;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (stream_r == NULL) {
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen return index_mail_get_binary_size(_mail, part, include_hdr,
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen size_r, lines_r);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen /* current implementation doesn't bother implementing this,
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen because it's not needed by anything. */
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen i_assert(lines_r == NULL);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen /* FIXME: always put the header to temp file. skip it when needed. */
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (cache->box == _mail->box && cache->uid == _mail->uid &&
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen cache->orig_physical_pos == part->physical_pos &&
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen cache->include_hdr == include_hdr) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen /* we have this cached already */
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen i_stream_seek(cache->input, 0);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen timeout_reset(cache->to);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen binary = TRUE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen converted = TRUE;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen } else {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (index_mail_read_binary_to_cache(_mail, part, include_hdr,
02b78558dc03daa2e7da2010b63f247b49936a38Timo Sirainen "binary stream", &binary, &converted) < 0)
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return -1;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen mail->data.cache_fetch_fields |= MAIL_FETCH_STREAM_BINARY;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen *size_r = cache->size;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen *binary_r = binary;
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen if (!converted) {
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen /* don't keep this cached. it's exactly the same as
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen the original stream */
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen i_assert(mail->data.stream != NULL);
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen i_stream_seek(mail->data.stream, part->physical_pos +
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen (include_hdr ? 0 :
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen part->header_size.physical_size));
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen input = i_stream_create_crlf(mail->data.stream);
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen *stream_r = i_stream_create_limit(input, *size_r);
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen i_stream_unref(&input);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen mail_storage_free_binary_cache(_mail->box->storage);
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen } else {
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen *stream_r = cache->input;
471eed36d9aee2f7d99072a4fc5e164a6741194bTimo Sirainen i_stream_ref(cache->input);
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen }
6e8f0036cad59d1d6bcd9ef69bfe712d01656ca3Timo Sirainen return 0;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen}