/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "buffer.h"
#include "ioloop.h"
#include "istream.h"
#include "hex-binary.h"
#include "str.h"
#include "mailbox-recent-flags.h"
#include "message-date.h"
#include "message-part-data.h"
#include "message-part-serialize.h"
#include "message-parser.h"
#include "message-snippet.h"
#include "imap-bodystructure.h"
#include "imap-envelope.h"
#include "mail-cache.h"
#include "mail-index-modseq.h"
#include "index-storage.h"
#include "istream-mail.h"
#include "index-mail.h"
#include <fcntl.h>
{ .name = "flags",
.field_size = sizeof(uint32_t) },
.field_size = sizeof(struct mail_sent_date) },
{ .name = "date.received",
.field_size = sizeof(uint32_t) },
.field_size = sizeof(uint32_t) },
{ .name = "size.virtual",
.field_size = sizeof(uoff_t) },
{ .name = "size.physical",
.field_size = sizeof(uoff_t) },
{ .name = "imap.body",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "imap.bodystructure",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "imap.envelope",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "pop3.uidl",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "pop3.order",
.field_size = sizeof(uint32_t) },
{ .name = "guid",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "mime.parts",
{ .name = "binary.parts",
/* FIXME: for now need to update get_metadata_precache_fields() in
index-status.c when adding more fields. those fields should probably
just be moved here to the same struct. */
};
enum index_cache_field field);
unsigned int field_idx)
{
int ret;
if (ret > 0)
return ret;
}
{
const unsigned int field_idx =
}
{
const char *error;
return NULL;
"Corrupted cached mime.parts data: %s (parts=%s)",
}
return parts;
}
{
return TRUE;
return TRUE;
}
}
return FALSE;
}
{
T_BEGIN {
} T_END;
return FALSE;
/* we know the NULs now, update them */
if (message_parts_have_nuls(part)) {
} else {
}
return TRUE;
}
{
const char *parts_str;
parts_str = "";
else
"Cached MIME parts don't match message during parsing: %s (parts=%s)",
}
enum index_cache_field field,
{
bool ret;
else {
}
return ret;
}
{
}
{
return TRUE;
/* no private view (set by view syncing) -> no private flags */
return FALSE;
}
/* mail is still being saved, it has no private flags yet */
return FALSE;
}
}
{
flags |= MAIL_RECENT;
if (index_mail_get_pvt(_mail)) {
/* mailbox has private flags */
flags &= ~pvt_flags_mask;
}
return flags;
}
{
}
{
return 0;
}
{
const char *const *names;
const unsigned int *keyword_indexes;
(void)index_mail_get_keyword_indexes(_mail);
for (i = 0; i < count; i++) {
const char *name;
}
/* end with NULL */
}
{
&data->keyword_indexes);
}
return &data->keyword_indexes;
}
{
return 0;
}
const char *reason =
return -1;
/* parts may be set now as a result of some plugin */
}
if (index_mail_parse_body(mail, 0) < 0)
return -1;
}
return 0;
}
{
uint32_t t;
&t, sizeof(t)))
data->received_date = t;
}
}
{
uint32_t t;
&t, sizeof(t)))
}
}
{
const char *str;
time_t t;
return 0;
return ret;
if (ret == 0 ||
!message_date_parse((const unsigned char *)str,
/* 0 = not found / invalid */
t = 0;
tz = 0;
}
return 0;
}
{
return 0;
}
if (index_mail_cache_sent_date(mail) < 0)
return -1;
return 0;
}
{
(void)get_cached_parts(mail);
}
}
{
const void *idata;
}
return vsize;
}
{
/* see if we can get it from index */
&size))
else {
if (!get_cached_msgpart_sizes(mail))
return FALSE;
}
}
}
/* if vsize is present and wanted for index, but missing from index
add it to index. */
}
return TRUE;
}
{
if (!data->hdr_size_set)
return;
/* we've already called get_cached_msgpart_sizes() and it didn't work.
try to do this by using cached virtual size and a quick physical
size lookup. */
return;
if (!data->body_size_set) {
/* get the physical size, but not if it requires reading
through the whole message */
/* we should have everything now. try again. */
}
}
}
{
return 0;
return -1;
return 0;
}
{
&size))
else
(void)get_cached_msgpart_sizes(mail);
}
}
{
}
{
if (set->mail_cache_min_mail_count > 0) {
/* First check if we've configured caching not to be used with
low enough message count. */
return;
}
}
}
struct message_header_line *hdr,
{
}
{
const unsigned int cache_field_body =
const unsigned int cache_field_bodystructure =
MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0)
return TRUE;
return TRUE;
return TRUE;
return FALSE;
}
{
unsigned int cache_flags_idx;
bool want_cached;
if (data->parsed_bodystructure &&
/* we need message_parts cached to be able to
actually use it in BODY/BODYSTRUCTURE reply */
want_cached = TRUE;
}
/* cache flags should never get unset as long as the message doesn't
change, but try to handle it anyway */
} else {
}
&cache_flags, sizeof(cache_flags));
}
}
{
const unsigned int cache_field =
if (data->messageparts_saved_to_cache ||
cache_field) != 0) {
/* already cached */
return;
}
/* we never want it cached */
return;
}
if (decision == MAIL_CACHE_DECISION_NO &&
!data->save_message_parts &&
/* we didn't really care about the message parts themselves,
just wanted to use something that depended on it */
return;
}
T_BEGIN {
} T_END;
}
static void
enum index_cache_field field)
{
const unsigned int cache_field_parts =
const unsigned int cache_field_body =
const unsigned int cache_field_bodystructure =
if (data->messageparts_saved_to_cache ||
/* cached it as flag + message_parts */
}
}
if (!data->parsed_bodystructure)
return;
/* If BODY is fetched first but BODYSTRUCTURE is also wanted, we don't
normally want to first cache BODY and then BODYSTRUCTURE. So check
the wanted_fields also in here. */
if (plain_bodystructure)
else if (field == MAIL_CACHE_IMAP_BODYSTRUCTURE ||
} else {
}
if (cache_bodystructure) {
} else {
}
/* normally don't cache both BODY and BODYSTRUCTURE, but do it
if BODY is forced to be cached */
if (plain_bodystructure ||
cache_body = FALSE;
else if (field == MAIL_CACHE_IMAP_BODY) {
} else {
}
if (cache_body) {
}
}
{
unsigned int cache_field;
switch (field) {
case MAIL_CACHE_SENT_DATE:
break;
case MAIL_CACHE_RECEIVED_DATE:
break;
case MAIL_CACHE_SAVE_DATE:
break;
break;
break;
case MAIL_CACHE_BODY_SNIPPET:
break;
default:
i_unreached();
}
return FALSE;
/* If a field has been explicitly requested to be fetched, it's
included in data.cache_fetch_fields. In that case use _can_add() to
add it to the cache file if at all possible. Otherwise, use
_want_add() to use previous caching decisions. */
} else {
}
}
{
if (index_mail_write_body_snippet(mail) < 0)
return;
}
}
}
{
};
unsigned int i;
/* store the virtual size in index if
extension for it exists or
extension for box virtual size exists and
size fits and is present and
size is not cached or
cached size differs
*/
/* vsize = 0 means it's not present in index, consult cache.
we store vsize for every +4GB-1 mail to cache because
index can only hold 2^32-1 size. Cache will not be used
when vsize is stored in index. */
}
/* it's already in index, so don't update cache */
}
for (i = 0; i < N_ELEMENTS(size_fields); i++) {
}
}
}
{
};
unsigned int i;
uint32_t t;
for (i = 0; i < N_ELEMENTS(date_fields); i++) {
t = dates[i];
&t, sizeof(t));
}
}
(void)index_mail_cache_sent_date(mail);
}
static struct message_part *
{
/* use any text/ part, even if we don't know what exactly
it is. */
return parts;
}
/* for now we support only text Content-Types */
return NULL;
}
return part;
else
}
}
}
/* find the first usable MIME part */
return subpart;
}
return NULL;
}
{
int ret;
return 0;
}
return -1;
if (ret == 0)
return ret;
}
static int
{
int ret;
if (parser_input == NULL) {
} else {
/* do one final read, which verifies that the message
size is correct. */
i_unreached();
}
/* EPIPE = input already closed. allow the caller to
decide if that is an error or not. (for example we
could be coming here from IMAP APPEND when IMAP
client has closed the connection too early. we
don't want to log an error in that case.) */
if (parser_input->stream_errno != 0 &&
ret = -1;
}
}
if (ret <= 0) {
if (ret == 0) {
}
return -1;
}
}
/* if we're here because we aborted parsing, don't get any
further or we may crash while generating output from
incomplete data */
return 0;
}
(void)get_cached_msgpart_sizes(mail);
return 0;
}
{
}
{
return 0;
return -1;
}
{
}
{
/* was the mail just expunged? we could get here especially if
external attachments are used and the attachment is deleted
before we've opened the file. */
return;
}
"read(%s) failed: %s (read reason=%s)",
}
enum index_cache_field field)
{
int ret;
if (data->save_bodystructure_body) {
/* bodystructure header is parsed, we want the body's mime
headers too */
} else {
}
ret = -1;
return ret;
}
{
}
{
unsigned int block_size;
}
struct message_size *hdr_size,
struct message_size *body_size,
{
bool has_nuls;
int ret;
i_debug("Mailbox %s: Opened mail UID=%u because: %s",
}
if (!data->initialized_wrapper_stream &&
}
if (!data->destroy_callback_set) {
/* do this only once in case a plugin changes the stream.
otherwise the check would break. */
}
(void)get_cached_msgpart_sizes(mail);
if (!data->hdr_size_set) {
(void)get_cached_parts(mail);
return -1;
} else {
&has_nuls) < 0) {
return -1;
}
}
}
}
if (!data->body_size_set)
if (!data->body_size_set) {
if (index_mail_parse_body(mail, 0) < 0)
return -1;
} else {
&has_nuls) < 0) {
return -1;
}
}
}
}
}
if (ret < 0)
return -1;
return 0;
}
enum index_cache_field field)
{
/* we have everything parsed already, but just not written to
a string */
} else {
if (data->save_bodystructure_header ||
field == MAIL_CACHE_BODY_SNIPPET) {
/* we haven't parsed the header yet */
const char *reason =
(void)get_cached_parts(mail);
return -1;
}
}
return -1;
}
/* if we didn't want to have the body(structure) cached,
it's still not written. */
switch (field) {
case MAIL_CACHE_IMAP_BODY:
}
break;
}
break;
case MAIL_CACHE_BODY_SNIPPET:
if (index_mail_write_body_snippet(mail) < 0)
return -1;
}
break;
default:
i_unreached();
}
return 0;
}
static void
bool extended)
{
if (extended)
}
static int
{
const unsigned int cache_field =
}
return 0;
}
/* reuse the IMAP bodystructure parsing code to get all the useful
headers that we need. */
return -1;
return 0;
}
{
const unsigned int body_cache_field =
const unsigned int bodystructure_cache_field =
const char *error;
return TRUE;
}
get_cached_parts(mail)) {
return TRUE;
}
/* 2) get BODY if it exists */
return TRUE;
}
/* 3) get it using BODYSTRUCTURE if it exists */
str_truncate(str, 0);
/* broken, continue.. */
"Invalid BODYSTRUCTURE %s: %s",
} else {
return TRUE;
}
}
return FALSE;
}
const char **value_r)
{
const unsigned int bodystructure_cache_field =
return TRUE;
}
get_cached_parts(mail)) {
return TRUE;
}
return TRUE;
}
return FALSE;
}
{
switch (field) {
case MAIL_FETCH_IMAP_BODY:
return 0;
/* parse body structure, and save BODY/BODYSTRUCTURE
depending on what we want cached */
return -1;
return 0;
return 0;
return -1;
return 0;
case MAIL_FETCH_IMAP_ENVELOPE:
if (index_mail_headers_get_envelope(mail) < 0)
return -1;
}
return 0;
case MAIL_FETCH_FROM_ENVELOPE:
return 0;
case MAIL_FETCH_BODY_SNIPPET:
case MAIL_FETCH_STORAGE_ID:
case MAIL_FETCH_UIDL_BACKEND:
case MAIL_FETCH_GUID:
case MAIL_FETCH_HEADER_MD5:
case MAIL_FETCH_POP3_ORDER:
case MAIL_FETCH_REFCOUNT:
*value_r = "";
return 0;
case MAIL_FETCH_MAILBOX_NAME:
return 0;
default:
i_unreached();
}
}
struct mail **real_mail_r)
{
*real_mail_r = mail;
return 0;
}
struct mail *
struct mailbox_header_lookup_ctx *wanted_headers)
{
}
{
}
struct mailbox_transaction_context *t,
struct mailbox_header_lookup_ctx *wanted_headers)
{
sizeof(void *), 5);
t->mail_ref_count++;
if (wanted_headers != NULL) {
}
}
{
const char *error;
}
/* we're replacing the stream with a new one. it's
allowed to have references until the mail is closed
(but we can't really check that) */
}
/* there must be no references to the mail when the
mail is being closed. */
if (!closing)
i_panic("Input stream %s unexpectedly has references",
}
}
}
{
}
{
}
}
{
}
{
/* mail_set_seq*() hasn't been called yet, or is being called
right now. Don't reset anything yet. We especially don't
want to reset wanted_fields or wanted_headers so that
mail_add_temp_wanted_fields() can be called by plugins
before mail_set_seq_saving() for
mail_save_context.dest_mail. */
return;
}
/* make sure old mail isn't visible in the event anymore even if it's
attempted to be used. */
/* If uid == 0 but seq != 0, we came here from saving a (non-mbox)
message. If that happens, don't bother checking if anything should
be cached since it was already checked. Also by now the transaction
may have already been rollbacked and seq point to a nonexistent
message. */
}
/* Notify cache that the mail is no longer open. This mainly helps
with INDEX=MEMORY to keep all data added with mail_cache_add() in
memory until this point. */
}
{
const unsigned int cache_field_envelope =
unsigned int cache_field_hdr;
return;
}
/* if "imap.envelope" is cached, that's all we need */
return;
/* don't waste time doing full checks for all required
headers. assume that if we have "hdr.message-id" cached,
we don't need to parse the header. */
"hdr.message-id");
if (cache_field_hdr == UINT_MAX ||
}
{
/* mail_add_temp_wanted_fields() called before mail_set_seq*().
We'll allow this, since it can be useful for plugins to
call it for mail_save_context.dest_mail. This function
is called again in mail_set_seq*(). */
return;
}
MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0 &&
&data->cache_flags,
sizeof(data->cache_flags));
_mail->has_no_nuls =
/* we currently don't forcibly set the nul state. if it's not
already cached, the caller can figure out itself what to
do when neither is set */
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
/* we need either imap.body or imap.bodystructure */
const unsigned int cache_field1 =
const unsigned int cache_field2 =
cache_field1) <= 0 &&
cache_field2) <= 0) {
}
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
MAIL_FETCH_STREAM_BODY)) != 0) {
}
}
{
/* see index_mail_update_access_parts_pre() */
return;
}
/* when mail_prefetch_count>1, at this point we've started the
prefetching to all the mails and we're now starting to access the
first mail. */
if (data->access_part != 0) {
/* open stream immediately to set expunged flag if
it's already lost */
/* open the stream only if we didn't get here from
mailbox_save_init() */
else
}
}
}
{
if (!saving)
return;
/* we started saving a mail, aborted it, and now we're saving
another mail with the same sequence. make sure the mail
gets reset. */
}
return;
}
if (!mail->search_mail) {
} else {
/* searching code will call the
index_mail_update_access_parts_*() after we know the mail is
actually wanted to be fetched. */
}
}
{
/* HAVE_POSIX_FADVISE alone isn't enough for CentOS 4.9 */
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
int fd;
/* we're handling only file-per-msg storages for now. */
return TRUE;
}
/* everything we need is cached */
return TRUE;
}
return TRUE;
}
/* tell OS to start reading the file into memory */
if (fd != -1) {
len = 0;
else
i_error("posix_fadvise(%s) failed: %m",
}
}
#endif
}
{
return TRUE;
} else {
return FALSE;
}
}
enum mail_fetch_field fields,
struct mailbox_header_lookup_ctx *headers)
{
unsigned int i;
/* keep old ones */
} else {
/* merge headers */
}
/* Don't call _post(), which would try to open the stream. It should be
enough to delay the opening until it happens anyway.
Otherwise there's not really any good place to call this in the
plugins: set_seq() call get_stream() internally, which can already
start parsing the headers, so it's too late. If we use get_stream()
and there's a _post() call here, it gets into infinite loop. The
loop could probably be prevented in some way, but it's probably
better to eventually try to remove the _post() call entirely
everywhere. */
}
{
}
{
/* make sure mailbox_search_*() users don't try to free the mail
directly */
}
{
&block) > 0) {
continue;
} else {
}
}
}
bool success)
{
if (!success) {
/* we're going to delete this mail anyway,
don't bother trying to update cache file */
/* we didn't even start cache parsing */
return;
}
}
/* This is needed with 0 byte mails to get hdr=NULL call done. */
/* this save_date may not be exactly the same as what we get
in future, but then again neither mbox nor maildir
guarantees it anyway. */
}
}
static bool
enum mail_flags pvt_flags)
{
if (!index_mail_get_pvt(_mail))
return FALSE;
return FALSE;
/* see if the flags actually change anything */
switch (modify_type) {
case MODIFY_ADD:
case MODIFY_REPLACE:
return old_pvt_flags != pvt_flags;
case MODIFY_REMOVE:
return (old_pvt_flags & pvt_flags) != 0;
}
i_unreached();
}
enum mail_flags flags)
{
/* mailbox has private flags */
flags &= ~pvt_flags_mask;
}
}
if (!update_modseq) {
/* no forced modseq update */
} else if (modify_type == MODIFY_REMOVE) {
/* add the modseq update separately */
} else {
/* add as part of the flag updates */
}
modify_type, flags);
}
struct mail_keywords *keywords)
{
/* clear the keywords array so the next mail_get_keywords()
returns the updated keywords. don't free the array, because
then any existing mail_get_keywords() return values would
point to broken data. this won't leak memory because the
array is allocated from mail's memory pool. */
}
}
{
}
{
return;
}
{
const char *value;
else {
}
}
{
if (parse_body) {
(void)index_mail_parse_body(imail, 0);
}
}
}
{
const char *str;
/* already cached this mail (we should get here only if FTS
plugin decreased the first precached seq) */
return;
}
if ((cache & MAIL_FETCH_RECEIVED_DATE) != 0)
if ((cache & MAIL_FETCH_SAVE_DATE) != 0)
if ((cache & MAIL_FETCH_VIRTUAL_SIZE) != 0)
if ((cache & MAIL_FETCH_PHYSICAL_SIZE) != 0)
if ((cache & MAIL_FETCH_UIDL_BACKEND) != 0)
if ((cache & MAIL_FETCH_POP3_ORDER) != 0)
if ((cache & MAIL_FETCH_GUID) != 0)
}
static void
{
unsigned int idx;
&idx)) {
}
}
enum mail_fetch_field field,
const char *reason)
{
const char *field_name;
switch ((int)field) {
case 0:
field_name = "fields";
break;
case MAIL_FETCH_PHYSICAL_SIZE:
field_name = "physical size";
break;
case MAIL_FETCH_VIRTUAL_SIZE:
field_name = "virtual size";
break;
case MAIL_FETCH_MESSAGE_PARTS:
field_name = "MIME parts";
break;
case MAIL_FETCH_IMAP_BODY:
field_name = "IMAP BODY";
break;
field_name = "IMAP BODYSTRUCTURE";
break;
default:
}
/* make sure we don't cache invalid values */
"BUG: Broken %s found while saving a new mail: %s",
field_name, reason);
} else if (reason[0] == '\0') {
"Broken %s in mailbox %s",
} else {
"Broken %s in mailbox %s: %s",
}
}
{
return 0;
}
{
}
}
{
const char *cache_reason =
}