index-mail.c revision 21aaa6affb9f134112b75b5db737309fc35ef1cf
/* Copyright (c) 2002-2016 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "buffer.h"
#include "ioloop.h"
#include "istream.h"
#include "hex-binary.h"
#include "str.h"
#include "mailbox-recent-flags.h"
#include "message-date.h"
#include "message-part-serialize.h"
#include "message-parser.h"
#include "message-snippet.h"
#include "imap-bodystructure.h"
#include "imap-envelope.h"
#include "mail-cache.h"
#include "mail-index-modseq.h"
#include "index-storage.h"
#include "istream-mail.h"
#include "index-mail.h"
#include <fcntl.h>
#define BODY_SNIPPET_ALGO_V1 "1"
#define BODY_SNIPPET_MAX_CHARS 100
{ .name = "flags",
.field_size = sizeof(uint32_t) },
.field_size = sizeof(struct mail_sent_date) },
{ .name = "date.received",
.field_size = sizeof(uint32_t) },
.field_size = sizeof(uint32_t) },
{ .name = "size.virtual",
.field_size = sizeof(uoff_t) },
{ .name = "size.physical",
.field_size = sizeof(uoff_t) },
{ .name = "imap.body",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "imap.bodystructure",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "imap.envelope",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "pop3.uidl",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "pop3.order",
.field_size = sizeof(uint32_t) },
{ .name = "guid",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "mime.parts",
{ .name = "binary.parts",
/* FIXME: for now need to update get_metadata_precache_fields() in
index-status.c when adding more fields. those fields should probably
just be moved here to the same struct. */
};
enum index_cache_field field);
unsigned int field_idx)
{
int ret;
if (ret > 0)
return ret;
}
{
const unsigned int field_idx =
}
{
struct message_part *parts;
const char *error;
return NULL;
"Corrupted cached mime.parts data: %s (parts=%s)",
}
return parts;
}
{
return TRUE;
return TRUE;
}
}
return FALSE;
}
{
struct message_part *part;
T_BEGIN {
} T_END;
return FALSE;
/* we know the NULs now, update them */
if (message_parts_have_nuls(part)) {
} else {
}
return TRUE;
}
{
const char *parts_str;
parts_str = "";
else
"Cached MIME parts don't match message during parsing: %s (parts=%s)",
}
enum index_cache_field field,
{
bool ret;
else {
}
return ret;
}
{
}
{
return TRUE;
/* no private view (set by view syncing) -> no private flags */
return FALSE;
}
/* mail is still being saved, it has no private flags yet */
return FALSE;
}
}
{
const struct mail_index_record *rec;
flags |= MAIL_RECENT;
if (index_mail_get_pvt(_mail)) {
/* mailbox has private flags */
flags &= ~pvt_flags_mask;
}
return flags;
}
{
}
{
return 0;
}
{
const char *const *names;
const unsigned int *keyword_indexes;
unsigned int i, count, names_count;
(void)index_mail_get_keyword_indexes(_mail);
for (i = 0; i < count; i++) {
const char *name;
}
/* end with NULL */
}
const ARRAY_TYPE(keyword_indexes) *
{
&data->keyword_indexes);
}
return &data->keyword_indexes;
}
{
return 0;
}
const char *reason =
return -1;
}
if (index_mail_parse_body(mail, 0) < 0)
return -1;
return 0;
}
{
uint32_t t;
&t, sizeof(t)))
data->received_date = t;
}
}
{
uint32_t t;
&t, sizeof(t)))
}
}
{
const char *str;
time_t t;
return 0;
return ret;
if (ret == 0 ||
!message_date_parse((const unsigned char *)str,
/* 0 = not found / invalid */
t = 0;
tz = 0;
}
return 0;
}
{
struct mail_sent_date sentdate;
return 0;
}
if (index_mail_cache_sent_date(mail) < 0)
return -1;
return 0;
}
{
(void)get_cached_parts(mail);
}
}
{
&size))
else {
if (!get_cached_msgpart_sizes(mail))
return FALSE;
}
}
}
return TRUE;
}
{
if (!data->hdr_size_set)
return;
/* we've already called get_cached_msgpart_sizes() and it didn't work.
try to do this by using cached virtual size and a quick physical
size lookup. */
return;
if (!data->body_size_set) {
/* get the physical size, but not if it requires reading
through the whole message */
/* we should have everything now. try again. */
}
}
}
{
return 0;
return -1;
return 0;
}
{
&size))
else
(void)get_cached_msgpart_sizes(mail);
}
}
{
}
{
const struct mail_index_header *hdr;
if (set->mail_cache_min_mail_count > 0) {
/* First check if we've configured caching not to be used with
low enough message count. */
return;
}
}
}
struct message_header_line *hdr,
{
}
{
const unsigned int cache_field_body =
const unsigned int cache_field_bodystructure =
MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0)
return TRUE;
return TRUE;
return TRUE;
return FALSE;
}
{
unsigned int cache_flags_idx;
bool want_cached;
if (data->parsed_bodystructure &&
/* we need message_parts cached to be able to
actually use it in BODY/BODYSTRUCTURE reply */
want_cached = TRUE;
}
/* cache flags should never get unset as long as the message doesn't
change, but try to handle it anyway */
} else {
}
&cache_flags, sizeof(cache_flags));
}
}
{
const unsigned int cache_field =
if (data->messageparts_saved_to_cache ||
cache_field) != 0) {
/* already cached */
return;
}
/* we never want it cached */
return;
}
if (decision == MAIL_CACHE_DECISION_NO &&
!data->save_message_parts &&
/* we didn't really care about the message parts themselves,
just wanted to use something that depended on it */
return;
}
T_BEGIN {
} T_END;
}
static void
enum index_cache_field field)
{
const unsigned int cache_field_parts =
const unsigned int cache_field_body =
const unsigned int cache_field_bodystructure =
enum mail_cache_decision_type dec;
bool bodystructure_cached = FALSE;
bool plain_bodystructure = FALSE;
bool cache_bodystructure, cache_body;
if (data->messageparts_saved_to_cache ||
/* cached it as flag + message_parts */
}
}
if (!data->parsed_bodystructure)
return;
/* If BODY is fetched first but BODYSTRUCTURE is also wanted, we don't
normally want to first cache BODY and then BODYSTRUCTURE. So check
the wanted_fields also in here. */
if (plain_bodystructure)
else if (field == MAIL_CACHE_IMAP_BODYSTRUCTURE ||
} else {
}
if (cache_bodystructure) {
} else {
}
/* normally don't cache both BODY and BODYSTRUCTURE, but do it
if BODY is forced to be cached */
if (plain_bodystructure ||
cache_body = FALSE;
else if (field == MAIL_CACHE_IMAP_BODY) {
} else {
}
if (cache_body) {
}
}
static bool
{
enum mail_fetch_field fetch_field;
unsigned int cache_field;
switch (field) {
case MAIL_CACHE_SENT_DATE:
break;
case MAIL_CACHE_RECEIVED_DATE:
break;
case MAIL_CACHE_SAVE_DATE:
break;
break;
break;
case MAIL_CACHE_BODY_SNIPPET:
break;
default:
i_unreached();
}
return FALSE;
} else {
}
}
{
}
}
{
static enum index_cache_field size_fields[] = {
};
unsigned int i;
for (i = 0; i < N_ELEMENTS(size_fields); i++) {
}
}
}
{
static enum index_cache_field date_fields[] = {
};
unsigned int i;
uint32_t t;
for (i = 0; i < N_ELEMENTS(date_fields); i++) {
t = dates[i];
&t, sizeof(t));
}
}
(void)index_mail_cache_sent_date(mail);
}
static struct message_part *
{
struct message_part *part;
/* use any text/ part, even if we don't know what exactly
it is. */
return parts;
}
/* for now we support only text Content-Types */
return NULL;
}
struct message_part_body_data *sub_body_data =
return part;
else
}
}
}
/* find the first usable MIME part */
struct message_part *subpart =
return subpart;
}
return NULL;
}
{
struct message_part *part;
int ret;
return 0;
}
return -1;
if (ret == 0)
return ret;
}
static int
{
int ret;
if (parser_input == NULL) {
} else {
/* do one final read, which verifies that the message
size is correct. */
i_unreached();
}
/* EPIPE = input already closed. allow the caller to
decide if that is an error or not. (for example we
could be coming here from IMAP APPEND when IMAP
client has closed the connection too early. we
don't want to log an error in that case.) */
if (parser_input->stream_errno != 0 &&
ret = -1;
}
}
if (ret <= 0) {
if (ret == 0) {
}
return -1;
}
}
if (index_mail_write_body_snippet(mail) < 0)
return -1;
}
/* if we're here because we aborted parsing, don't get any
further or we may crash while generating output from
incomplete data */
return 0;
}
(void)get_cached_msgpart_sizes(mail);
return 0;
}
{
}
{
return 0;
return -1;
}
{
}
{
/* was the mail just expunged? we could get here especially if
external attachments are used and the attachment is deleted
before we've opened the file. */
return;
}
"read(%s) failed: %s (uid=%u, box=%s, read reason=%s)",
}
enum index_cache_field field)
{
int ret;
if (data->save_bodystructure_body) {
/* bodystructure header is parsed, we want the body's mime
headers too */
} else {
*null_message_part_header_callback, (void *)NULL);
}
ret = -1;
return ret;
}
{
}
{
unsigned int block_size;
}
struct message_size *hdr_size,
struct message_size *body_size,
{
bool has_nuls;
int ret;
i_debug("Mailbox %s: Opened mail UID=%u because: %s",
}
if (!data->initialized_wrapper_stream &&
}
if (!data->destroy_callback_set) {
/* do this only once in case a plugin changes the stream.
otherwise the check would break. */
}
(void)get_cached_msgpart_sizes(mail);
if (!data->hdr_size_set) {
(void)get_cached_parts(mail);
return -1;
} else {
&has_nuls) < 0) {
return -1;
}
}
}
}
if (!data->body_size_set)
if (!data->body_size_set) {
if (index_mail_parse_body(mail, 0) < 0)
return -1;
} else {
&has_nuls) < 0) {
return -1;
}
}
}
}
}
if (ret < 0)
return -1;
return 0;
}
enum index_cache_field field)
{
/* we have everything parsed already, but just not written to
a string */
} else {
if (data->save_bodystructure_header ||
field == MAIL_CACHE_BODY_SNIPPET) {
/* we haven't parsed the header yet */
const char *reason =
(void)get_cached_parts(mail);
return -1;
}
}
return -1;
}
/* if we didn't want to have the body(structure) cached,
it's still not written. */
switch (field) {
case MAIL_CACHE_IMAP_BODY:
}
break;
}
break;
case MAIL_CACHE_BODY_SNIPPET:
break;
default:
i_unreached();
}
return 0;
}
static void
bool extended)
{
if (extended)
}
static int
{
const unsigned int cache_field =
}
return 0;
}
/* reuse the IMAP bodystructure parsing code to get all the useful
headers that we need. */
return -1;
return 0;
}
{
const char *error;
switch (field) {
case MAIL_FETCH_IMAP_BODY: {
const unsigned int body_cache_field =
const unsigned int bodystructure_cache_field =
return 0;
}
/* 1) use plain-7bit-ascii flag if it exists
2) get BODY if it exists
3) get it using BODYSTRUCTURE if it exists
4) parse body structure, and save BODY/BODYSTRUCTURE
depending on what we want cached */
get_cached_parts(mail)) {
body_cache_field) > 0)
bodystructure_cache_field) > 0) {
str_truncate(str, 0);
/* broken, continue.. */
"Invalid BODYSTRUCTURE %s: %s",
} else {
}
}
MAIL_CACHE_IMAP_BODY) < 0)
return -1;
}
return 0;
}
case MAIL_FETCH_IMAP_BODYSTRUCTURE: {
const unsigned int bodystructure_cache_field =
return 0;
}
get_cached_parts(mail)) {
bodystructure_cache_field) > 0) {
} else {
return -1;
}
return 0;
}
case MAIL_FETCH_IMAP_ENVELOPE:
if (index_mail_headers_get_envelope(mail) < 0)
return -1;
}
return 0;
case MAIL_FETCH_FROM_ENVELOPE:
return 0;
case MAIL_FETCH_BODY_SNIPPET:
case MAIL_FETCH_STORAGE_ID:
case MAIL_FETCH_UIDL_BACKEND:
case MAIL_FETCH_GUID:
case MAIL_FETCH_HEADER_MD5:
case MAIL_FETCH_POP3_ORDER:
case MAIL_FETCH_REFCOUNT:
*value_r = "";
return 0;
case MAIL_FETCH_MAILBOX_NAME:
return 0;
default:
i_unreached();
return -1;
}
}
struct mail **real_mail_r)
{
*real_mail_r = mail;
return 0;
}
struct mail *
index_mail_alloc(struct mailbox_transaction_context *t,
struct mailbox_header_lookup_ctx *wanted_headers)
{
struct index_mail *mail;
}
struct mailbox_transaction_context *t,
struct mailbox_header_lookup_ctx *wanted_headers)
{
sizeof(void *), 5);
t->mail_ref_count++;
if (wanted_headers != NULL) {
}
}
{
struct message_part *parts;
const char *error;
}
/* we're replacing the stream with a new one. it's
allowed to have references until the mail is closed
(but we can't really check that) */
}
/* there must be no references to the mail when the
mail is being closed. */
if (!closing)
i_panic("Input stream %s unexpectedly has references",
}
}
}
{
}
{
}
}
{
/* If uid == 0 but seq != 0, we came here from saving a (non-mbox)
message. If that happens, don't bother checking if anything should
be cached since it was already checked. Also by now the transaction
may have already been rollbacked and seq point to a nonexistent
message. */
}
}
{
const unsigned int cache_field_envelope =
unsigned int cache_field_hdr;
return;
}
/* if "imap.envelope" is cached, that's all we need */
return;
/* don't waste time doing full checks for all required
headers. assume that if we have "hdr.message-id" cached,
we don't need to parse the header. */
"hdr.message-id");
if (cache_field_hdr == UINT_MAX ||
}
{
MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0 &&
&data->cache_flags,
sizeof(data->cache_flags));
_mail->has_no_nuls =
/* we currently don't forcibly set the nul state. if it's not
already cached, the caller can figure out itself what to
do when neither is set */
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
/* we need either imap.body or imap.bodystructure */
const unsigned int cache_field1 =
const unsigned int cache_field2 =
cache_field1) <= 0 &&
cache_field2) <= 0) {
}
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
MAIL_FETCH_STREAM_BODY)) != 0) {
}
}
{
const struct mail_index_header *hdr;
/* when mail_prefetch_count>1, at this point we've started the
prefetching to all the mails and we're now starting to access the
first mail. */
if (data->access_part != 0) {
/* open stream immediately to set expunged flag if
it's already lost */
/* open the stream only if we didn't get here from
mailbox_save_init() */
else
}
}
}
{
if (!saving)
return;
/* we started saving a mail, aborted it, and now we're saving
another mail with the same sequence. make sure the mail
gets reset. */
}
return;
}
if (!mail->search_mail) {
} else {
/* searching code will call the
index_mail_update_access_parts_*() after we know the mail is
actually wanted to be fetched. */
}
}
{
/* HAVE_POSIX_FADVISE alone isn't enough for CentOS 4.9 */
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
int fd;
/* we're handling only file-per-msg storages for now. */
return TRUE;
}
/* everything we need is cached */
return TRUE;
}
return TRUE;
}
/* tell OS to start reading the file into memory */
if (fd != -1) {
len = 0;
else
i_error("posix_fadvise(%s) failed: %m",
}
}
#endif
}
{
return TRUE;
} else {
return FALSE;
}
}
enum mail_fetch_field fields,
struct mailbox_header_lookup_ctx *headers)
{
unsigned int i;
/* keep old ones */
} else {
/* merge headers */
}
}
{
}
{
struct mailbox_header_lookup_ctx *headers_ctx =
/* make sure mailbox_search_*() users don't try to free the mail
directly */
if (headers_ctx != NULL)
}
{
struct message_block block;
&block) > 0) {
continue;
} else {
}
}
}
bool success)
{
if (!success) {
/* we're going to delete this mail anyway,
don't bother trying to update cache file */
/* we didn't even start cache parsing */
return;
}
}
/* This is needed with 0 byte mails to get hdr=NULL call done. */
/* this save_date may not be exactly the same as what we get
in future, but then again neither mbox nor maildir
guarantees it anyway. */
}
}
{
const struct mail_index_header *hdr;
}
}
static bool
enum mail_flags pvt_flags)
{
const struct mail_index_record *rec;
enum mail_flags old_pvt_flags;
if (!index_mail_get_pvt(_mail))
return FALSE;
return FALSE;
/* see if the flags actually change anything */
switch (modify_type) {
case MODIFY_ADD:
case MODIFY_REPLACE:
return old_pvt_flags != pvt_flags;
case MODIFY_REMOVE:
return (old_pvt_flags & pvt_flags) != 0;
}
i_unreached();
}
enum mail_flags flags)
{
bool update_modseq = FALSE;
if ((flags & MAIL_RECENT) == 0 &&
/* mailbox has private flags */
flags &= ~pvt_flags_mask;
}
}
if (!update_modseq) {
/* no forced modseq update */
} else if (modify_type == MODIFY_REMOVE) {
/* add the modseq update separately */
} else {
/* add as part of the flag updates */
}
modify_type, flags);
}
struct mail_keywords *keywords)
{
/* clear the keywords array so the next mail_get_keywords()
returns the updated keywords. don't free the array, because
then any existing mail_get_keywords() return values would
point to broken data. this won't leak memory because the
array is allocated from mail's memory pool. */
}
}
{
}
{
return;
}
{
const char *value;
else {
}
}
{
if (parse_body) {
(void)index_mail_parse_body(imail, 0);
}
}
}
{
enum mail_fetch_field cache;
const char *str;
/* already cached this mail (we should get here only if FTS
plugin decreased the first precached seq) */
return;
}
if ((cache & MAIL_FETCH_RECEIVED_DATE) != 0)
if ((cache & MAIL_FETCH_SAVE_DATE) != 0)
if ((cache & MAIL_FETCH_VIRTUAL_SIZE) != 0)
if ((cache & MAIL_FETCH_PHYSICAL_SIZE) != 0)
if ((cache & MAIL_FETCH_UIDL_BACKEND) != 0)
if ((cache & MAIL_FETCH_POP3_ORDER) != 0)
if ((cache & MAIL_FETCH_GUID) != 0)
}
enum mail_fetch_field field,
const char *reason)
{
const char *field_name;
switch ((int)field) {
case 0:
field_name = "fields";
break;
case MAIL_FETCH_PHYSICAL_SIZE:
field_name = "physical size";
break;
case MAIL_FETCH_VIRTUAL_SIZE:
field_name = "virtual size";
break;
case MAIL_FETCH_MESSAGE_PARTS:
field_name = "MIME parts";
break;
case MAIL_FETCH_IMAP_BODY:
field_name = "IMAP BODY";
break;
field_name = "IMAP BODYSTRUCTURE";
break;
default:
}
/* make sure we don't cache invalid values */
if (reason[0] == '\0') {
"Broken %s for mail UID %u in mailbox %s",
} else {
"Broken %s for mail UID %u in mailbox %s: %s",
}
}
{
return 0;
}
{
return;
}
}
{
const char *cache_reason =
}