index-mail.c revision 0000bf8b0e0fa31e875ec55b585e9f7291d6f3d0
/* Copyright (c) 2002-2012 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "buffer.h"
#include "ioloop.h"
#include "istream.h"
#include "str.h"
#include "message-date.h"
#include "message-part-serialize.h"
#include "message-parser.h"
#include "imap-bodystructure.h"
#include "imap-envelope.h"
#include "mail-cache.h"
#include "mail-index-modseq.h"
#include "index-storage.h"
#include "istream-mail.h"
#include "index-mail.h"
#include <fcntl.h>
{ .name = "flags",
.field_size = sizeof(uint32_t) },
.field_size = sizeof(struct mail_sent_date) },
{ .name = "date.received",
.field_size = sizeof(uint32_t) },
.field_size = sizeof(uint32_t) },
{ .name = "size.virtual",
.field_size = sizeof(uoff_t) },
{ .name = "size.physical",
.field_size = sizeof(uoff_t) },
{ .name = "imap.body",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "imap.bodystructure",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "imap.envelope",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "pop3.uidl",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "guid",
.type = MAIL_CACHE_FIELD_STRING },
{ .name = "mime.parts",
{ .name = "binary.parts",
};
enum index_cache_field field);
unsigned int field_idx)
{
int ret;
if (ret > 0)
return ret;
}
{
const unsigned int field_idx =
struct message_part *parts;
const char *error;
int ret;
if (ret <= 0)
return NULL;
"Corrupted cached mime.parts data (%s)", error);
}
return parts;
}
{
struct message_part *part;
T_BEGIN {
} T_END;
return FALSE;
/* we know the NULs now, update them */
} else {
}
return TRUE;
}
enum index_cache_field field,
{
int ret;
else {
}
return ret;
}
{
}
{
return TRUE;
/* no private view (set by view syncing) -> no private flags */
return FALSE;
}
}
{
const struct mail_index_record *rec;
flags |= MAIL_RECENT;
if (index_mail_get_pvt(_mail)) {
/* mailbox has private flags */
flags &= ~pvt_flags_mask;
}
return flags;
}
{
}
{
const char *const *names;
const unsigned int *keyword_indexes;
unsigned int i, count, names_count;
(void)index_mail_get_keyword_indexes(_mail);
for (i = 0; i < count; i++) {
const char *name;
}
/* end with NULL */
}
const ARRAY_TYPE(keyword_indexes) *
{
&data->keyword_indexes);
}
return &data->keyword_indexes;
}
{
return 0;
}
return -1;
}
if (index_mail_parse_body(mail, 0) < 0)
return -1;
return 0;
}
{
uint32_t t;
&t, sizeof(t)))
data->received_date = t;
}
}
{
uint32_t t;
&t, sizeof(t)))
}
}
{
const char *str;
time_t t;
return 0;
return ret;
if (ret == 0 ||
!message_date_parse((const unsigned char *)str,
/* 0 = not found / invalid */
t = 0;
tz = 0;
}
return 0;
}
{
struct mail_sent_date sentdate;
return 0;
}
if (index_mail_cache_sent_date(mail) < 0)
return -1;
return 0;
}
{
(void)get_cached_parts(mail);
}
}
{
&size))
else {
if (!get_cached_msgpart_sizes(mail))
return FALSE;
}
}
}
return TRUE;
}
{
if (!data->hdr_size_set)
return;
/* we've already called get_cached_msgpart_sizes() and it didn't work.
try to do this by using cached virtual size and a quick physical
size lookup. */
return;
if (!data->body_size_set) {
return;
/* we should have everything now. try again. */
}
}
{
return 0;
return -1;
return 0;
}
{
&size))
else
(void)get_cached_msgpart_sizes(mail);
}
}
{
}
{
const struct mail_index_header *hdr;
if (set->mail_cache_min_mail_count > 0) {
/* First check if we've configured caching not to be used with
low enough message count. */
return;
}
}
}
struct message_header_line *hdr,
{
}
{
const unsigned int cache_field_body =
const unsigned int cache_field_bodystructure =
MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0)
return TRUE;
return TRUE;
return TRUE;
return FALSE;
}
{
unsigned int cache_flags_idx;
bool want_cached;
if (data->parsed_bodystructure &&
/* we need message_parts cached to be able to
actually use it in BODY/BODYSTRUCTURE reply */
want_cached = TRUE;
}
/* cache flags should never get unset as long as the message doesn't
change, but try to handle it anyway */
} else {
}
&cache_flags, sizeof(cache_flags));
}
}
{
const unsigned int cache_field =
if (data->messageparts_saved_to_cache ||
cache_field) != 0) {
/* already cached */
return;
}
/* we never want it cached */
return;
}
if (decision == MAIL_CACHE_DECISION_NO &&
!data->save_message_parts &&
/* we didn't really care about the message parts themselves,
just wanted to use something that depended on it */
return;
}
T_BEGIN {
} T_END;
}
static void
enum index_cache_field field)
{
const unsigned int cache_field_parts =
const unsigned int cache_field_body =
const unsigned int cache_field_bodystructure =
enum mail_cache_decision_type dec;
bool bodystructure_cached = FALSE;
bool plain_bodystructure = FALSE;
bool cache_bodystructure, cache_body;
if (data->messageparts_saved_to_cache ||
/* cached it as flag + message_parts */
}
}
if (!data->parsed_bodystructure)
return;
/* If BODY is fetched first but BODYSTRUCTURE is also wanted, we don't
normally want to first cache BODY and then BODYSTRUCTURE. So check
the wanted_fields also in here. */
if (plain_bodystructure)
else if (field == MAIL_CACHE_IMAP_BODYSTRUCTURE ||
} else {
}
if (cache_bodystructure) {
} else {
}
/* normally don't cache both BODY and BODYSTRUCTURE, but do it
if BODY is forced to be cached */
if (plain_bodystructure ||
cache_body = FALSE;
else if (field == MAIL_CACHE_IMAP_BODY) {
} else {
}
if (cache_body) {
}
}
static bool
{
enum mail_fetch_field fetch_field;
unsigned int cache_field;
switch (field) {
case MAIL_CACHE_SENT_DATE:
break;
case MAIL_CACHE_RECEIVED_DATE:
break;
case MAIL_CACHE_SAVE_DATE:
break;
break;
break;
default:
i_unreached();
}
return FALSE;
} else {
}
}
{
static enum index_cache_field size_fields[] = {
};
unsigned int i;
for (i = 0; i < N_ELEMENTS(size_fields); i++) {
}
}
}
{
static enum index_cache_field date_fields[] = {
};
unsigned int i;
uint32_t t;
for (i = 0; i < N_ELEMENTS(date_fields); i++) {
t = dates[i];
&t, sizeof(t));
}
}
(void)index_mail_cache_sent_date(mail);
}
enum index_cache_field field)
{
return -1;
}
/* if we're here because we aborted parsing, don't get any
further or we may crash while generating output from
incomplete data */
return 0;
}
(void)get_cached_msgpart_sizes(mail);
return 0;
}
{
return 0;
"read(%s) failed: %m (uid=%u)",
return -1;
}
enum index_cache_field field)
{
int ret;
if (data->save_bodystructure_body) {
/* bodystructure header is parsed, we want the body's mime
headers too */
} else {
*null_message_part_header_callback, (void *)NULL);
}
ret = -1;
return ret;
}
{
}
{
unsigned int block_size;
}
struct message_size *hdr_size,
struct message_size *body_size,
{
bool has_nuls;
int ret;
if (!data->initialized_wrapper_stream &&
}
if (!data->destroy_callback_set) {
/* do this only once in case a plugin changes the stream.
otherwise the check would break. */
}
(void)get_cached_msgpart_sizes(mail);
if (!data->hdr_size_set) {
(void)get_cached_parts(mail);
return -1;
} else {
&has_nuls) < 0) {
"read(%s) failed: %m",
return -1;
}
}
}
}
if (!data->body_size_set)
if (!data->body_size_set) {
if (index_mail_parse_body(mail, 0) < 0)
return -1;
} else {
&has_nuls) < 0) {
"read(%s) failed: %m",
return -1;
}
}
}
}
}
if (ret < 0)
return -1;
return 0;
}
enum index_cache_field field)
{
if (data->parsed_bodystructure) {
/* we have everything parsed already, but just not written to
a string */
} else {
if (data->save_bodystructure_header ||
/* we haven't parsed the header yet */
(void)get_cached_parts(mail);
return -1;
}
return -1;
}
/* if we didn't want to have the body(structure) cached,
it's still not written. */
switch (field) {
case MAIL_CACHE_IMAP_BODY:
}
break;
}
break;
default:
i_unreached();
}
return 0;
}
static void
bool extended)
{
if (extended)
}
{
const char *error;
switch (field) {
case MAIL_FETCH_IMAP_BODY: {
const unsigned int body_cache_field =
const unsigned int bodystructure_cache_field =
return 0;
}
/* 1) use plain-7bit-ascii flag if it exists
2) get BODY if it exists
3) get it using BODYSTRUCTURE if it exists
4) parse body structure, and save BODY/BODYSTRUCTURE
depending on what we want cached */
get_cached_parts(mail)) {
body_cache_field) > 0)
bodystructure_cache_field) > 0) {
str_truncate(str, 0);
/* broken, continue.. */
"Invalid BODYSTRUCTURE %s: %s",
} else {
}
}
MAIL_CACHE_IMAP_BODY) < 0)
return -1;
}
return 0;
}
case MAIL_FETCH_IMAP_BODYSTRUCTURE: {
const unsigned int bodystructure_cache_field =
return 0;
}
get_cached_parts(mail)) {
bodystructure_cache_field) > 0) {
} else {
return -1;
}
return 0;
}
case MAIL_FETCH_IMAP_ENVELOPE:
if (index_mail_headers_get_envelope(mail) < 0)
return -1;
}
return 0;
case MAIL_FETCH_FROM_ENVELOPE:
return 0;
case MAIL_FETCH_UIDL_BACKEND:
case MAIL_FETCH_GUID:
case MAIL_FETCH_HEADER_MD5:
case MAIL_FETCH_POP3_ORDER:
*value_r = "";
return 0;
case MAIL_FETCH_MAILBOX_NAME:
return 0;
default:
i_unreached();
return -1;
}
}
{
return mail;
}
struct mail *
index_mail_alloc(struct mailbox_transaction_context *t,
struct mailbox_header_lookup_ctx *wanted_headers)
{
struct index_mail *mail;
}
struct mailbox_transaction_context *t,
struct mailbox_header_lookup_ctx *wanted_headers)
{
sizeof(void *), 5);
t->mail_ref_count++;
if (wanted_headers != NULL) {
}
}
{
struct message_part *parts;
}
}
if (!closing) {
/* we're replacing the stream with a new one. it's
allowed to have references until the mail is closed
(but we can't really check that) */
}
if (closing) {
/* there must be no references to the mail when the
mail is being closed. */
} else {
}
}
}
{
}
{
/* If uid == 0 but seq != 0, we came here from saving a (non-mbox)
message. If that happens, don't bother checking if anything should
be cached since it was already checked. Also by now the transaction
may have already been rollbacked and seq point to a nonexistent
message. */
}
}
{
}
}
{
const unsigned int cache_field_envelope =
unsigned int cache_field_hdr;
return;
}
/* if "imap.envelope" is cached, that's all we need */
return;
/* don't waste time doing full checks for all required
headers. assume that if we have "hdr.message-id" cached,
we don't need to parse the header. */
"hdr.message-id");
if (cache_field_hdr == (unsigned int)-1 ||
}
{
const struct mail_index_header *hdr;
MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0 &&
&data->cache_flags,
sizeof(data->cache_flags));
_mail->has_no_nuls =
/* we currently don't forcibly set the nul state. if it's not
already cached, the caller can figure out itself what to
do when neither is set */
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
/* we need either imap.body or imap.bodystructure */
const unsigned int cache_field1 =
const unsigned int cache_field2 =
cache_field1) <= 0 &&
cache_field2) <= 0) {
}
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
const unsigned int cache_field =
cache_field) <= 0) {
}
}
MAIL_FETCH_STREAM_BODY)) != 0) {
/* open stream immediately to set expunged flag if
it's already lost */
/* open the stream only if we didn't get here from
mailbox_save_init() */
else
}
}
/* we want virtual size, and we'd prefer not to read the entire
message for it. see if it's possible. */
}
}
{
return;
return;
}
}
{
/* HAVE_POSIX_FADVISE alone isn't enough for CentOS 4.9 */
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
int fd;
/* we're handling only file-per-msg storages for now. */
return TRUE;
}
/* everything we need is cached */
return TRUE;
}
return TRUE;
}
/* tell OS to start reading the file into memory */
if (fd != -1) {
len = 0;
else
i_error("posix_fadvise(%s) failed: %m",
}
}
#endif
}
{
return TRUE;
} else {
return FALSE;
}
}
enum mail_fetch_field fields,
struct mailbox_header_lookup_ctx *headers)
{
unsigned int i;
/* keep old ones */
} else {
/* merge headers */
}
}
{
}
{
struct mailbox_header_lookup_ctx *headers_ctx =
/* make sure mailbox_search_*() users don't try to free the mail
directly */
if (headers_ctx != NULL)
}
{
struct message_block block;
&block) > 0) {
continue;
} else {
}
}
}
bool success)
{
if (!success) {
/* we're going to delete this mail anyway,
don't bother trying to update cache file */
/* we didn't even start cache parsing */
return;
}
}
/* This is needed with 0 byte mails to get hdr=NULL call done. */
/* this save_date may not be exactly the same as what we get
in future, but then again neither mbox nor maildir
guarantees it anyway. */
}
(void)index_mail_parse_body_finish(mail, 0);
}
{
const struct mail_index_header *hdr;
}
}
enum mail_flags flags)
{
bool update_modseq = FALSE;
if ((flags & MAIL_RECENT) == 0 &&
/* mailbox has private flags */
flags &= ~pvt_flags_mask;
if (index_mail_get_pvt(_mail) &&
}
}
if (!update_modseq) {
/* no forced modseq update */
} else if (modify_type == MODIFY_REMOVE) {
/* add the modseq update separately */
} else {
/* add as part of the flag updates */
}
modify_type, flags);
}
struct mail_keywords *keywords)
{
/* clear the keywords array so the next mail_get_keywords()
returns the updated keywords. don't free the array, because
then any existing mail_get_keywords() return values would
point to broken data. this won't leak memory because the
array is allocated from mail's memory pool. */
}
}
{
}
{
const char *value;
else {
}
}
{
if (parse_body) {
(void)index_mail_parse_body(imail, 0);
}
}
}
{
enum mail_fetch_field cache;
const char *str;
/* already cached this mail (we should get here only if FTS
plugin decreased the first precached seq) */
return;
}
if ((cache & MAIL_FETCH_RECEIVED_DATE) != 0)
if ((cache & MAIL_FETCH_SAVE_DATE) != 0)
if ((cache & MAIL_FETCH_VIRTUAL_SIZE) != 0)
if ((cache & MAIL_FETCH_PHYSICAL_SIZE) != 0)
if ((cache & MAIL_FETCH_UIDL_BACKEND) != 0)
if ((cache & MAIL_FETCH_GUID) != 0)
}
enum mail_fetch_field field)
{
const char *field_name;
switch ((int)field) {
case 0:
field_name = "fields";
break;
case MAIL_FETCH_PHYSICAL_SIZE:
field_name = "physical size";
break;
case MAIL_FETCH_VIRTUAL_SIZE:
field_name = "virtual size";
break;
case MAIL_FETCH_MESSAGE_PARTS:
field_name = "MIME parts";
break;
case MAIL_FETCH_IMAP_BODY:
field_name = "IMAP BODY";
break;
field_name = "IMAP BODYSTRUCTURE";
break;
default:
}
/* make sure we don't cache invalid values */
"Broken %s for mail UID %u",
}
{
return 0;
}
{
return;
}
}