mbox-append.c revision 1969b0cbe66efb9a81ec2404f4903d7844e63729
2e37d45867d081db150ab78dad303b9077aea24fTimo Sirainen/* Copyright (C) 2002 Timo Sirainen */
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "lib.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "ioloop.h"
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen#include "iobuffer.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "hex-binary.h"
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen#include "md5.h"
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen#include "mbox-index.h"
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen#include "mail-index-util.h"
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic MailIndexRecord *
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainenmail_index_record_append(MailIndex *index, time_t internal_date,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen size_t full_virtual_size)
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen{
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen MailIndexRecord trec, *rec;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen memset(&trec, 0, sizeof(MailIndexRecord));
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen trec.internal_date = internal_date;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen trec.full_virtual_size = full_virtual_size;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen rec = &trec;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (!index->append(index, &rec))
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen return NULL;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return rec;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainenstatic void mbox_read_message(IOBuffer *inbuf, unsigned int *virtual_size)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen{
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen unsigned char *msg;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen unsigned int i, size, startpos, vsize;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen int lastmsg;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen /* read until "[\r]\nFrom " is found */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen startpos = i = vsize = 0; lastmsg = TRUE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen while (io_buffer_read_data(inbuf, &msg, &size, startpos) >= 0) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen for (i = startpos; i < size; i++) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (msg[i] == '\n') {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen if (i == 0 || msg[i-1] != '\r') {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* missing CR */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen vsize++;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen } else if (msg[i] == ' ' && i >= 5) {
85a4ae7e8df7ea45a7665828e5edf48a5fc85730Timo Sirainen /* See if it's space after "From" */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (msg[i-5] == '\n' && msg[i-4] == 'F' &&
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen msg[i-3] == 'r' && msg[i-2] == 'o' &&
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen msg[i-1] == 'm') {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* yes, see if we had \r too */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i -= 5;
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen if (i > 0 && msg[i-1] == '\r')
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i--;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen else
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen vsize--;
0fd246126fece57712566c725d6353f255f5fcfaTimo Sirainen break;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
d0ef8bc2b961a68dd0f75662c2160bd296b9476bTimo Sirainen if (i < size) {
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen startpos = i;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen lastmsg = FALSE;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen break;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen }
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (i > 0) {
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen startpos = i < 7 ? i : 7;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen i -= startpos;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen io_buffer_skip(inbuf, i);
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen vsize += i;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (lastmsg && startpos > 0) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* end of file, remove the last [\r]\n */
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen msg = io_buffer_get_data(inbuf, &size);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (size == startpos) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (msg[startpos-1] == '\n')
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen startpos--;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (startpos > 0 && msg[startpos-1] == '\r')
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen startpos--;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen else
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen vsize--;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen }
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen }
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen io_buffer_skip(inbuf, startpos);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen vsize += startpos;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen *virtual_size = vsize;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic int mbox_index_append_next(MailIndex *index, IOBuffer *inbuf)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen{
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen MailIndexRecord *rec;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen MailIndexUpdate *update;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen MboxHeaderContext ctx;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen time_t internal_date;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen uoff_t abs_start_offset, stop_offset, old_size;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen unsigned char *data, md5_digest[16];
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen unsigned int size, pos, virtual_size;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* get the From-line */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen pos = 0;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen while (io_buffer_read_data(inbuf, &data, &size, pos) >= 0) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen for (; pos < size; pos++) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (data[pos] == '\n')
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen break;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen if (pos < size)
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen break;
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen }
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen if (pos == size || size <= 5 || strncmp(data, "From ", 5) != 0) {
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen /* a) no \n found, or line too long
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen b) not a From-line */
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen index_set_error(index, "Error indexing mbox file %s: "
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen "From-line not found where expected",
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen index->mbox_path);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen index->set_flags |= MAIL_INDEX_FLAG_FSCK;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen return FALSE;
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen }
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen /* parse the From-line */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen internal_date = mbox_from_parse_date(data, size);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (internal_date <= 0)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen internal_date = ioloop_time;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen io_buffer_skip(inbuf, pos+1);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen abs_start_offset = inbuf->start_offset + inbuf->offset;
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen /* now, find the ending "[\r]\nFrom " */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mbox_read_message(inbuf, &virtual_size);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen stop_offset = inbuf->offset;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* add message to index */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen rec = mail_index_record_append(index, internal_date, virtual_size);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (rec == NULL)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return FALSE;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen update = index->update_begin(index, rec);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen /* location = offset to beginning of headers in message */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen index->update_field_raw(update, FIELD_TYPE_LOCATION,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen &abs_start_offset, sizeof(uoff_t));
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* parse the header and cache wanted fields. get the message flags
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen from Status and X-Status fields. temporarily limit the buffer size
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen so the message body is parsed properly (FIXME: does this have
a0c453a8edaec90fb0d945c874de0b1845bc7d7eTimo Sirainen side effects?) */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mbox_header_init_context(&ctx);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen old_size = inbuf->size;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen inbuf->size = stop_offset;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen io_buffer_seek(inbuf, abs_start_offset - inbuf->start_offset);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen mail_index_update_headers(update, inbuf, 0, mbox_header_func, &ctx);
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen
2793e3bd31d212d6506686aa70773e13d9d98195Timo Sirainen inbuf->size = old_size;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen io_buffer_seek(inbuf, stop_offset);
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen /* save MD5 */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen md5_final(&ctx.md5, md5_digest);
bc93929cdd9000ca560a5f42a27f50ab307f1efbTimo Sirainen index->update_field(update, FIELD_TYPE_MD5,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen binary_to_hex(md5_digest, sizeof(md5_digest)), 0);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (!index->update_end(update)) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* failed - delete the record */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen (void)index->expunge(index, rec, 0, FALSE);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return FALSE;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* save message flags, after location field is saved */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (!index->update_flags(index, rec, 0, ctx.flags, FALSE))
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return FALSE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen return TRUE;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen}
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainenint mbox_index_append(MailIndex *index, IOBuffer *inbuf)
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen{
f210ec6b25f80d06419921e9231465bb114ee971Timo Sirainen if (inbuf->offset == inbuf->size) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* no new data */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return TRUE;
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen return FALSE;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen for (;;) {
437a8b0fe254057b0c1f1723d689bafa91cae2abTimo Sirainen if (inbuf->start_offset + inbuf->offset != 0) {
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen /* we're at the [\r]\n before the From-line,
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen skip it */
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen if (!mbox_skip_crlf(inbuf)) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen index_set_error(index,
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen "Error indexing mbox file %s: "
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen "LF not found where expected",
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen index->mbox_path);
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->set_flags |= MAIL_INDEX_FLAG_FSCK;
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen return FALSE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
88dc563319efecd6e68bad16b0d92672da05584aTimo Sirainen }
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (inbuf->offset == inbuf->size)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen break;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (!mbox_index_append_next(index, inbuf))
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return FALSE;
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
1b0cfbf3cc77a670b92fff5c30f7b1eb17a63ab1Timo Sirainen return TRUE;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen