mbox-rewrite.c revision 62505210a7e6d1b2e35fac335a6c875a7c98ccfb
5a580c3a38ced62d4bcc95b8ac7c4f2935b5d294Timo Sirainen/* Copyright (C) 2002 Timo Sirainen */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainentypedef struct {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen/* Remove dirty flag from all messages */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void reset_dirty_flags(MailIndex *index)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index->header->flags &= ~(MAIL_INDEX_FLAG_DIRTY_MESSAGES |
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mbox_write(MailIndex *index, IOBuffer *inbuf, IOBuffer *outbuf,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* fsck should have noticed it.. */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index_set_error(index, "Error rewriting mbox file %s: "
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mbox_write_ximapbase(MboxRewriteContext *ctx)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *str;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (io_buffer_send(ctx->outbuf, ctx->custom_flags[i],
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mbox_write_xkeywords(MboxRewriteContext *ctx, const char *x_keywords)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int field;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if ((ctx->msg_flags & MAIL_CUSTOM_FLAGS_MASK) == 0 &&
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (io_buffer_send(ctx->outbuf, "X-Keywords:", 11) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++, field <<= 1) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if ((ctx->msg_flags & field) && ctx->custom_flags[i] != NULL) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (io_buffer_send(ctx->outbuf, ctx->custom_flags[i],
049da065aa64c1a5ed46eed6cde7382b011612a9Timo Sirainen /* X-Keywords that aren't custom flags */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mbox_write_status(MboxRewriteContext *ctx, const char *status)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *str;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen str = (ctx->msg_flags & MAIL_SEEN) ? "Status: RO" : "Status: O";
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mbox_write_xstatus(MboxRewriteContext *ctx, const char *x_status)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *str;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* X-Status field */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if ((ctx->msg_flags & (MAIL_SYSTEM_FLAGS_MASK^MAIL_SEEN)) == 0 &&
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int mbox_write_content_length(MboxRewriteContext *ctx)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_snprintf(str, sizeof(str), "Content-Length: %"PRIuUOFF_T"\n",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic const char *strip_chars(const char *value, size_t value_len,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* leave only unknown flags, very likely none */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int i;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen for (i = 0; i < value_len; i++) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void update_stripped_custom_flags(const char *value, size_t len,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* not found, keep it */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic const char *strip_custom_flags(const char *value, size_t len,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mbox_keywords_parse(value, len, ctx->custom_flags,
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic void header_func(MessagePart *part __attr_unused__,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *str;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (name_len == 6 && strncasecmp(name, "Status", 6) == 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen } else if (name_len == 8 && strncasecmp(name, "X-Status", 8) == 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen } else if (name_len == 10 && strncasecmp(name, "X-Keywords", 10) == 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen str = strip_custom_flags(value, value_len, ctx);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen } else if (name_len == 10 && strncasecmp(name, "X-IMAPbase", 10) == 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* temporarily copy the value to make sure we
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen don't overflow it */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen strncasecmp(name, "Content-Length", 14) == 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen } else if (name_len > 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* save this header */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (void)io_buffer_send(ctx->outbuf, name, name_len);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (void)io_buffer_send(ctx->outbuf, value, value_len);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* We need to update fields that define message flags. Standard fields
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen are stored in Status and X-Status. For custom flags we use
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uw-imapd compatible format, by first listing them in first message's
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen X-IMAPbase field and actually defining them in X-Keywords field.
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen Format of X-IMAPbase is: <UID validity> <last used UID> <flag names>
c8c4bbf6b1415e9d0845bc8f1cd6d19b76ab0392Timo Sirainen We don't want to sync our UIDs with the mbox file, so the UID
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen validity is always kept different from our internal UID validity.
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen Last used UID is also not updated, and set to 0 initially.
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* fsck should have noticed it.. */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index_set_error(index, "Error rewriting mbox file %s: "
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* parse the header, write the fields we don't want to change */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ctx.uid_validity = index->header->uid_validity-1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ctx.custom_flags = mail_custom_flags_list_get(index->custom_flags);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen io_buffer_set_read_limit(inbuf, inbuf->offset + rec->header_size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen message_parse_header(NULL, inbuf, &hdr_size, header_func, &ctx);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_assert(hdr_size.physical_size == rec->header_size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* append the flag fields */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* write X-IMAPbase header to first message */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_custom_flags_list_unref(index->custom_flags);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* empty line ends headers */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int fd_copy(int in_fd, int out_fd, uoff_t out_offset)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (lseek(out_fd, (off_t)out_offset, SEEK_SET) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen inbuf = io_buffer_create_mmap(in_fd, data_stack_pool,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen outbuf = io_buffer_create_file(out_fd, data_stack_pool, 1024, FALSE);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = io_buffer_send_iobuffer(outbuf, inbuf, inbuf->size);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* we may have shrinked the file */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_assert(out_offset + inbuf->size <= OFF_T_MAX);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen ret = ftruncate(out_fd, (off_t) (out_offset + inbuf->size));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* Write messages beginning from the first dirty one to temp file,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen then copy it over the mbox file. This may create data loss if
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen interrupted (see below). This rewriting relies quite a lot on
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen valid header/body sizes which fsck() should have ensured. */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int seq;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen int tmp_fd, failed, dirty_found, locked, rewrite;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if ((index->header->flags & MAIL_INDEX_FLAG_DIRTY_MESSAGES) == 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* no need to rewrite */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* lock before fscking to prevent race conditions between
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen fsck's unlock and our lock. */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* fsck() figured out there's no dirty messages
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen tmp_fd = mail_index_create_temp_file(index, &path);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (index->header->flags & MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* need to update X-IMAPbase in first message */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen outbuf = io_buffer_create_file(tmp_fd, data_stack_pool, 8192, FALSE);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (dirty_found || (rec->index_flags & INDEX_MAIL_FLAG_DIRTY)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* get offset to beginning of mail headers */
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen if (!mbox_mail_get_start_offset(index, rec, &offset)) {
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen /* fsck should have fixed it */
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen if (offset + rec->header_size + rec->body_size > inbuf->size) {
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen "Invalid message size");
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* first dirty message */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* write the From-line */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (!mbox_write(index, inbuf, outbuf, offset)) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* write header, updating flag fields */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!mbox_write_header(index, rec, seq, inbuf, outbuf,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* write body */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!mbox_write(index, inbuf, outbuf, offset)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen index_set_error(index, "Expected dirty messages not found "
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* always end with a \n */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* POSSIBLE DATA LOSS HERE. We're writing to the mbox file,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen so if we get killed here before finished, we'll lose some
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen bytes. I can't really think of any way to fix this,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen rename() is problematic too especially because of file
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen locking issues (new mail could be lost).
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen Usually we're moving the data by just a few bytes, so
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen the data loss should never be more than those few bytes..
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen If we moved more, we could have written the file from end
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen to beginning in blocks (it'd be a bit slow to do it in
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen blocks of ~1-10 bytes which is the usual case, so we don't
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen Also, we might as well be shrinking the file, in which
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen case we can't lose data. */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (fd_copy(tmp_fd, index->mbox_fd, dirty_offset) == 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* all ok, we need to fsck the index next time.
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen use set_flags because set_lock() would remove it
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if we modified it directly */