mbox-sync.c revision b8765f6093ab35fc2345293d78132d35794cbff5
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (C) 2004 Timo Sirainen */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen/*
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen required disk I/O. We may need to:
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen - Write missing X-UID and X-IMAPbase headers
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen - Write missing or broken Content-Length header if there's space
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen - Expunge specified messages
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen Here's how we do it:
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen - Start reading the mails from the beginning
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen of them, remember how much each message has and offset to beginning of the
8b2cf1c1bd8ddcea0525b62fd35ba76e136828a1Timo Sirainen padding
8b2cf1c1bd8ddcea0525b62fd35ba76e136828a1Timo Sirainen - If header needs to be rewritten and there's enough space, do it
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen - If we didn't have enough space, remember how much was missing
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen - Continue reading and counting the padding in each message. If available
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen padding is enough to rewrite all the previous messages needing it, do it
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen - When we encounter expunged message, treat all of it as padding and
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen rewrite previous messages if needed (and there's enough space).
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen Afterwards keep moving messages backwards to fill the expunged space.
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen Moving is done by rewriting each message's headers, with possibly adding
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen missing Content-Length header and padding. Message bodies are moved
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen without modifications.
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen - If we encounter end of file, grow the file and rewrite needed messages
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen - Rewriting is done by moving message body forward, rewriting message's
9b7eeffb5752b500ac62ba1fd01c4a8c4ada14e9Timo Sirainen header and doing the same for previous message, until all of them are
9b7eeffb5752b500ac62ba1fd01c4a8c4ada14e9Timo Sirainen rewritten.
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen*/
93fa87cf1a96c4f279ec4f5c311820313ba12c34Timo Sirainen
43834f87bf431198f986e86052a4f6e558fdb07dTimo Sirainen#include "lib.h"
43834f87bf431198f986e86052a4f6e558fdb07dTimo Sirainen#include "ioloop.h"
93fa87cf1a96c4f279ec4f5c311820313ba12c34Timo Sirainen#include "buffer.h"
93fa87cf1a96c4f279ec4f5c311820313ba12c34Timo Sirainen#include "hostpid.h"
93fa87cf1a96c4f279ec4f5c311820313ba12c34Timo Sirainen#include "istream.h"
93fa87cf1a96c4f279ec4f5c311820313ba12c34Timo Sirainen#include "file-set-size.h"
43834f87bf431198f986e86052a4f6e558fdb07dTimo Sirainen#include "str.h"
b565a6a7a66fb9f224d00c06a950e3c1c585c18eTimo Sirainen#include "read-full.h"
b565a6a7a66fb9f224d00c06a950e3c1c585c18eTimo Sirainen#include "write-full.h"
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen#include "message-date.h"
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen#include "istream-raw-mbox.h"
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen#include "mbox-storage.h"
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen#include "mbox-from.h"
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen#include "mbox-file.h"
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen#include "mbox-lock.h"
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen#include "mbox-sync-private.h"
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include <stddef.h>
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include <stdlib.h>
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include <sys/stat.h>
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#define MBOX_SYNC_SECS 1
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen/* The text below was taken exactly as c-client wrote it to my mailbox,
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen so it's probably copyrighted by University of Washington. */
f988b93c2ef773987bcdcbfb4cca39b955e3a392Timo Sirainen#define PSEUDO_MESSAGE_BODY \
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen"This text is part of the internal format of your mail folder, and is not\n" \
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen"a real message. It is created automatically by the mail system software.\n" \
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen"If deleted, important folder data will be lost, and it will be re-created\n" \
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen"with the data reset to initial values.\n"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen{
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen " from mbox file %s", from_offset,
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen sync_ctx->mbox->path);
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen return -1;
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen }
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen return 0;
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen}
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainenstatic void mbox_sync_array_delete_to(array_t *syncs_arr, uint32_t last_uid)
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen{
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen ARRAY_SET_TYPE(syncs_arr, struct mail_index_sync_rec);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen struct mail_index_sync_rec *syncs;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen unsigned int src, dest, count;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen syncs = array_get_modifyable(syncs_arr, &count);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen for (src = dest = 0; src < count; src++) {
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen i_assert(last_uid >= syncs[src].uid1);
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen if (last_uid <= syncs[src].uid2) {
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen /* keep it */
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen if (src != dest)
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen syncs[dest] = syncs[src];
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen dest++;
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen }
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen }
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen array_delete(syncs_arr, dest, count - dest);
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen}
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainenstatic int
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen struct mbox_sync_mail_context *mail_ctx)
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen{
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen /* get EOF */
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
10ff47d5d6146995e16da00d36eca7d162064a7bTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input))
683eebe490bbe5caec246c535a10ea9f93f5c330Timo Sirainen return 0;
683eebe490bbe5caec246c535a10ea9f93f5c330Timo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainen mail_ctx->sync_ctx = sync_ctx;
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainen mail_ctx->seq = ++sync_ctx->seq;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen mail_ctx->header = sync_ctx->header;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen mail_ctx->mail.from_offset =
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen mail_ctx->mail.offset =
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen sync_ctx->input->eof);
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen mail_ctx->mail.body_size =
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen mail_ctx->content_length);
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
ceb43cc04edb94445fab8f914bc4da6d740403d1Timo Sirainen
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 && !mail_ctx->pseudo) {
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen if (!sync_ctx->mbox->ibox.keep_recent) {
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen /* need to add 'O' flag to Status-header */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_ctx->need_rewrite = TRUE;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_ctx->recent = TRUE;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen return 1;
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen}
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainenstatic int mbox_sync_buf_have_expunges(array_t *syncs_arr)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen{
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen ARRAY_SET_TYPE(syncs_arr, struct mail_index_sync_rec);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen const struct mail_index_sync_rec *syncs;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen unsigned int i, count;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen syncs = array_get(syncs_arr, &count);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen for (i = 0; i < count; i++) {
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (syncs[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen return TRUE;
d77fb3be552eefbfb9cdd43ff23d794796d7a36cTimo Sirainen }
d77fb3be552eefbfb9cdd43ff23d794796d7a36cTimo Sirainen return FALSE;
d77fb3be552eefbfb9cdd43ff23d794796d7a36cTimo Sirainen}
03f5c621d06d6b6d77a145196c9633a7aa64dc78Timo Sirainen
d7e72877b7a5085c3addf9729d0bfbe1b5357853Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
d7e72877b7a5085c3addf9729d0bfbe1b5357853Timo Sirainen uint32_t uid, int *sync_expunge_r)
d7e72877b7a5085c3addf9729d0bfbe1b5357853Timo Sirainen{
d7e72877b7a5085c3addf9729d0bfbe1b5357853Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
03f5c621d06d6b6d77a145196c9633a7aa64dc78Timo Sirainen int ret;
03f5c621d06d6b6d77a145196c9633a7aa64dc78Timo Sirainen
03f5c621d06d6b6d77a145196c9633a7aa64dc78Timo Sirainen *sync_expunge_r = FALSE;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen if (sync_ctx->index_sync_ctx == NULL)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen return 0;
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen if (uid == 0) {
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen /* nothing for this or the future ones */
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen uid = (uint32_t)-1;
71da447014454c84828d9dface77219875554d7dTimo Sirainen }
71da447014454c84828d9dface77219875554d7dTimo Sirainen
71da447014454c84828d9dface77219875554d7dTimo Sirainen mbox_sync_array_delete_to(&sync_ctx->syncs, uid);
71da447014454c84828d9dface77219875554d7dTimo Sirainen while (uid >= sync_rec->uid1) {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen if (uid <= sync_rec->uid2 &&
sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND &&
(sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE ||
!sync_ctx->mbox->mbox_readonly)) {
array_append(&sync_ctx->syncs, sync_rec, 1);
if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
*sync_expunge_r = TRUE;
}
ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
if (ret < 0) {
mail_storage_set_index_error(&sync_ctx->mbox->ibox);
return -1;
}
if (ret == 0) {
memset(sync_rec, 0, sizeof(*sync_rec));
break;
}
if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) {
if (sync_rec->uid2 >= sync_ctx->next_uid)
sync_ctx->next_uid = sync_rec->uid2 + 1;
memset(sync_rec, 0, sizeof(*sync_rec));
}
}
if (!*sync_expunge_r)
*sync_expunge_r = mbox_sync_buf_have_expunges(&sync_ctx->syncs);
return 0;
}
void mbox_sync_apply_index_syncs(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mail,
int *keywords_changed_r)
{
const struct mail_index_sync_rec *syncs;
unsigned int i, count;
*keywords_changed_r = FALSE;
syncs = array_get(&sync_ctx->syncs, &count);
for (i = 0; i < count; i++) {
switch (syncs[i].type) {
case MAIL_INDEX_SYNC_TYPE_FLAGS:
mail_index_sync_flags_apply(&syncs[i], &mail->flags);
break;
case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:
if (!array_is_created(&mail->keywords)) {
/* no existing keywords */
if (syncs[i].type !=
MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD)
break;
/* adding, create the array */
ARRAY_CREATE(&mail->keywords,
sync_ctx->mail_keyword_pool,
unsigned int,
I_MIN(10, count - i));
}
if (mail_index_sync_keywords_apply(&syncs[i],
&mail->keywords))
*keywords_changed_r = TRUE;
break;
default:
break;
}
}
}
static int
mbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
uint32_t uid, const struct mail_index_record **rec_r)
{
const struct mail_index_record *rec = NULL;
uint32_t messages_count;
int ret = 0;
messages_count =
mail_index_view_get_messages_count(sync_ctx->sync_view);
while (sync_ctx->idx_seq <= messages_count) {
ret = mail_index_lookup(sync_ctx->sync_view,
sync_ctx->idx_seq, &rec);
if (ret < 0) {
mail_storage_set_index_error(&sync_ctx->mbox->ibox);
return -1;
}
if (uid <= rec->uid)
break;
/* externally expunged message, remove from index */
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
sync_ctx->idx_seq++;
rec = NULL;
}
if (ret == 0 && uid < sync_ctx->idx_next_uid) {
/* this UID was already in index and it was expunged */
mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
"mbox sync: Expunged message reappeared in mailbox %s "
"(UID %u < %u)", sync_ctx->mbox->path, uid,
sync_ctx->idx_next_uid);
ret = 0; rec = NULL;
} else if (rec != NULL && rec->uid != uid) {
/* new UID in the middle of the mailbox - shouldn't happen */
mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
"mbox sync: UID inserted in the middle of mailbox %s "
"(%u > %u)", sync_ctx->mbox->path, rec->uid, uid);
ret = 0; rec = NULL;
} else {
ret = 1;
}
*rec_r = rec;
return ret;
}
static int mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
unsigned char hdr_md5_sum[],
const struct mail_index_record **rec_r)
{
const struct mail_index_record *rec = NULL;
uint32_t messages_count;
const void *data;
int ret;
messages_count =
mail_index_view_get_messages_count(sync_ctx->sync_view);
while (sync_ctx->idx_seq <= messages_count) {
ret = mail_index_lookup(sync_ctx->sync_view,
sync_ctx->idx_seq, &rec);
if (ret < 0) {
mail_storage_set_index_error(&sync_ctx->mbox->ibox);
return -1;
}
if (mail_index_lookup_ext(sync_ctx->sync_view,
sync_ctx->idx_seq,
sync_ctx->mbox->ibox.md5hdr_ext_idx,
&data) < 0) {
mail_storage_set_index_error(&sync_ctx->mbox->ibox);
return -1;
}
if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0)
break;
/* externally expunged message, remove from index */
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
sync_ctx->idx_seq++;
rec = NULL;
}
*rec_r = rec;
return 0;
}
static int
mbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mail,
int nocheck)
{
const void *data;
uint64_t offset;
if (!nocheck) {
/* see if from_offset needs updating */
if (mail_index_lookup_ext(sync_ctx->sync_view,
sync_ctx->idx_seq,
sync_ctx->mbox->mbox_ext_idx,
&data) < 0) {
mail_storage_set_index_error(&sync_ctx->mbox->ibox);
return -1;
}
if (data != NULL &&
*((const uint64_t *)data) == mail->from_offset)
return 0;
}
offset = mail->from_offset;
mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->mbox->mbox_ext_idx, &offset, NULL);
return 0;
}
static void
mbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
struct mail_keywords *keywords;
keywords = !array_is_created(&mail_ctx->mail.keywords) ?
mail_index_keywords_create(sync_ctx->t, NULL) :
mail_index_keywords_create_from_indexes(sync_ctx->t,
&mail_ctx->mail.keywords);
mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, keywords);
mail_index_keywords_free(keywords);
}
static int mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
const struct mail_index_record *rec)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
struct mbox_sync_mail *mail = &mail_ctx->mail;
uint8_t mbox_flags;
mbox_flags = mail->flags & MAIL_FLAGS_MASK;
if (mail_ctx->dirty)
mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
else if (!sync_ctx->delay_writes)
mbox_flags &= ~MAIL_INDEX_MAIL_FLAG_DIRTY;
if (rec == NULL) {
/* new message */
mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags);
mbox_sync_update_index_keywords(mail_ctx);
if (sync_ctx->mbox->mbox_save_md5 != 0) {
mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->mbox->ibox.md5hdr_ext_idx,
mail_ctx->hdr_md5_sum, NULL);
}
} else {
/* see if we need to update flags in index file. the flags in
sync records are automatically applied to rec->flags at the
end of index syncing, so calculate those new flags first */
struct mbox_sync_mail idx_mail;
int keywords_changed;
memset(&idx_mail, 0, sizeof(idx_mail));
idx_mail.flags = rec->flags;
/* get old keywords */
t_push();
ARRAY_CREATE(&idx_mail.keywords, pool_datastack_create(),
unsigned int, 32);
if (mail_index_lookup_keywords(sync_ctx->sync_view,
sync_ctx->idx_seq,
&idx_mail.keywords) < 0) {
mail_storage_set_index_error(&sync_ctx->mbox->ibox);
t_pop();
return -1;
}
mbox_sync_apply_index_syncs(sync_ctx, &idx_mail,
&keywords_changed);
#define SYNC_FLAGS (MAIL_RECENT | MAIL_INDEX_MAIL_FLAG_DIRTY)
if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* flags are dirty. ignore whatever was in the mbox,
but update recent/dirty flag states if needed. */
mbox_flags &= SYNC_FLAGS;
mbox_flags |= idx_mail.flags & ~SYNC_FLAGS;
} else {
/* keep index's internal flags */
mbox_flags &= MAIL_FLAGS_MASK | SYNC_FLAGS;
mbox_flags |= idx_mail.flags &
~(MAIL_FLAGS_MASK | SYNC_FLAGS);
}
if ((idx_mail.flags & ~SYNC_FLAGS) !=
(mbox_flags & ~SYNC_FLAGS)) {
/* flags other than recent/dirty have changed */
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags);
} else {
if (((idx_mail.flags ^ mbox_flags) &
MAIL_RECENT) != 0) {
/* drop recent flag (it can only be dropped) */
mail_index_update_flags(sync_ctx->t,
sync_ctx->idx_seq,
MODIFY_REMOVE, MAIL_RECENT);
}
if (((idx_mail.flags ^ mbox_flags) &
MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* dirty flag state changed */
int dirty = (mbox_flags &
MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
mail_index_update_flags(sync_ctx->t,
sync_ctx->idx_seq,
dirty ? MODIFY_ADD : MODIFY_REMOVE,
MAIL_INDEX_MAIL_FLAG_DIRTY);
}
}
if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0 &&
!array_cmp(&idx_mail.keywords, &mail_ctx->mail.keywords))
mbox_sync_update_index_keywords(mail_ctx);
t_pop();
}
if (mail_ctx->recent &&
(rec == NULL || (rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0 ||
(rec->flags & MAIL_RECENT) != 0)) {
index_mailbox_set_recent(&sync_ctx->mbox->ibox,
sync_ctx->idx_seq);
}
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
return -1;
}
return 0;
}
static int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
{
struct istream *input = ctx->sync_ctx->file_input;
const unsigned char *data;
size_t size, from_line_size;
buffer_set_used_size(ctx->sync_ctx->from_line, 0);
from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
i_stream_seek(input, ctx->mail.from_offset);
for (;;) {
data = i_stream_get_data(input, &size);
if (size >= from_line_size)
size = from_line_size;
buffer_append(ctx->sync_ctx->from_line, data, size);
i_stream_skip(input, size);
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx)
{
unsigned char buf[10];
const char *str;
uint32_t uid_last;
unsigned int i;
int ret;
i_assert(sync_ctx->base_uid_last_offset != 0);
/* first check that the 10 bytes are there and they're exactly as
expected. just an extra safety check to make sure we never write
to wrong location in the mbox file. */
ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf),
sync_ctx->base_uid_last_offset);
if (ret < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "pread_full()");
return -1;
}
if (ret == 0) {
mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
"X-IMAPbase uid-last unexpectedly points outside "
"mbox file %s", sync_ctx->mbox->path);
return -1;
}
for (i = 0, uid_last = 0; i < sizeof(buf); i++) {
if (buf[i] < '0' || buf[i] > '9') {
uid_last = (uint32_t)-1;
break;
}
uid_last = uid_last * 10 + (buf[i] - '0');
}
if (uid_last != sync_ctx->base_uid_last) {
mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
"X-IMAPbase uid-last unexpectedly lost in mbox file %s",
sync_ctx->mbox->path);
return -1;
}
/* and write it */
str = t_strdup_printf("%010u", sync_ctx->next_uid - 1);
if (pwrite_full(sync_ctx->write_fd, str, 10,
sync_ctx->base_uid_last_offset) < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
return -1;
}
sync_ctx->base_uid_last = sync_ctx->next_uid - 1;
return 0;
}
static int
mbox_write_from_line(struct mbox_sync_mail_context *ctx)
{
string_t *str = ctx->sync_ctx->from_line;
if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str),
ctx->mail.from_offset) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()");
return -1;
}
i_stream_sync(ctx->sync_ctx->input);
return 0;
}
static void update_from_offsets(struct mbox_sync_context *sync_ctx)
{
const struct mbox_sync_mail *mails;
unsigned int i, count;
uint32_t ext_idx;
uint64_t offset;
ext_idx = sync_ctx->mbox->mbox_ext_idx;
mails = array_get(&sync_ctx->mails, &count);
for (i = 0; i < count; i++) {
if (mails[i].idx_seq == 0 ||
(mails[i].flags & MBOX_EXPUNGED) != 0)
continue;
offset = mails[i].from_offset;
mail_index_update_ext(sync_ctx->t, mails[i].idx_seq,
ext_idx, &offset, NULL);
}
}
static void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
{
mail_ctx->mail.flags = MBOX_EXPUNGED;
mail_ctx->mail.offset = mail_ctx->mail.from_offset;
mail_ctx->mail.space =
mail_ctx->body_offset - mail_ctx->mail.from_offset +
mail_ctx->mail.body_size;
mail_ctx->mail.body_size = 0;
if (mail_ctx->sync_ctx->seq == 1) {
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
mail_ctx->mail.space++;
/* uid-last offset is invalid now */
mail_ctx->sync_ctx->base_uid_last_offset = 0;
}
mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
}
static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
off_t move_diff;
int ret;
if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
/* move the header backwards to fill expunged space */
move_diff = -sync_ctx->expunged_space;
if (sync_ctx->dest_first_mail) {
/* we're moving this mail to beginning of file.
skip the initial \n (it's already counted in
expunged_space) */
mail_ctx->mail.from_offset++;
}
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
mbox_sync_update_header(mail_ctx);
ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
if (ret < 0)
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
mail_ctx->mail.from_offset += move_diff;
mail_ctx->mail.offset += move_diff;
if (mbox_write_from_line(mail_ctx) < 0)
return -1;
} else {
if (sync_ctx->dest_first_mail) {
/* didn't have enough space, move the offset
back so seeking into it doesn't fail */
mail_ctx->mail.from_offset--;
}
}
} else if (mail_ctx->need_rewrite ||
array_count(&sync_ctx->syncs) != 0) {
mbox_sync_update_header(mail_ctx);
if (sync_ctx->delay_writes) {
/* mark it dirty and do it later */
mail_ctx->dirty = TRUE;
return 0;
}
if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
return -1;
} else {
/* nothing to do */
return 0;
}
if (ret == 0 && sync_ctx->need_space_seq == 0) {
/* first mail with no space to write it */
sync_ctx->need_space_seq = sync_ctx->seq;
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
memset(&mail, 0, sizeof(mail));
mail.flags = MBOX_EXPUNGED;
mail.offset = mail.from_offset =
(sync_ctx->dest_first_mail ? 1 : 0) +
mail_ctx->mail.from_offset -
sync_ctx->expunged_space;
mail.space = sync_ctx->expunged_space;
sync_ctx->space_diff = sync_ctx->expunged_space;
sync_ctx->expunged_space = 0;
i_assert(sync_ctx->space_diff < -mail_ctx->mail.space);
sync_ctx->need_space_seq--;
array_append(&sync_ctx->mails, &mail, 1);
}
}
return 0;
}
static int
mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t end_offset, move_diff, extra_space, needed_space;
uint32_t last_seq;
i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 ||
mail_ctx->mail.offset == mail_ctx->hdr_offset);
array_append(&sync_ctx->mails, &mail_ctx->mail, 1);
sync_ctx->space_diff += mail_ctx->mail.space;
if (sync_ctx->space_diff < 0) {
if (sync_ctx->expunged_space > 0) {
i_assert(sync_ctx->expunged_space ==
mail_ctx->mail.space);
sync_ctx->expunged_space = 0;
}
return 0;
}
/* we have enough space now */
if (mail_ctx->mail.uid == 0) {
/* this message was expunged. fill more or less of the space.
space_diff now consists of a negative "bytes needed" sum,
plus the expunged space of this message. so it contains how
many bytes of _extra_ space we have. */
i_assert(mail_ctx->mail.space >= sync_ctx->space_diff);
extra_space = MBOX_HEADER_PADDING *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
/* don't waste too much on padding */
move_diff = needed_space + extra_space;
sync_ctx->expunged_space =
mail_ctx->mail.space - move_diff;
} else {
move_diff = mail_ctx->mail.space;
extra_space = sync_ctx->space_diff;
sync_ctx->expunged_space = 0;
}
last_seq = sync_ctx->seq - 1;
array_delete(&sync_ctx->mails,
array_count(&sync_ctx->mails) - 1, 1);
end_offset = mail_ctx->mail.from_offset;
} else {
/* this message gave enough space from headers. rewriting stops
at the end of this message's headers. */
sync_ctx->expunged_space = 0;
last_seq = sync_ctx->seq;
end_offset = mail_ctx->body_offset;
move_diff = 0;
extra_space = sync_ctx->space_diff;
}
if (mbox_sync_rewrite(sync_ctx, end_offset, move_diff, extra_space,
sync_ctx->need_space_seq, last_seq) < 0)
return -1;
update_from_offsets(sync_ctx);
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
memset(mail_ctx, 0, sizeof(*mail_ctx));
sync_ctx->need_space_seq = 0;
sync_ctx->space_diff = 0;
array_clear(&sync_ctx->mails);
return 0;
}
static int
mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
{
struct mbox_mailbox *mbox = sync_ctx->mbox;
uoff_t old_offset;
uint32_t uid;
int ret, deleted;
if (seq == 0) {
if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) {
mail_storage_set_error(STORAGE(mbox->storage),
"Mailbox isn't a valid mbox file");
return -1;
}
seq++;
} else {
old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted);
if (ret < 0)
return -1;
if (ret == 0) {
if (istream_raw_mbox_seek(mbox->mbox_stream,
old_offset) < 0) {
mail_storage_set_critical(
STORAGE(mbox->storage),
"Error seeking back to original "
"offset %s in mbox file %s",
dec2str(old_offset), mbox->path);
return -1;
}
return 0;
}
}
if (seq <= 1)
uid = 0;
else if (mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid) < 0) {
mail_storage_set_index_error(&mbox->ibox);
return -1;
}
sync_ctx->prev_msg_uid = uid;
/* set to -1, since it's always increased later */
sync_ctx->seq = seq-1;
if (sync_ctx->seq == 0 &&
istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
/* this mbox has pseudo mail which contains the X-IMAP header */
sync_ctx->seq++;
}
sync_ctx->idx_seq = seq;
sync_ctx->dest_first_mail = sync_ctx->seq == 0;
(void)istream_raw_mbox_get_body_offset(sync_ctx->input);
return 1;
}
static int
mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
{
struct mail_index_view *sync_view = sync_ctx->sync_view;
uint32_t seq1, seq2;
const struct stat *st;
if (mail_index_lookup_uid_range(sync_view, uid, (uint32_t)-1,
&seq1, &seq2) < 0) {
mail_storage_set_index_error(&sync_ctx->mbox->ibox);
return -1;
}
if (seq1 == 0) {
/* doesn't exist anymore, seek to end of file */
st = i_stream_stat(sync_ctx->mbox->mbox_file_stream);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox,
"i_stream_stat()");
return -1;
}
if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream,
st->st_size) < 0) {
mail_storage_set_critical(
STORAGE(sync_ctx->mbox->storage),
"Error seeking to end of mbox file %s",
sync_ctx->mbox->path);
return -1;
}
sync_ctx->idx_seq =
mail_index_view_get_messages_count(sync_view) + 1;
return 1;
}
return mbox_sync_seek_to_seq(sync_ctx, seq1);
}
static int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx,
uint32_t next_uid, int *partial,
int *skipped_mails)
{
uint32_t messages_count;
int ret;
/* delete sync records up to next message. so if there's still
something left in array, it means the next message needs modifying */
mbox_sync_array_delete_to(&sync_ctx->syncs, next_uid);
if (array_count(&sync_ctx->syncs) > 0)
return 1;
if (sync_ctx->sync_rec.uid1 != 0) {
/* we can skip forward to next record which needs updating. */
if (sync_ctx->sync_rec.uid1 != next_uid) {
*skipped_mails = TRUE;
next_uid = sync_ctx->sync_rec.uid1;
}
ret = mbox_sync_seek_to_uid(sync_ctx, next_uid);
} else {
/* if there's no sync records left, we can stop. except if
this is a dirty sync, check if there are new messages. */
if (!sync_ctx->mbox->mbox_sync_dirty)
return 0;
messages_count =
mail_index_view_get_messages_count(sync_ctx->sync_view);
if (sync_ctx->seq + 1 != messages_count) {
ret = mbox_sync_seek_to_seq(sync_ctx, messages_count);
*skipped_mails = TRUE;
} else {
ret = 1;
}
*partial = FALSE;
}
if (ret == 0) {
/* seek failed because the offset is dirty. just ignore and
continue from where we are now. */
*partial = FALSE;
ret = 1;
}
return ret;
}
static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
int partial)
{
const struct mail_index_record *rec;
uint32_t uid, messages_count;
uoff_t offset;
int ret, expunged, skipped_mails;
messages_count =
mail_index_view_get_messages_count(sync_ctx->sync_view);
/* always start from first message so we can read X-IMAP or
X-IMAPbase header */
ret = mbox_sync_seek_to_seq(sync_ctx, 0);
if (ret <= 0)
return ret;
skipped_mails = FALSE;
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
uid = mail_ctx->mail.uid;
if (mail_ctx->seq == 1 && sync_ctx->base_uid_validity != 0 &&
sync_ctx->hdr->uid_validity != 0 &&
sync_ctx->base_uid_validity !=
sync_ctx->hdr->uid_validity) {
mail_storage_set_critical(
STORAGE(sync_ctx->mbox->storage),
"UIDVALIDITY changed (%u -> %u) "
"in mbox file %s",
sync_ctx->hdr->uid_validity,
sync_ctx->base_uid_validity,
sync_ctx->mbox->path);
mail_index_mark_corrupted(sync_ctx->mbox->ibox.index);
return -1;
}
if (mail_ctx->uid_broken && partial) {
/* UID ordering problems, resync everything to make
sure we get everything right */
if (sync_ctx->mbox->mbox_sync_dirty)
return 0;
mail_storage_set_critical(
STORAGE(sync_ctx->mbox->storage),
"UIDs broken with partial sync in mbox file %s",
sync_ctx->mbox->path);
sync_ctx->mbox->mbox_sync_dirty = TRUE;
return 0;
}
if (mail_ctx->pseudo)
uid = 0;
rec = NULL; ret = 1;
if (uid != 0) {
ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
if (ret < 0)
return -1;
}
if (ret == 0) {
/* UID found but it's broken */
uid = 0;
} else if (uid == 0 &&
!mail_ctx->pseudo &&
(sync_ctx->delay_writes ||
sync_ctx->idx_seq <= messages_count)) {
/* If we can't use/store X-UID header, use MD5 sum.
Also check for existing MD5 sums when we're actually
able to write X-UIDs. */
sync_ctx->mbox->mbox_save_md5 = TRUE;
if (mbox_sync_find_index_md5(sync_ctx,
mail_ctx->hdr_md5_sum,
&rec) < 0)
return -1;
if (rec != NULL)
uid = mail_ctx->mail.uid = rec->uid;
}
/* get all sync records related to this message. with pseudo
message just get the first sync record so we can jump to
it with partial seeking. */
if (mbox_sync_read_index_syncs(sync_ctx,
mail_ctx->pseudo ? 1 : uid,
&expunged) < 0)
return -1;
if (mail_ctx->pseudo) {
/* if it was set, it was for the next message */
expunged = FALSE;
} else {
if (rec == NULL) {
/* message wasn't found from index. we have to
read everything from now on, no skipping */
partial = FALSE;
}
}
if (uid == 0 && !mail_ctx->pseudo) {
/* missing/broken X-UID. all the rest of the mails
need new UIDs. */
while (sync_ctx->idx_seq <= messages_count) {
mail_index_expunge(sync_ctx->t,
sync_ctx->idx_seq++);
}
mail_ctx->need_rewrite = TRUE;
mail_ctx->mail.uid = sync_ctx->next_uid++;
sync_ctx->prev_msg_uid = mail_ctx->mail.uid;
}
mail_ctx->mail.idx_seq = sync_ctx->idx_seq;
if (!expunged) {
if (mbox_sync_handle_header(mail_ctx) < 0)
return -1;
sync_ctx->dest_first_mail = FALSE;
} else {
mail_ctx->mail.uid = 0;
mbox_sync_handle_expunge(mail_ctx);
}
if (!mail_ctx->pseudo) {
if (!expunged) {
if (mbox_sync_update_index(mail_ctx, rec) < 0)
return -1;
}
sync_ctx->idx_seq++;
}
istream_raw_mbox_next(sync_ctx->input,
mail_ctx->mail.body_size);
offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
if (mbox_move(sync_ctx,
mail_ctx->body_offset -
sync_ctx->expunged_space,
mail_ctx->body_offset,
mail_ctx->mail.body_size) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
}
} else if (partial) {
ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1,
&partial,
&skipped_mails);
if (ret <= 0) {
if (ret < 0)
return -1;
break;
}
}
}
if (istream_raw_mbox_is_eof(sync_ctx->input)) {
/* rest of the messages in index don't exist -> expunge them */
while (sync_ctx->idx_seq <= messages_count)
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
}
if (!skipped_mails)
sync_ctx->mbox->mbox_sync_dirty = FALSE;
return 1;
}
static int mbox_write_pseudo(struct mbox_sync_context *sync_ctx)
{
string_t *str;
unsigned int uid_validity;
i_assert(sync_ctx->write_fd != -1);
uid_validity = sync_ctx->base_uid_validity != 0 ?
sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity;
i_assert(uid_validity != 0);
str = t_str_new(1024);
str_printfa(str, "%sDate: %s\n"
"From: Mail System Internal Data <MAILER-DAEMON@%s>\n"
"Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA"
"\nMessage-ID: <%s@%s>\n"
"X-IMAP: %u %010u\n"
"Status: RO\n"
"\n"
PSEUDO_MESSAGE_BODY
"\n",
mbox_from_create("MAILER_DAEMON", ioloop_time),
message_date_create(ioloop_time),
my_hostname, dec2str(ioloop_time), my_hostname,
uid_validity, sync_ctx->next_uid-1);
if (pwrite_full(sync_ctx->write_fd,
str_data(str), str_len(str), 0) < 0) {
if (!ENOSPACE(errno)) {
mbox_set_syscall_error(sync_ctx->mbox,
"pwrite_full()");
return -1;
}
/* out of disk space, truncate to empty */
(void)ftruncate(sync_ctx->write_fd, 0);
}
sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */
sync_ctx->base_uid_last = sync_ctx->next_uid-1;
return 0;
}
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
const struct stat *st;
uoff_t file_size, offset, padding, trailer_size;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
i_assert(sync_ctx->expunged_space == 0);
return 0;
}
st = i_stream_stat(sync_ctx->file_input);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
return -1;
}
file_size = st->st_size;
i_assert(file_size >= sync_ctx->file_input->v_offset);
trailer_size = file_size - sync_ctx->file_input->v_offset;
i_assert(trailer_size <= 1);
if (sync_ctx->need_space_seq != 0) {
i_assert(sync_ctx->write_fd != -1);
i_assert(sync_ctx->space_diff < 0);
padding = MBOX_HEADER_PADDING *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
sync_ctx->space_diff -= padding;
i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff);
sync_ctx->space_diff += sync_ctx->expunged_space;
sync_ctx->expunged_space = 0;
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
i_assert(sync_ctx->space_diff < 0);
if (file_set_size(sync_ctx->write_fd,
file_size + -sync_ctx->space_diff) < 0) {
mbox_set_syscall_error(sync_ctx->mbox,
"file_set_size()");
return -1;
}
i_stream_sync(sync_ctx->input);
if (mbox_sync_rewrite(sync_ctx, file_size,
-sync_ctx->space_diff, padding,
sync_ctx->need_space_seq,
sync_ctx->seq) < 0)
return -1;
update_from_offsets(sync_ctx);
sync_ctx->need_space_seq = 0;
array_clear(&sync_ctx->mails);
}
if (sync_ctx->expunged_space > 0) {
i_assert(sync_ctx->write_fd != -1);
/* copy trailer, then truncate the file */
st = i_stream_stat(sync_ctx->file_input);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox,
"i_stream_stat()");
return -1;
}
file_size = st->st_size;
if (file_size == (uoff_t)sync_ctx->expunged_space) {
/* everything deleted, the trailer_size still contains
the \n trailer though */
trailer_size = 0;
}
i_assert(file_size >= sync_ctx->expunged_space + trailer_size);
offset = file_size - sync_ctx->expunged_space - trailer_size;
i_assert(offset == 0 || offset > 31);
if (mbox_move(sync_ctx, offset,
offset + sync_ctx->expunged_space,
trailer_size) < 0)
return -1;
if (ftruncate(sync_ctx->write_fd,
offset + trailer_size) < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
return -1;
}
if (offset == 0) {
if (mbox_write_pseudo(sync_ctx) < 0)
return -1;
}
sync_ctx->expunged_space = 0;
i_stream_sync(sync_ctx->input);
}
return 0;
}
static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
{
const struct stat *st;
st = i_stream_stat(sync_ctx->file_input);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
return -1;
}
/* only reason not to have UID validity at this point is if the file
is entirely empty. In that case just make up a new one if needed. */
i_assert(sync_ctx->base_uid_validity != 0 || st->st_size == 0);
if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity ||
sync_ctx->base_uid_validity == 0) {
if (sync_ctx->base_uid_validity == 0) {
sync_ctx->base_uid_validity =
sync_ctx->hdr->uid_validity != 0 ?
sync_ctx->hdr->uid_validity :
(unsigned int)ioloop_time;
}
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, uid_validity),
&sync_ctx->base_uid_validity,
sizeof(sync_ctx->base_uid_validity), TRUE);
}
if (istream_raw_mbox_is_eof(sync_ctx->input) &&
sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
i_assert(sync_ctx->next_uid != 0);
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, next_uid),
&sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
}
if ((uint32_t)st->st_mtime != sync_ctx->hdr->sync_stamp &&
!sync_ctx->mbox->mbox_sync_dirty) {
uint32_t sync_stamp = st->st_mtime;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_stamp),
&sync_stamp, sizeof(sync_stamp), TRUE);
}
if ((uint64_t)st->st_size != sync_ctx->hdr->sync_size &&
!sync_ctx->mbox->mbox_sync_dirty) {
uint64_t sync_size = st->st_size;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_size),
&sync_size, sizeof(sync_size), TRUE);
}
sync_ctx->mbox->mbox_dirty_stamp = st->st_mtime;
sync_ctx->mbox->mbox_dirty_size = st->st_size;
return 0;
}
static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->base_uid_last_offset = 0;
array_clear(&sync_ctx->mails);
array_clear(&sync_ctx->syncs);
memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
mail_index_sync_reset(sync_ctx->index_sync_ctx);
sync_ctx->prev_msg_uid = 0;
sync_ctx->next_uid = sync_ctx->hdr->next_uid;
sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
sync_ctx->seq = 0;
sync_ctx->idx_seq = 1;
sync_ctx->need_space_seq = 0;
sync_ctx->expunged_space = 0;
sync_ctx->space_diff = 0;
sync_ctx->dest_first_mail = TRUE;
}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx,
enum mbox_sync_flags flags)
{
struct mbox_sync_mail_context mail_ctx;
const struct stat *st;
int ret, partial;
st = i_stream_stat(sync_ctx->file_input);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox,
"i_stream_stat()");
return -1;
}
if ((flags & MBOX_SYNC_FORCE_SYNC) != 0) {
/* forcing a full sync. assume file has changed. */
partial = FALSE;
sync_ctx->mbox->mbox_sync_dirty = TRUE;
} else if ((uint32_t)st->st_mtime == sync_ctx->hdr->sync_stamp &&
(uint64_t)st->st_size == sync_ctx->hdr->sync_size) {
/* file is fully synced */
partial = TRUE;
sync_ctx->mbox->mbox_sync_dirty = FALSE;
} else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
(uint64_t)st->st_size == sync_ctx->hdr->sync_size) {
/* we want to do full syncing. always do this if
file size hasn't changed but timestamp has. it most
likely means that someone had modified some header
and we probably want to know about it */
partial = FALSE;
sync_ctx->mbox->mbox_sync_dirty = TRUE;
} else {
/* see if we can delay syncing the whole file.
normally we only notice expunges and appends
in partial syncing. */
partial = TRUE;
sync_ctx->mbox->mbox_sync_dirty = TRUE;
}
mbox_sync_restart(sync_ctx);
ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial);
if (ret <= 0) {
if (ret < 0)
return -1;
/* partial syncing didn't work, do it again */
i_assert(sync_ctx->mbox->mbox_sync_dirty);
mbox_sync_restart(sync_ctx);
mail_index_transaction_rollback(sync_ctx->t);
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view,
FALSE, TRUE);
ret = mbox_sync_loop(sync_ctx, &mail_ctx, FALSE);
if (ret <= 0) {
i_assert(ret != 0);
return -1;
}
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
array_clear(&sync_ctx->syncs);
memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync_has_changed(struct mbox_mailbox *mbox, int leave_dirty)
{
const struct mail_index_header *hdr;
const struct stat *st;
struct stat statbuf;
if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) {
/* read-only stream */
st = i_stream_stat(mbox->mbox_file_stream);
if (st == NULL) {
mbox_set_syscall_error(mbox, "i_stream_stat()");
return -1;
}
} else {
if (stat(mbox->path, &statbuf) < 0) {
mbox_set_syscall_error(mbox, "stat()");
return -1;
}
st = &statbuf;
}
hdr = mail_index_get_header(mbox->ibox.view);
if ((uint32_t)st->st_mtime == hdr->sync_stamp &&
(uint64_t)st->st_size == hdr->sync_size) {
/* fully synced */
mbox->mbox_sync_dirty = FALSE;
return 0;
}
if (!mbox->mbox_sync_dirty || !leave_dirty)
return 1;
return st->st_mtime != mbox->mbox_dirty_stamp ||
st->st_size != mbox->mbox_dirty_size;
}
int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
uint32_t seq;
uoff_t offset;
unsigned int lock_id = 0;
int ret, changed;
mbox->ibox.sync_last_check = ioloop_time;
if (!mbox->mbox_do_dirty_syncs)
flags |= MBOX_SYNC_UNDIRTY;
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
if (mbox_lock(mbox, F_RDLCK, &lock_id) <= 0)
return -1;
}
if ((flags & MBOX_SYNC_HEADER) != 0 ||
(flags & MBOX_SYNC_FORCE_SYNC) != 0)
changed = 1;
else {
int leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
if ((changed = mbox_sync_has_changed(mbox, leave_dirty)) < 0) {
if ((flags & MBOX_SYNC_LOCK_READING) != 0)
(void)mbox_unlock(mbox, lock_id);
return -1;
}
}
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
/* we just want to lock it for reading. if mbox hasn't been
modified don't do any syncing. */
if (!changed)
return 0;
/* have to sync to make sure offsets have stayed the same */
(void)mbox_unlock(mbox, lock_id);
lock_id = 0;
}
/* reopen input stream to make sure it has nothing buffered */
mbox_file_close_stream(mbox);
__again:
if (changed) {
/* we're most likely modifying the mbox while syncing, just
lock it for writing immediately. the mbox must be locked
before index syncing is started to avoid deadlocks, so we
don't have much choice either (well, easy ones anyway). */
int lock_type = mbox->mbox_readonly ? F_RDLCK : F_WRLCK;
if (mbox_lock(mbox, lock_type, &lock_id) <= 0)
return -1;
}
if ((flags & MBOX_SYNC_LAST_COMMIT) != 0) {
seq = mbox->ibox.commit_log_file_seq;
offset = mbox->ibox.commit_log_file_offset;
} else {
seq = (uint32_t)-1;
offset = (uoff_t)-1;
}
ret = mail_index_sync_begin(mbox->ibox.index, &index_sync_ctx,
&sync_view, seq, offset,
!mbox->ibox.keep_recent,
(flags & MBOX_SYNC_REWRITE) != 0);
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(&mbox->ibox);
if (lock_id != 0)
(void)mbox_unlock(mbox, lock_id);
return ret;
}
if (!changed && !mail_index_sync_have_more(index_sync_ctx)) {
/* nothing to do */
if (lock_id != 0)
(void)mbox_unlock(mbox, lock_id);
/* index may need to do internal syncing though, so commit
instead of rollbacking. */
if (mail_index_sync_commit(index_sync_ctx) < 0) {
mail_storage_set_index_error(&mbox->ibox);
return -1;
}
return 0;
}
if (lock_id == 0) {
/* ok, we have something to do but no locks. we'll have to
restart syncing to avoid deadlocking. */
mail_index_sync_rollback(index_sync_ctx);
changed = 1;
goto __again;
}
if (mbox_file_open_stream(mbox) < 0) {
mail_index_sync_rollback(index_sync_ctx);
(void)mbox_unlock(mbox, lock_id);
return -1;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.mbox = mbox;
sync_ctx.hdr = mail_index_get_header(sync_view);
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE, TRUE);
sync_ctx.mail_keyword_pool = pool_alloconly_create("keywords", 4096);
/* make sure we've read the latest keywords in index */
(void)mail_index_get_keywords(mbox->ibox.index);
ARRAY_CREATE(&sync_ctx.mails, default_pool,
struct mbox_sync_mail, 64);
ARRAY_CREATE(&sync_ctx.syncs, default_pool,
struct mail_index_sync_rec, 32);
sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream;
sync_ctx.input = sync_ctx.mbox->mbox_stream;
sync_ctx.write_fd = sync_ctx.mbox->mbox_readonly ? -1 :
sync_ctx.mbox->mbox_fd;
sync_ctx.flags = flags;
sync_ctx.delay_writes = sync_ctx.mbox->mbox_readonly ||
sync_ctx.mbox->ibox.readonly ||
((flags & MBOX_SYNC_REWRITE) == 0 &&
getenv("MBOX_LAZY_WRITES") != NULL);
ret = mbox_sync_do(&sync_ctx, flags);
if (ret < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0) {
mail_storage_set_index_error(&mbox->ibox);
ret = -1;
} else {
mbox->ibox.commit_log_file_seq = 0;
mbox->ibox.commit_log_file_offset = 0;
}
sync_ctx.t = NULL;
if (ret < 0)
mail_index_sync_rollback(index_sync_ctx);
else if (mail_index_sync_commit(index_sync_ctx) < 0) {
mail_storage_set_index_error(&mbox->ibox);
ret = -1;
}
if (sync_ctx.base_uid_last != sync_ctx.next_uid-1 &&
ret == 0 && !sync_ctx.delay_writes &&
sync_ctx.base_uid_last_offset != 0) {
/* Rewrite uid_last in X-IMAPbase header if we've seen it
(ie. the file isn't empty) */
ret = mbox_rewrite_base_uid_last(&sync_ctx);
}
if (ret == 0 && mbox->mbox_lock_type == F_WRLCK &&
!mbox->mbox_writeonly) {
if (fsync(mbox->mbox_fd) < 0) {
mbox_set_syscall_error(mbox, "fsync()");
ret = -1;
}
}
if (lock_id != 0 && mbox->mbox_lock_type != F_RDLCK) {
/* drop to read lock */
unsigned int read_lock_id = 0;
if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0)
ret = -1;
else {
if (mbox_unlock(mbox, lock_id) < 0)
ret = -1;
lock_id = read_lock_id;
}
}
if (lock_id != 0 && (flags & MBOX_SYNC_LOCK_READING) == 0) {
/* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
notice changes made by others .. and this has to be done
even if lock_reading is set.. except if
mbox_sync_dirty = TRUE */
if (mbox_unlock(mbox, lock_id) < 0)
ret = -1;
}
pool_unref(sync_ctx.mail_keyword_pool);
str_free(sync_ctx.header);
str_free(sync_ctx.from_line);
array_free(&sync_ctx.mails);
array_free(&sync_ctx.syncs);
return ret;
}
struct mailbox_sync_context *
mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
enum mbox_sync_flags mbox_sync_flags = 0;
int ret = 0;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
ioloop_time) {
if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
!mbox->mbox_very_dirty_syncs)
mbox_sync_flags |= MBOX_SYNC_UNDIRTY;
if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0)
mbox_sync_flags |= MBOX_SYNC_REWRITE;
ret = mbox_sync(mbox, mbox_sync_flags);
}
return index_mailbox_sync_init(box, flags, ret < 0);
}