mbox-sync.c revision a393d9d6dabdc46cf724f8cb004a652b4036d53d
bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (C) 2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/*
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen required disk I/O. We may need to:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen - Write missing X-UID and X-IMAPbase headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing or broken Content-Length header if there's space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Expunge specified messages
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen
6fe91298731abf0b70dfd796ecc30d3be81fa5d1Timo Sirainen Here's how we do it:
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen
436adac819e7cbeef04af08dcc6a4f3ecd4a1d9eMartti Rannanjärvi - Start reading the mails mail headers from the beginning
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - X-Keywords and X-UID headers may contain extra spaces at the end of them,
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen remember how much extra each message has and offset to beginning of the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen spaces
73a552a9ed06cd6017ad4ee4b252a8b38c8ac42dTimo Sirainen - If message flags are dirty and there's enough space to write them, do it
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we didn't have enough space, remember how much was missing and keep
73a552a9ed06cd6017ad4ee4b252a8b38c8ac42dTimo Sirainen the total amount of them
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - When we encounter expunged message, check if the amount of empty space in
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen previous messages plus size of expunged message is enough to cover the
7ccdf78cd45aea9d14e048a5b9f077515c71978fTimo Sirainen missing space. If yes,
7ccdf78cd45aea9d14e048a5b9f077515c71978fTimo Sirainen - execute the rewrite plan
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen - forget all the messages before the expunged message. only remember
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen how much data we still have to move to cover the expunged message
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen - If we encounter end of file, grow the file and execute the rewrite plan
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen Rewrite plan goes:
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen - Start from the first message that needs more space
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen - If there's expunged messages before us, we have to write over them.
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen - Move all messages after it backwards to fill it
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen - Each moved message's X-Keywords header should have n bytes extra
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen space, unless there's not enough space to do it.
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen - If there's no expunged messages, we can move data either forward or
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen backward to get it. Calculate which requires less moving. Forward
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen counting may encounter more messages which require extra space, count
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen that too.
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen - If we decide to move forwards and we had to go through dirty
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen messages, do the moving from last to first dirty message
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen - If we encounter end of file, grow the file enough to get the required
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen amount of space plus enough space to fill X-Keywords headers full of
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen spaces.
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen*/
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen#include "lib.h"
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen#include "ioloop.h"
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen#include "buffer.h"
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen#include "istream.h"
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen#include "file-set-size.h"
d22301419109ed4a38351715e6760011421dadecTimo Sirainen#include "str.h"
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen#include "write-full.h"
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen#include "istream-raw-mbox.h"
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen#include "mbox-storage.h"
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen#include "mbox-file.h"
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen#include "mbox-lock.h"
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen#include "mbox-sync-private.h"
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen#include <stddef.h>
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen#include <sys/stat.h>
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen
817d027593510c3ba70ad542ce0011f5f6916d1eTimo Sirainen#define MBOX_SYNC_SECS 1
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen/* returns -1 = error, 0 = mbox changed since previous lock, 1 = didn't */
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainenstatic int mbox_sync_lock(struct mbox_sync_context *sync_ctx, int lock_type)
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen{
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen struct index_mailbox *ibox = sync_ctx->ibox;
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen struct stat old_st, st;
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen uoff_t old_from_offset = 0, old_offset = 0;
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen i_assert(lock_type != F_WRLCK || !ibox->mbox_readonly);
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (sync_ctx->lock_id == 0 || sync_ctx->input == NULL) {
00efa7d99981e18e286c02b18c1163dde18ee521Timo Sirainen memset(&old_st, 0, sizeof(old_st));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (sync_ctx->lock_id != 0) {
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen (void)mbox_unlock(ibox, sync_ctx->lock_id);
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen sync_ctx->lock_id = 0;
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen }
641f0c0900ee6e7cf9667f4b40ed95cec7d0cdcaTimo Sirainen } else {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (fstat(sync_ctx->fd, &old_st) < 0) {
6013fbad6638795a00e6c2a2dd2cdbee19612494Timo Sirainen mbox_set_syscall_error(ibox, "stat()");
7ede6554e451ec039a67beec7d6ee4aff61d386eTimo Sirainen return -1;
7ede6554e451ec039a67beec7d6ee4aff61d386eTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen old_from_offset =
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen old_offset = sync_ctx->input->v_offset;
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen
e0cc94d52184944c5da76b509da184e8449b3a5cTimo Sirainen (void)mbox_unlock(ibox, sync_ctx->lock_id);
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen sync_ctx->lock_id = 0;
1235bbd139a1ff97f641fa0e77205eb9adbb0400Timo Sirainen }
1235bbd139a1ff97f641fa0e77205eb9adbb0400Timo Sirainen
1235bbd139a1ff97f641fa0e77205eb9adbb0400Timo Sirainen if (mbox_lock(ibox, lock_type, &sync_ctx->lock_id) <= 0)
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen return -1;
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen if (mbox_file_open_stream(ibox) < 0)
641f0c0900ee6e7cf9667f4b40ed95cec7d0cdcaTimo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen sync_ctx->file_input = sync_ctx->ibox->mbox_file_stream;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen sync_ctx->input = sync_ctx->ibox->mbox_stream;
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen sync_ctx->fd = sync_ctx->ibox->mbox_fd;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen if (old_st.st_mtime == 0) {
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen /* we didn't have the file open before -> it changed */
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen return 0;
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen }
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen if (fstat(sync_ctx->fd, &st) < 0) {
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen mbox_set_syscall_error(ibox, "fstat()");
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen return -1;
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen }
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen if (st.st_mtime != old_st.st_mtime || st.st_size != old_st.st_size ||
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen st.st_ino != old_st.st_ino ||
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen !CMP_DEV_T(st.st_dev, old_st.st_dev) ||
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen time(NULL) - st.st_mtime <= MBOX_SYNC_SECS)
2bde8972f2dcec46d96543407cc5b56954054359Timo Sirainen return 0;
2f2823ad8928654ed405467c6c1f4fd4c6f5cf7cTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* same as before. we'll have to fix mbox stream to contain
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen correct from_offset, hdr_offset and body_offset. so, seek
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen to from_offset and read through the header. */
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, old_from_offset) < 0) {
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen mail_storage_set_critical(ibox->box.storage,
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen "Message offset %s changed unexpectedly for mbox file "
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen "%s", dec2str(old_from_offset), sync_ctx->ibox->path);
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen return 0;
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen }
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen i_stream_seek(sync_ctx->input, old_offset);
7fc0f80480063a9d4cb9e8c07b50db2a5627799eTimo Sirainen return 1;
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi}
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen{
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen " from mbox file %s", from_offset,
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi sync_ctx->ibox->path);
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen return -1;
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen }
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen return 0;
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen}
825f6569a5276488133796c2f529c65128a09ba0Timo Sirainen
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainenstatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen struct mbox_sync_mail_context *mail_ctx,
00efa7d99981e18e286c02b18c1163dde18ee521Timo Sirainen uoff_t grow_size)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uoff_t src_offset, file_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen i_assert(grow_size > 0);
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen /* put the extra space between last message's header and body */
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen if (file_set_size(sync_ctx->fd, file_size) < 0) {
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen return -1;
a8c1d873ebe624cf65893d79e1a509203116cb9aTimo Sirainen }
7ccdf78cd45aea9d14e048a5b9f077515c71978fTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen src_offset = mail_ctx->body_offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_ctx->body_offset += grow_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
f7ad1162969feff6b08f0e640a928db1783daae9Timo Sirainen file_size - mail_ctx->body_offset) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen istream_raw_mbox_flush(sync_ctx->input);
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen return 0;
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen{
b0ab23e6fd425bf864cdc9fdab0631573d4d1b5fTimo Sirainen struct mail_index_sync_rec *sync;
785d9cca224d33ca3938e9166784f6483e8a27d7Timo Sirainen size_t size, src, dest;
039e42997fe5e0d1c5ad9306dc0ae69bf0e1ca10Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen sync = buffer_get_modifyable_data(syncs_buf, &size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen size /= sizeof(*sync);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen for (src = dest = 0; src < size; src++) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (sync[src].uid2 >= uid) {
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen if (src != dest)
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen sync[dest] = sync[src];
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen dest++;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen }
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen }
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen}
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainenstatic int
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen struct mbox_sync_mail_context *mail_ctx)
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen{
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* get EOF */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input))
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen return 0;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen mail_ctx->sync_ctx = sync_ctx;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen mail_ctx->seq = ++sync_ctx->seq;
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen mail_ctx->header = sync_ctx->header;
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen mail_ctx->from_offset =
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen mail_ctx->mail.offset =
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi if (mail_ctx->seq == 1)
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen sync_ctx->seen_first_mail = TRUE;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen if (mail_ctx->seq > 1 && sync_ctx->dest_first_mail) {
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen /* First message was expunged and this is the next one.
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen Skip \n header */
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen mail_ctx->from_offset++;
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen }
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen mail_ctx->mail.body_size =
ce8244c05b9f89d7729a5abe6f4838c1dce46c64Timo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen mail_ctx->content_length);
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* save the offset permanently with recent flag state */
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen mail_ctx->mail.from_offset = mail_ctx->from_offset;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* need to add 'O' flag to Status-header */
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen mail_ctx->need_rewrite = TRUE;
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen // FIXME: save it somewhere
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen }
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen return 1;
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen}
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainenstatic int mbox_sync_buf_have_expunges(buffer_t *syncs_buf)
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen{
ce8244c05b9f89d7729a5abe6f4838c1dce46c64Timo Sirainen const struct mail_index_sync_rec *sync;
ce8244c05b9f89d7729a5abe6f4838c1dce46c64Timo Sirainen size_t size, i;
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen
ce8244c05b9f89d7729a5abe6f4838c1dce46c64Timo Sirainen sync = buffer_get_data(syncs_buf, &size);
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen size /= sizeof(*sync);
ce8244c05b9f89d7729a5abe6f4838c1dce46c64Timo Sirainen
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen for (i = 0; i < size; i++) {
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen if (sync[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi return TRUE;
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen }
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen return FALSE;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen uint32_t uid, int *sync_expunge_r)
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen{
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen int ret;
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen *sync_expunge_r = FALSE;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen if (sync_ctx->ibox->mbox_readonly || sync_ctx->index_sync_ctx == NULL)
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen return 0;
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen if (uid == 0) {
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen /* nothing for this or the future ones */
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi uid = (uint32_t)-1;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen }
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen
6cfb1377b2c034cfc4bd37361cc1520f2d8acd6dTimo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen while (uid >= sync_rec->uid1) {
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen if (uid <= sync_rec->uid2 &&
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND) {
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen buffer_append(sync_ctx->syncs, sync_rec,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen sizeof(*sync_rec));
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen
6cfb1377b2c034cfc4bd37361cc1520f2d8acd6dTimo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
32ee977e189266744ef69ac4e832fd3111d6f949Timo Sirainen *sync_expunge_r = TRUE;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen }
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen if (ret < 0) {
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen return -1;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen }
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen if (ret == 0) {
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen memset(sync_rec, 0, sizeof(*sync_rec));
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen break;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen }
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) {
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen if (sync_rec->uid2 >= sync_ctx->next_uid) {
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen sync_ctx->next_uid = sync_rec->uid2 + 1;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen sync_ctx->update_base_uid_last = sync_rec->uid2;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen }
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen memset(sync_rec, 0, sizeof(*sync_rec));
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen }
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen }
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen if (!*sync_expunge_r)
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen *sync_expunge_r = mbox_sync_buf_have_expunges(sync_ctx->syncs);
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen return 0;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen}
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainenstatic void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen keywords_mask_t keywords)
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen{
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen const struct mail_index_sync_rec *sync;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen size_t size, i;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen sync = buffer_get_data(syncs_buf, &size);
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen size /= sizeof(*sync);
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen for (i = 0; i < size; i++) {
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen if (sync[i].type != MAIL_INDEX_SYNC_TYPE_FLAGS)
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen continue;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen mail_index_sync_flags_apply(&sync[i], flags, keywords);
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen }
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen}
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainenstatic int
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen{
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen const struct mail_index_record *rec = NULL;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen uint32_t messages_count;
f6f021c133f680cf3d559187524fd9abcbaae9b9Timo Sirainen int ret = 0;
messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
while (sync_ctx->idx_seq <= messages_count) {
ret = mail_index_lookup(sync_ctx->sync_view,
sync_ctx->idx_seq, &rec);
if (ret < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
if (uid <= rec->uid)
break;
/* externally expunged message, remove from index */
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
sync_ctx->idx_seq++;
rec = NULL;
}
if (rec != NULL && rec->uid != uid) {
/* new UID in the middle of the mailbox - shouldn't happen */
mail_storage_set_critical(sync_ctx->ibox->box.storage,
"mbox sync: UID inserted in the middle of mailbox %s "
"(%u > %u)", sync_ctx->ibox->path, rec->uid, uid);
mail_index_mark_corrupted(sync_ctx->ibox->index);
return -1;
}
*rec_r = rec;
return ret;
}
static int mbox_sync_get_from_offset(struct mbox_sync_context *sync_ctx,
uint32_t seq, uint64_t *offset_r)
{
const void *data;
/* see if from_offset needs updating */
if (mail_index_lookup_extra(sync_ctx->sync_view, seq,
sync_ctx->ibox->mbox_extra_idx,
&data) < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
*offset_r = *((const uint64_t *)data);
return 0;
}
static int
mbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mail,
int nocheck)
{
uint64_t offset;
if (!nocheck) {
if (mbox_sync_get_from_offset(sync_ctx, sync_ctx->idx_seq,
&offset) < 0)
return -1;
if (offset == mail->from_offset)
return 0;
}
offset = mail->from_offset;
mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->ibox->mbox_extra_idx, &offset);
return 0;
}
static int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mail,
const struct mail_index_record *rec)
{
keywords_mask_t idx_keywords;
uint8_t idx_flags, mbox_flags;
if (rec == NULL) {
/* new message */
mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags,
mail->keywords);
} else {
/* see if flags changed */
idx_flags = rec->flags;
memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
mbox_sync_apply_index_syncs(sync_ctx->syncs,
&idx_flags, idx_keywords);
mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
(mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT));
if (idx_flags != mbox_flags ||
memcmp(idx_keywords, mail->keywords,
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags,
mail->keywords);
}
}
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
return -1;
}
return 0;
}
static int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
{
struct istream *input = ctx->sync_ctx->file_input;
const unsigned char *data;
size_t size, from_line_size;
buffer_set_used_size(ctx->sync_ctx->from_line, 0);
from_line_size = ctx->hdr_offset - ctx->from_offset;
i_stream_seek(input, ctx->from_offset);
for (;;) {
data = i_stream_get_data(input, &size);
if (size >= from_line_size)
size = from_line_size;
buffer_append(ctx->sync_ctx->from_line, data, size);
i_stream_skip(input, size);
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int
mbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
{
string_t *str = ctx->sync_ctx->from_line;
if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
ctx->from_offset + move_diff) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
return -1;
}
istream_raw_mbox_flush(ctx->sync_ctx->input);
return 0;
}
static void update_from_offsets(struct mbox_sync_context *sync_ctx)
{
const struct mbox_sync_mail *mails;
uint32_t idx, idx_seq, extra_idx = sync_ctx->ibox->mbox_extra_idx;
uint64_t offset;
size_t size;
mails = buffer_get_modifyable_data(sync_ctx->mails, &size);
size /= sizeof(*mails);
i_assert(sync_ctx->need_space_seq - sync_ctx->seq + 1 == size);
idx = 0;
idx_seq = sync_ctx->need_space_idx_seq;
if (idx_seq == 0) {
idx++; idx_seq++;
}
for (; idx < size; idx++, idx_seq++, mails++) {
if (mails->uid == 0)
continue;
offset = mails->from_offset;
mail_index_update_extra_rec(sync_ctx->t, idx_seq, extra_idx,
&offset);
}
}
static int mbox_sync_check_excl_lock(struct mbox_sync_context *sync_ctx)
{
int ret;
if (sync_ctx->ibox->mbox_lock_type == F_RDLCK) {
if ((ret = mbox_sync_lock(sync_ctx, F_WRLCK)) < 0)
return -1;
if (ret == 0)
return -2;
}
return 0;
}
static int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
{
int ret;
if ((ret = mbox_sync_check_excl_lock(mail_ctx->sync_ctx)) < 0)
return ret;
mail_ctx->mail.offset = mail_ctx->from_offset;
mail_ctx->mail.space =
mail_ctx->body_offset - mail_ctx->from_offset +
mail_ctx->mail.body_size;
mail_ctx->mail.body_size = 0;
if (mail_ctx->sync_ctx->dest_first_mail) {
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
mail_ctx->mail.space++;
}
mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
return 0;
}
static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
off_t move_diff;
int ret;
if (sync_ctx->ibox->mbox_readonly)
return 0;
if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
/* move the header backwards to fill expunged space */
if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
return ret;
move_diff = -sync_ctx->expunged_space;
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
mail_ctx->mail.from_offset += move_diff;
mail_ctx->mail.offset += move_diff;
if (mbox_write_from_line(mail_ctx, move_diff) < 0)
return -1;
}
} else if (mail_ctx->need_rewrite ||
buffer_get_used_size(sync_ctx->syncs) != 0 ||
(mail_ctx->seq == 1 &&
sync_ctx->update_base_uid_last != 0)) {
if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
return ret;
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
return -1;
} else {
/* nothing to do */
return 0;
}
if (ret == 0 && sync_ctx->need_space_seq == 0) {
/* first mail with no space to write it */
sync_ctx->need_space_seq = sync_ctx->seq;
sync_ctx->need_space_idx_seq = sync_ctx->idx_seq;
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
memset(&mail, 0, sizeof(mail));
mail.offset = mail_ctx->from_offset -
sync_ctx->expunged_space;
mail.space = sync_ctx->expunged_space;
sync_ctx->need_space_seq--;
buffer_append(sync_ctx->mails, &mail, sizeof(mail));
}
}
return 0;
}
static int
mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t extra_space;
buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
sync_ctx->space_diff += mail_ctx->mail.space;
if (sync_ctx->space_diff < 0)
return 0;
/* we have enough space now */
extra_space = MBOX_HEADER_EXTRA_SPACE *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
if (mail_ctx->mail.uid == 0 &&
(uoff_t)sync_ctx->space_diff > extra_space) {
/* don't waste too much on extra spacing */
sync_ctx->expunged_space = sync_ctx->space_diff - extra_space;
sync_ctx->space_diff = extra_space;
} else {
sync_ctx->expunged_space = 0;
}
if (mbox_sync_rewrite(sync_ctx, sync_ctx->space_diff,
sync_ctx->need_space_seq, sync_ctx->seq) < 0)
return -1;
update_from_offsets(sync_ctx);
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
memset(mail_ctx, 0, sizeof(*mail_ctx));
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
return 0;
}
static int
mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
{
uint32_t seq1, seq2;
uint64_t offset;
if (mail_index_lookup_uid_range(sync_ctx->sync_view, uid, (uint32_t)-1,
&seq1, &seq2) < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
if (seq1 == 0)
return 0;
if (mbox_sync_get_from_offset(sync_ctx, seq1, &offset) < 0)
return -1;
/* set to -1, since it's always increased later */
sync_ctx->seq = seq1-1;
sync_ctx->idx_seq = seq1;
sync_ctx->dest_first_mail = sync_ctx->seq == 0;
if (istream_raw_mbox_seek(sync_ctx->input, offset) < 0) {
mail_storage_set_critical(sync_ctx->ibox->box.storage,
"Cached message offset %s is invalid for mbox file %s",
dec2str(offset), sync_ctx->ibox->path);
mail_index_mark_corrupted(sync_ctx->ibox->index);
return -1;
}
(void)istream_raw_mbox_get_body_offset(sync_ctx->input);
return 1;
}
static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
uint32_t min_message_count)
{
const struct mail_index_record *rec;
uint32_t uid, messages_count;
uoff_t offset;
int ret, expunged;
if (min_message_count != 0)
ret = 0;
else {
/* we sync only what we need to. jump to first record that
needs updating */
const struct mail_index_sync_rec *sync_rec;
size_t size;
if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
sync_ctx->sync_rec.uid1 == 0) {
if (mbox_sync_read_index_syncs(sync_ctx, 1,
&expunged) < 0)
return -1;
if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
sync_ctx->sync_rec.uid1 == 0) {
/* nothing to do */
return 0;
}
}
sync_rec = buffer_get_data(sync_ctx->syncs, &size);
if (size == 0)
sync_rec = &sync_ctx->sync_rec;
ret = mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1);
if (ret < 0)
return -1;
}
if (ret == 0) {
if (istream_raw_mbox_seek(sync_ctx->input, 0) < 0) {
/* doesn't begin with a From-line */
mail_storage_set_error(sync_ctx->ibox->box.storage,
"Mailbox isn't a valid mbox file");
return -1;
}
sync_ctx->dest_first_mail = TRUE;
}
messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
uid = mail_ctx->mail.uid;
/* get all sync records related to this message */
if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
return -1;
rec = NULL;
if (uid != 0 && !mail_ctx->pseudo) {
ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
if (ret < 0)
return -1;
if (ret == 0 && uid < sync_ctx->hdr->next_uid) {
/* this UID was already in index and it was
expunged */
uid = 0;
rec = NULL;
}
}
if (uid == 0) {
/* missing/broken X-UID. all the rest of the mails
need new UIDs. */
while (sync_ctx->idx_seq <= messages_count) {
mail_index_expunge(sync_ctx->t,
sync_ctx->idx_seq++);
}
mail_ctx->need_rewrite = TRUE;
mail_ctx->mail.uid = sync_ctx->next_uid++;
sync_ctx->prev_msg_uid = mail_ctx->mail.uid;
}
if (!expunged) {
ret = mbox_sync_handle_header(mail_ctx);
sync_ctx->dest_first_mail = FALSE;
} else {
mail_ctx->mail.uid = 0;
ret = mbox_sync_handle_expunge(mail_ctx);
}
if (ret < 0) {
/* -1 = error, -2 = need to restart */
return ret;
}
if (!expunged && !mail_ctx->pseudo) {
if (mbox_sync_update_index(sync_ctx, &mail_ctx->mail,
rec) < 0)
return -1;
}
sync_ctx->idx_seq++;
istream_raw_mbox_next(sync_ctx->input,
mail_ctx->mail.body_size);
offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
if (mbox_move(sync_ctx,
mail_ctx->body_offset -
sync_ctx->expunged_space,
mail_ctx->body_offset,
mail_ctx->mail.body_size) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
}
} else if (sync_ctx->seq >= min_message_count) {
mbox_sync_buffer_delete_old(sync_ctx->syncs, uid+1);
if (buffer_get_used_size(sync_ctx->syncs) == 0) {
/* if there's no sync records left,
we can stop */
if (sync_ctx->sync_rec.uid1 == 0)
break;
/* we can skip forward to next record which
needs updating. */
uid = sync_ctx->sync_rec.uid1;
if (mbox_sync_seek_to_uid(sync_ctx, uid) < 0)
return -1;
}
}
}
if (istream_raw_mbox_is_eof(sync_ctx->input)) {
/* rest of the messages in index don't exist -> expunge them */
while (sync_ctx->idx_seq <= messages_count)
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
}
return 0;
}
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
uoff_t offset, extra_space, trailer_size;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
i_assert(sync_ctx->expunged_space == 0);
return 0;
}
trailer_size = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->file_input->v_offset;
if (sync_ctx->need_space_seq != 0) {
i_assert(sync_ctx->space_diff < 0);
extra_space = MBOX_HEADER_EXTRA_SPACE *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
sync_ctx->space_diff -= extra_space;
sync_ctx->space_diff += sync_ctx->expunged_space;
if (sync_ctx->expunged_space <= -sync_ctx->space_diff)
sync_ctx->expunged_space = 0;
else
sync_ctx->expunged_space -= -sync_ctx->space_diff;
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
if (sync_ctx->space_diff < 0 &&
mbox_sync_grow_file(sync_ctx, mail_ctx,
-sync_ctx->space_diff) < 0)
return -1;
if (mbox_sync_try_rewrite(mail_ctx, 0) < 0)
return -1;
if (sync_ctx->seq != sync_ctx->need_space_seq) {
buffer_set_used_size(sync_ctx->mails,
(sync_ctx->seq -
sync_ctx->need_space_seq) *
sizeof(mail_ctx->mail));
buffer_append(sync_ctx->mails, &mail_ctx->mail,
sizeof(mail_ctx->mail));
if (mbox_sync_rewrite(sync_ctx, extra_space,
sync_ctx->need_space_seq,
sync_ctx->seq) < 0)
return -1;
}
update_from_offsets(sync_ctx);
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
}
if (sync_ctx->expunged_space > 0) {
/* copy trailer, then truncate the file */
offset = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->expunged_space - trailer_size;
if (mbox_move(sync_ctx, offset,
offset + sync_ctx->expunged_space,
trailer_size) < 0)
return -1;
if (ftruncate(sync_ctx->fd, offset + trailer_size) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
return -1;
}
sync_ctx->expunged_space = 0;
istream_raw_mbox_flush(sync_ctx->input);
}
return 0;
}
static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
{
struct stat st;
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
return -1;
}
if ((sync_ctx->base_uid_validity != 0 &&
sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) ||
(sync_ctx->hdr->uid_validity == 0 && sync_ctx->seen_first_mail)) {
if (sync_ctx->base_uid_validity == 0) {
/* we couldn't rewrite X-IMAPbase because it's
a read-only mbox */
i_assert(sync_ctx->ibox->mbox_readonly);
sync_ctx->base_uid_validity = time(NULL);
}
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, uid_validity),
&sync_ctx->base_uid_validity,
sizeof(sync_ctx->base_uid_validity));
}
if (istream_raw_mbox_is_eof(sync_ctx->input) &&
sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
i_assert(sync_ctx->next_uid != 0);
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, next_uid),
&sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
}
if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
uint32_t sync_stamp = st.st_mtime;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_stamp),
&sync_stamp, sizeof(sync_stamp));
}
if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
uint64_t sync_size = st.st_size;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_size),
&sync_size, sizeof(sync_size));
}
return 0;
}
static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->next_uid = sync_ctx->hdr->next_uid;
sync_ctx->prev_msg_uid = 0;
sync_ctx->seq = 0;
sync_ctx->idx_seq = 1;
sync_ctx->dest_first_mail = TRUE;
sync_ctx->seen_first_mail = FALSE;
}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx, int sync_header)
{
struct mbox_sync_mail_context mail_ctx;
struct stat st;
uint32_t min_msg_count;
int ret;
if (sync_header)
min_msg_count = 1;
else {
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "stat()");
return -1;
}
min_msg_count =
(uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
(uint64_t)st.st_size == sync_ctx->hdr->sync_size ?
0 : (uint32_t)-1;
}
mbox_sync_restart(sync_ctx);
if ((ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count)) == -1)
return -1;
if (ret == -2) {
/* initially we had mbox read-locked, but later we needed a
write-lock. doing it required dropping the read lock.
we're here because mbox was modified before we got the
write-lock. so, restart the whole syncing. */
i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
mail_index_transaction_rollback(sync_ctx->t);
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view,
FALSE);
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, (uint32_t)-1) < 0)
return -1;
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
buffer_set_used_size(sync_ctx->syncs, 0);
memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync_has_changed(struct index_mailbox *ibox)
{
const struct mail_index_header *hdr;
struct stat st;
if (mail_index_get_header(ibox->view, &hdr) < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
if (stat(ibox->path, &st) < 0) {
mbox_set_syscall_error(ibox, "stat()");
return -1;
}
return (uint32_t)st.st_mtime != hdr->sync_stamp ||
(uint64_t)st.st_size != hdr->sync_size;
}
static int mbox_sync_update_imap_base(struct mbox_sync_context *sync_ctx)
{
struct mbox_sync_mail_context mail_ctx;
if (mbox_sync_seek(sync_ctx, 0) < 0)
return -1;
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
sync_ctx->update_base_uid_last = sync_ctx->next_uid-1;
if (mbox_sync_check_excl_lock(sync_ctx) == -1)
return -1;
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, 1) < 0)
return -1;
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync(struct index_mailbox *ibox, int last_commit,
int sync_header, int lock)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
uint32_t seq;
uoff_t offset;
unsigned int lock_id = 0;
int ret, lock_type;
if (lock) {
if (mbox_lock(ibox, F_RDLCK, &lock_id) <= 0)
return -1;
}
if (sync_header)
ret = 0;
else {
if ((ret = mbox_sync_has_changed(ibox)) < 0) {
if (lock)
(void)mbox_unlock(ibox, lock_id);
return -1;
}
}
if (ret == 0 && !last_commit)
return 0;
if (last_commit) {
seq = ibox->commit_log_file_seq;
offset = ibox->commit_log_file_offset;
} else {
seq = (uint32_t)-1;
offset = (uoff_t)-1;
}
ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
seq, offset);
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(ibox);
return ret;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.ibox = ibox;
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.lock_id = lock_id;
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
i_assert(ret == 0);
lock_type = mail_index_sync_have_more(index_sync_ctx) &&
!ibox->mbox_readonly ? F_WRLCK : F_RDLCK;
if (lock_type == F_WRLCK && lock) {
(void)mbox_unlock(ibox, lock_id);
lock_id = 0;
}
if (mbox_sync_lock(&sync_ctx, lock_type) < 0)
ret = -1;
else if (mbox_sync_do(&sync_ctx, sync_header) < 0)
ret = -1;
if (ret < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
} else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
sync_ctx.t = NULL;
if (mail_index_sync_end(index_sync_ctx) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
if (sync_ctx.seen_first_mail &&
sync_ctx.base_uid_last != sync_ctx.next_uid-1 &&
ret == 0 && !ibox->mbox_readonly) {
/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
so previous transactions have been committed. */
/* FIXME: ugly .. */
ret = mail_index_sync_begin(ibox->index,
&sync_ctx.index_sync_ctx,
&sync_ctx.sync_view,
(uint32_t)-1, (uoff_t)-1);
if (ret < 0)
mail_storage_set_index_error(ibox);
else {
(void)mail_index_get_header(sync_ctx.sync_view,
&sync_ctx.hdr);
if ((ret = mbox_sync_update_imap_base(&sync_ctx)) < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t,
&seq,
&offset) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
if (mail_index_sync_end(sync_ctx.index_sync_ctx) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
}
}
if (ret == 0 && ibox->mbox_lock_type == F_WRLCK) {
if (fsync(ibox->mbox_fd) < 0) {
mbox_set_syscall_error(ibox, "fsync()");
ret = -1;
}
}
if (sync_ctx.lock_id != 0 && ibox->mbox_lock_type != F_RDLCK) {
/* drop to read lock */
unsigned int lock_id = 0;
if (mbox_lock(ibox, F_RDLCK, &lock_id) <= 0)
ret = -1;
else {
if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
ret = -1;
sync_ctx.lock_id = lock_id;
}
}
if (sync_ctx.lock_id != 0 && !lock) {
/* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
notice changes made by others */
if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
ret = -1;
}
str_free(sync_ctx.header);
str_free(sync_ctx.from_line);
buffer_free(sync_ctx.mails);
buffer_free(sync_ctx.syncs);
return ret;
}
int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
if (mbox_sync(ibox, FALSE, FALSE, FALSE) < 0)
return -1;
}
return index_storage_sync(box, flags);
}