mbox-sync.c revision 60d3fa9883237e896a8704275b6116fa46f7ffda
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/*
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen required disk I/O. We may need to:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing X-UID and X-IMAPbase headers
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Write missing or broken Content-Length header if there's space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Expunge specified messages
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Here's how we do it:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Start reading the mails mail headers from the beginning
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - X-Keywords and X-UID headers may contain extra spaces at the end of them,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen remember how much extra each message has and offset to beginning of the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen spaces
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If message flags are dirty and there's enough space to write them, do it
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we didn't have enough space, remember how much was missing and keep
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the total amount of them
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - When we encounter expunged message, check if the amount of empty space in
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen previous messages plus size of expunged message is enough to cover the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen missing space. If yes,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - execute the rewrite plan
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - forget all the messages before the expunged message. only remember
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen how much data we still have to move to cover the expunged message
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we encounter end of file, grow the file and execute the rewrite plan
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Rewrite plan goes:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Start from the first message that needs more space
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If there's expunged messages before us, we have to write over them.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Move all messages after it backwards to fill it
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - Each moved message's X-Keywords header should have n bytes extra
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen space, unless there's not enough space to do it.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If there's no expunged messages, we can move data either forward or
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen backward to get it. Calculate which requires less moving. Forward
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen counting may encounter more messages which require extra space, count
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen that too.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we decide to move forwards and we had to go through dirty
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen messages, do the moving from last to first dirty message
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - If we encounter end of file, grow the file enough to get the required
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen amount of space plus enough space to fill X-Keywords headers full of
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen spaces.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen*/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#include "ioloop.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "buffer.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "istream.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "file-set-size.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "str.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "write-full.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "istream-raw-mbox.h"
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#include "mbox-storage.h"
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#include "mbox-file.h"
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen#include "mbox-lock.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mbox-sync-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen#include <stddef.h>
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#include <sys/stat.h>
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#define MBOX_SYNC_SECS 1
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* returns -1 = error, 0 = mbox changed since previous lock, 1 = didn't */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mbox_sync_lock(struct mbox_sync_context *sync_ctx, int lock_type)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct index_mailbox *ibox = sync_ctx->ibox;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct stat old_st, st;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uoff_t old_from_offset = 0, old_offset = 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen i_assert(lock_type != F_WRLCK || !ibox->mbox_readonly);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (sync_ctx->lock_id == 0 || sync_ctx->input == NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(&old_st, 0, sizeof(old_st));
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (sync_ctx->lock_id != 0) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen (void)mbox_unlock(ibox, sync_ctx->lock_id);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->lock_id = 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen } else {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (fstat(sync_ctx->fd, &old_st) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mbox_set_syscall_error(ibox, "stat()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen old_from_offset =
659fe5d24825b160cae512538088020d97a60239Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen old_offset = sync_ctx->input->v_offset;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen (void)mbox_unlock(ibox, sync_ctx->lock_id);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync_ctx->lock_id = 0;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (mbox_lock(ibox, lock_type, &sync_ctx->lock_id) <= 0)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return -1;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen if (mbox_file_open_stream(ibox) < 0)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return -1;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync_ctx->file_input = sync_ctx->ibox->mbox_file_stream;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync_ctx->input = sync_ctx->ibox->mbox_stream;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync_ctx->fd = sync_ctx->ibox->mbox_fd;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (old_st.st_mtime == 0) {
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen /* we didn't have the file open before -> it changed */
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (fstat(sync_ctx->fd, &st) < 0) {
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen mbox_set_syscall_error(ibox, "fstat()");
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return -1;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (st.st_mtime != old_st.st_mtime || st.st_size != old_st.st_size ||
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen st.st_ino != old_st.st_ino ||
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen !CMP_DEV_T(st.st_dev, old_st.st_dev) ||
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen time(NULL) - st.st_mtime <= MBOX_SYNC_SECS)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return 0;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen /* same as before. we'll have to fix mbox stream to contain
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen correct from_offset, hdr_offset and body_offset. so, seek
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen to from_offset and read through the header. */
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, old_from_offset) < 0) {
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen mail_storage_set_critical(ibox->box.storage,
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen "Message offset %s changed unexpectedly for mbox file "
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen "%s", dec2str(old_from_offset), sync_ctx->ibox->path);
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen return 0;
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen }
3cfff0ca01961d885bdbd6ef08d761880116af07Timo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen i_stream_seek(sync_ctx->input, old_offset);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return 1;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen}
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen{
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen " from mbox file %s", from_offset,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->ibox->path);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return -1;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return 0;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen}
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainenstatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen struct mbox_sync_mail_context *mail_ctx,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen uoff_t grow_size)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen{
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen uoff_t src_offset, file_size;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen i_assert(grow_size > 0);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* put the extra space between last message's header and body */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (file_set_size(sync_ctx->fd, file_size) < 0) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return -1;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen src_offset = mail_ctx->body_offset;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_ctx->body_offset += grow_size;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen file_size - mail_ctx->body_offset) < 0)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return -1;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen istream_raw_mbox_flush(sync_ctx->input);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return 0;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen}
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen{
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen struct mail_index_sync_rec *sync;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen size_t size, src, dest;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen sync = buffer_get_modifyable_data(syncs_buf, &size);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen size /= sizeof(*sync);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen for (src = dest = 0; src < size; src++) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (sync[src].uid2 >= uid) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (src != dest)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen sync[dest] = sync[src];
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen dest++;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen }
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen }
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen}
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic int
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen struct mbox_sync_mail_context *mail_ctx)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen{
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen /* get EOF */
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input))
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return 0;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_ctx->sync_ctx = sync_ctx;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_ctx->seq = ++sync_ctx->seq;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_ctx->header = sync_ctx->header;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_ctx->uidl = sync_ctx->uidl;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen str_truncate(mail_ctx->uidl, 0);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_ctx->from_offset =
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mail_ctx->mail.offset =
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_ctx->seq == 1)
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen sync_ctx->seen_first_mail = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_ctx->seq > 1 && sync_ctx->dest_first_mail) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* First message was expunged and this is the next one.
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen Skip \n header */
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen mail_ctx->from_offset++;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen }
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_ctx->mail.body_size =
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen mail_ctx->content_length);
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen /* save the offset permanently with recent flag state */
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen mail_ctx->mail.from_offset = mail_ctx->from_offset;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen if (!sync_ctx->ibox->keep_recent) {
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen /* need to add 'O' flag to Status-header */
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen mail_ctx->need_rewrite = TRUE;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen }
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen // FIXME: save it somewhere
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen }
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainenstatic int mbox_sync_buf_have_expunges(buffer_t *syncs_buf)
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen{
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen const struct mail_index_sync_rec *sync;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen size_t size, i;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
3cfff0ca01961d885bdbd6ef08d761880116af07Timo Sirainen sync = buffer_get_data(syncs_buf, &size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen size /= sizeof(*sync);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen for (i = 0; i < size; i++) {
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen if (sync[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen return TRUE;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen }
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen return FALSE;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen}
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
3cfff0ca01961d885bdbd6ef08d761880116af07Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen uint32_t uid, int *sync_expunge_r)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen{
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen int ret;
3cfff0ca01961d885bdbd6ef08d761880116af07Timo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen *sync_expunge_r = FALSE;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen
659fe5d24825b160cae512538088020d97a60239Timo Sirainen if (sync_ctx->ibox->mbox_readonly || sync_ctx->index_sync_ctx == NULL)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return 0;
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen
659fe5d24825b160cae512538088020d97a60239Timo Sirainen if (uid == 0) {
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen /* nothing for this or the future ones */
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen uid = (uint32_t)-1;
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen }
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen while (uid >= sync_rec->uid1) {
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen if (uid <= sync_rec->uid2 &&
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND) {
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen buffer_append(sync_ctx->syncs, sync_rec,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sizeof(*sync_rec));
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen *sync_expunge_r = TRUE;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen if (ret < 0) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return -1;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (ret == 0) {
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen memset(sync_rec, 0, sizeof(*sync_rec));
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen break;
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) {
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen if (sync_rec->uid2 >= sync_ctx->next_uid) {
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen sync_ctx->next_uid = sync_rec->uid2 + 1;
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen sync_ctx->update_base_uid_last = sync_rec->uid2;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen memset(sync_rec, 0, sizeof(*sync_rec));
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (!*sync_expunge_r)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen *sync_expunge_r = mbox_sync_buf_have_expunges(sync_ctx->syncs);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return 0;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen}
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen keywords_mask_t keywords)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen{
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen const struct mail_index_sync_rec *sync;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen size_t size, i;
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen sync = buffer_get_data(syncs_buf, &size);
96c253a039f102fa78a313ee05200ab3970112dcTimo Sirainen size /= sizeof(*sync);
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen for (i = 0; i < size; i++) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (sync[i].type != MAIL_INDEX_SYNC_TYPE_FLAGS)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen continue;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_index_sync_flags_apply(&sync[i], flags, keywords);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen }
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen}
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic int
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen{
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen const struct mail_index_record *rec = NULL;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen uint32_t messages_count;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen int ret = 0;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen while (sync_ctx->idx_seq <= messages_count) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen ret = mail_index_lookup(sync_ctx->sync_view,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen sync_ctx->idx_seq, &rec);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (ret < 0) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return -1;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen }
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (uid <= rec->uid)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen /* externally expunged message, remove from index */
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync_ctx->idx_seq++;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen rec = NULL;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen }
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen if (ret == 0 && uid < sync_ctx->hdr->next_uid) {
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen /* this UID was already in index and it was expunged */
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen "mbox sync: Expunged message reappeared in mailbox %s "
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen "(UID %u < %u)", sync_ctx->ibox->path, uid,
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen sync_ctx->hdr->next_uid);
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen ret = 0;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen } else if (rec != NULL && rec->uid != uid) {
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen "mbox sync: UID inserted in the middle of mailbox %s "
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen "(%u > %u)", sync_ctx->ibox->path, rec->uid, uid);
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen ret = 0;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen } else {
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen ret = 1;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen }
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen *rec_r = rec;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return ret;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen}
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainenstatic int mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen unsigned char hdr_md5_sum[],
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen const struct mail_index_record **rec_r)
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen{
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen const struct mail_index_record *rec = NULL;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen uint32_t messages_count;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen const void *data;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen int ret;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen while (sync_ctx->idx_seq <= messages_count) {
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen ret = mail_index_lookup(sync_ctx->sync_view,
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen sync_ctx->idx_seq, &rec);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (ret < 0) {
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen return -1;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen }
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (mail_index_lookup_extra(sync_ctx->sync_view,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen sync_ctx->idx_seq,
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen sync_ctx->ibox->md5hdr_extra_idx,
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen &data) < 0) {
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen return -1;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen }
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (memcmp(data, hdr_md5_sum, 16) == 0)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen break;
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen /* externally expunged message, remove from index */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->idx_seq++;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen rec = NULL;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen *rec_r = rec;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen}
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenstatic int mbox_sync_get_from_offset(struct mbox_sync_context *sync_ctx,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen uint32_t seq, uint64_t *offset_r)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen{
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen const void *data;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* see if from_offset needs updating */
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen if (mail_index_lookup_extra(sync_ctx->sync_view, seq,
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen sync_ctx->ibox->mbox_extra_idx,
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen &data) < 0) {
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return -1;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen *offset_r = *((const uint64_t *)data);
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen return 0;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen}
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainenstatic int
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen struct mbox_sync_mail *mail,
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen int nocheck)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen{
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen uint64_t offset;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen if (!nocheck) {
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (mbox_sync_get_from_offset(sync_ctx, sync_ctx->idx_seq,
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen &offset) < 0)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return -1;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (offset == mail->from_offset)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return 0;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen }
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen offset = mail->from_offset;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen sync_ctx->ibox->mbox_extra_idx, &offset);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return 0;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen}
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen struct mbox_sync_mail_context *mail_ctx,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen const struct mail_index_record *rec)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen{
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen keywords_mask_t idx_keywords;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen uint8_t idx_flags, mbox_flags;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen if (rec == NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* new message */
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen MODIFY_REPLACE, mbox_flags,
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen mail->keywords);
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen if (sync_ctx->ibox->md5hdr_extra_idx != 0) {
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen mail_index_update_extra_rec(sync_ctx->t,
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen sync_ctx->idx_seq,
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen sync_ctx->ibox->md5hdr_extra_idx,
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen mail_ctx->hdr_md5_sum);
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen }
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen if (str_len(mail_ctx->uidl) > 0) {
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen /*FIXME:mail_cache_add(sync_ctx->cache_trans,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MAIL_CACHE_UID_STRING,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_data(mail_ctx->uidl),
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_len(mail_ctx->uidl));*/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* see if flags changed */
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen idx_flags = rec->flags;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen mbox_sync_apply_index_syncs(sync_ctx->syncs,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen &idx_flags, idx_keywords);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT));
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (idx_flags != mbox_flags ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memcmp(idx_keywords, mail->keywords,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen INDEX_KEYWORDS_BYTE_COUNT) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen MODIFY_REPLACE, mbox_flags,
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen mail->keywords);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen }
3cfff0ca01961d885bdbd6ef08d761880116af07Timo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen rewriting would just move it anyway. */
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (sync_ctx->need_space_seq == 0) {
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
3cfff0ca01961d885bdbd6ef08d761880116af07Timo Sirainen if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen return -1;
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen }
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen return 0;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen}
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct istream *input = ctx->sync_ctx->file_input;
3cfff0ca01961d885bdbd6ef08d761880116af07Timo Sirainen const unsigned char *data;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen size_t size, from_line_size;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen from_line_size = ctx->hdr_offset - ctx->from_offset;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen i_stream_seek(input, ctx->from_offset);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen for (;;) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen data = i_stream_get_data(input, &size);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (size >= from_line_size)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen size = from_line_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen i_stream_skip(input, size);
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen from_line_size -= size;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (from_line_size == 0)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen break;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if (i_stream_read(input) < 0)
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return -1;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen }
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen return 0;
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen}
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainenstatic int
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen{
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen string_t *str = ctx->sync_ctx->from_line;
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen ctx->from_offset + move_diff) < 0) {
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return -1;
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen }
fc1696e32dd732a5bbabc3c8f64810448e327043Timo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen istream_raw_mbox_flush(ctx->sync_ctx->input);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen}
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainenstatic void update_from_offsets(struct mbox_sync_context *sync_ctx)
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen{
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen const struct mbox_sync_mail *mails;
ab6315aa0d5c83f4f1dc98b3715826a686aebffdTimo Sirainen uint32_t idx, idx_seq, extra_idx = sync_ctx->ibox->mbox_extra_idx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint64_t offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen size_t size;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mails = buffer_get_modifyable_data(sync_ctx->mails, &size);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen size /= sizeof(*mails);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen i_assert(sync_ctx->seq - sync_ctx->need_space_seq + 1 == size);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen idx = 0;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen idx_seq = sync_ctx->need_space_idx_seq;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen if (idx_seq == 0) {
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen idx++; idx_seq++;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen for (; idx < size; idx++, idx_seq++, mails++) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if (mails->uid == 0)
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen continue;
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen offset = mails->from_offset;
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen mail_index_update_extra_rec(sync_ctx->t, idx_seq, extra_idx,
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen &offset);
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen }
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen}
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainenstatic int mbox_sync_check_excl_lock(struct mbox_sync_context *sync_ctx)
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen{
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen int ret;
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen
fcfd317f7eb1f0216764c75c5fab3555020552d4Timo Sirainen if (sync_ctx->ibox->mbox_lock_type == F_RDLCK) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if ((ret = mbox_sync_lock(sync_ctx, F_WRLCK)) < 0)
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return -1;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if (ret == 0)
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return -2;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen }
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return 0;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen}
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainenstatic int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen{
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen int ret;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if ((ret = mbox_sync_check_excl_lock(mail_ctx->sync_ctx)) < 0)
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return ret;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen mail_ctx->mail.offset = mail_ctx->from_offset;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen mail_ctx->mail.space =
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen mail_ctx->body_offset - mail_ctx->from_offset +
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen mail_ctx->mail.body_size;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen mail_ctx->mail.body_size = 0;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if (mail_ctx->sync_ctx->dest_first_mail) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen /* expunging first message, fix space to contain next
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen message's \n header too since it will be removed. */
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen mail_ctx->mail.space++;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen{
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen off_t move_diff;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen int ret;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen if (sync_ctx->ibox->mbox_readonly)
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen return 0;
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen /* move the header backwards to fill expunged space */
e68309fcfa2eaa88217fd51e7b4900fc9c20ef5dTimo Sirainen if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return ret;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen move_diff = -sync_ctx->expunged_space;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen /* read the From-line before rewriting overwrites it */
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen if (mbox_read_from_line(mail_ctx) < 0)
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen return -1;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen ret = mbox_sync_try_rewrite(mail_ctx, move_diff, FALSE);
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen if (ret < 0)
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen return -1;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen if (ret > 0) {
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen /* rewrite successful, write From-line to
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen new location */
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen mail_ctx->mail.from_offset += move_diff;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen mail_ctx->mail.offset += move_diff;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen if (mbox_write_from_line(mail_ctx, move_diff) < 0)
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen return -1;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen }
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen } else if (mail_ctx->need_rewrite ||
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen buffer_get_used_size(sync_ctx->syncs) != 0 ||
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen (mail_ctx->seq == 1 &&
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->update_base_uid_last != 0)) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen return ret;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, 0, FALSE)) < 0)
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen return -1;
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen } else {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* nothing to do */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return 0;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen }
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* first mail with no space to write it */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->need_space_seq = sync_ctx->seq;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->need_space_idx_seq = sync_ctx->idx_seq;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->space_diff = 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (sync_ctx->expunged_space > 0) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* create dummy message to describe the expunged data */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen struct mbox_sync_mail mail;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen memset(&mail, 0, sizeof(mail));
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen mail.flags = MBOX_EXPUNGED;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen mail.offset = mail_ctx->from_offset -
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen sync_ctx->expunged_space;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen mail.space = sync_ctx->expunged_space;
0b17b95357d5d73d941f1eb4ca9fc543bc510e42Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->need_space_seq--;
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen buffer_append(sync_ctx->mails, &mail, sizeof(mail));
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen }
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen}
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainenstatic int
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainenmbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen{
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen uoff_t extra_space;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->space_diff += mail_ctx->mail.space;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (sync_ctx->space_diff < 0)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* we have enough space now */
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen extra_space = MBOX_HEADER_EXTRA_SPACE *
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (mail_ctx->mail.uid == 0 &&
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen (uoff_t)sync_ctx->space_diff > extra_space) {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen /* don't waste too much on extra spacing */
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->expunged_space = sync_ctx->space_diff - extra_space;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->space_diff = extra_space;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen } else {
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen sync_ctx->expunged_space = 0;
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen }
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen if (mbox_sync_rewrite(sync_ctx, sync_ctx->space_diff,
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen sync_ctx->need_space_seq, sync_ctx->seq) < 0)
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen return -1;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen update_from_offsets(sync_ctx);
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen /* mail_ctx may contain wrong data after rewrite, so make sure we
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen don't try to access it */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen sync_ctx->need_space_seq = 0;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen buffer_set_used_size(sync_ctx->mails, 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 0;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen}
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainenmbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t seq1, seq2;
uint64_t offset;
if (mail_index_lookup_uid_range(sync_ctx->sync_view, uid, (uint32_t)-1,
&seq1, &seq2) < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
if (seq1 == 0)
return 0;
if (mbox_sync_get_from_offset(sync_ctx, seq1, &offset) < 0)
return -1;
/* set to -1, since it's always increased later */
sync_ctx->seq = seq1-1;
if (sync_ctx->seq == 0 && offset != 0) {
/* this mbox has pseudo mail which contains the X-IMAP header */
sync_ctx->seq++;
}
sync_ctx->idx_seq = seq1;
sync_ctx->dest_first_mail = sync_ctx->seq == 0;
if (istream_raw_mbox_seek(sync_ctx->input, offset) < 0) {
mail_storage_set_critical(sync_ctx->ibox->box.storage,
"Cached message offset %s is invalid for mbox file %s",
dec2str(offset), sync_ctx->ibox->path);
mail_index_mark_corrupted(sync_ctx->ibox->index);
return -1;
}
(void)istream_raw_mbox_get_body_offset(sync_ctx->input);
return 1;
}
static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
uint32_t min_message_count)
{
const struct mail_index_record *rec;
uint32_t uid, messages_count;
uoff_t offset;
int ret, expunged;
if (min_message_count != 0)
ret = 0;
else {
/* we sync only what we need to. jump to first record that
needs updating */
const struct mail_index_sync_rec *sync_rec;
size_t size;
if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
sync_ctx->sync_rec.uid1 == 0) {
if (mbox_sync_read_index_syncs(sync_ctx, 1,
&expunged) < 0)
return -1;
if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
sync_ctx->sync_rec.uid1 == 0) {
/* nothing to do */
return 0;
}
}
sync_rec = buffer_get_data(sync_ctx->syncs, &size);
if (size == 0)
sync_rec = &sync_ctx->sync_rec;
ret = mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1);
if (ret < 0)
return -1;
}
if (ret == 0) {
if (istream_raw_mbox_seek(sync_ctx->input, 0) < 0) {
/* doesn't begin with a From-line */
mail_storage_set_error(sync_ctx->ibox->box.storage,
"Mailbox isn't a valid mbox file");
return -1;
}
sync_ctx->dest_first_mail = TRUE;
}
messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
uid = mail_ctx->mail.uid;
if (mail_ctx->pseudo)
uid = 0;
/* get all sync records related to this message */
if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
return -1;
rec = NULL;
if (uid != 0 && sync_ctx->ibox->md5hdr_extra_idx == 0) {
ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
if (ret < 0)
return -1;
if (ret == 0)
uid = 0;
}
if (uid == 0 && sync_ctx->ibox->mbox_readonly &&
!mail_ctx->pseudo) {
/* Use MD5 sums */
if (sync_ctx->ibox->md5hdr_extra_idx == 0) {
sync_ctx->ibox->md5hdr_extra_idx =
mail_index_register_record_extra(
sync_ctx->ibox->index,
"header-md5", 16);
}
if (mbox_sync_find_index_md5(sync_ctx,
mail_ctx->hdr_md5_sum,
&rec) < 0)
return -1;
if (rec != NULL)
uid = rec->uid;
}
if (uid == 0 && !mail_ctx->pseudo) {
/* missing/broken X-UID. all the rest of the mails
need new UIDs. */
while (sync_ctx->idx_seq <= messages_count) {
mail_index_expunge(sync_ctx->t,
sync_ctx->idx_seq++);
}
mail_ctx->need_rewrite = TRUE;
mail_ctx->mail.uid = sync_ctx->next_uid++;
sync_ctx->prev_msg_uid = mail_ctx->mail.uid;
}
if (!expunged) {
ret = mbox_sync_handle_header(mail_ctx);
sync_ctx->dest_first_mail = FALSE;
} else {
mail_ctx->mail.uid = 0;
ret = mbox_sync_handle_expunge(mail_ctx);
}
if (ret < 0) {
/* -1 = error, -2 = need to restart */
return ret;
}
if (!mail_ctx->pseudo) {
if (!expunged) {
if (mbox_sync_update_index(sync_ctx, mail_ctx,
rec) < 0)
return -1;
}
sync_ctx->idx_seq++;
}
istream_raw_mbox_next(sync_ctx->input,
mail_ctx->mail.body_size);
offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
if (mbox_move(sync_ctx,
mail_ctx->body_offset -
sync_ctx->expunged_space,
mail_ctx->body_offset,
mail_ctx->mail.body_size) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
}
} else if (sync_ctx->seq >= min_message_count) {
mbox_sync_buffer_delete_old(sync_ctx->syncs, uid+1);
if (buffer_get_used_size(sync_ctx->syncs) == 0) {
/* if there's no sync records left,
we can stop */
if (sync_ctx->sync_rec.uid1 == 0)
break;
/* we can skip forward to next record which
needs updating. */
uid = sync_ctx->sync_rec.uid1;
if (mbox_sync_seek_to_uid(sync_ctx, uid) < 0)
return -1;
}
}
}
if (istream_raw_mbox_is_eof(sync_ctx->input)) {
/* rest of the messages in index don't exist -> expunge them */
while (sync_ctx->idx_seq <= messages_count)
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
}
return 0;
}
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
uoff_t offset, extra_space, trailer_size;
int need_rewrite;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
i_assert(sync_ctx->expunged_space == 0);
return 0;
}
trailer_size = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->file_input->v_offset;
if (sync_ctx->need_space_seq != 0) {
i_assert(sync_ctx->space_diff < 0);
extra_space = MBOX_HEADER_EXTRA_SPACE *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
sync_ctx->space_diff -= extra_space;
sync_ctx->space_diff += sync_ctx->expunged_space;
if (sync_ctx->expunged_space <= -sync_ctx->space_diff)
sync_ctx->expunged_space = 0;
else
sync_ctx->expunged_space -= -sync_ctx->space_diff;
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
if (sync_ctx->space_diff < 0 &&
mbox_sync_grow_file(sync_ctx, mail_ctx,
-sync_ctx->space_diff) < 0)
return -1;
need_rewrite = sync_ctx->seq != sync_ctx->need_space_seq;
if (mbox_sync_try_rewrite(mail_ctx, 0, need_rewrite) < 0)
return -1;
if (need_rewrite) {
buffer_set_used_size(sync_ctx->mails,
(sync_ctx->seq -
sync_ctx->need_space_seq) *
sizeof(mail_ctx->mail));
buffer_append(sync_ctx->mails, &mail_ctx->mail,
sizeof(mail_ctx->mail));
if (mbox_sync_rewrite(sync_ctx, extra_space,
sync_ctx->need_space_seq,
sync_ctx->seq) < 0)
return -1;
}
update_from_offsets(sync_ctx);
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
}
if (sync_ctx->expunged_space > 0) {
/* copy trailer, then truncate the file */
offset = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->expunged_space - trailer_size;
if (mbox_move(sync_ctx, offset,
offset + sync_ctx->expunged_space,
trailer_size) < 0)
return -1;
if (ftruncate(sync_ctx->fd, offset + trailer_size) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
return -1;
}
sync_ctx->expunged_space = 0;
istream_raw_mbox_flush(sync_ctx->input);
}
return 0;
}
static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
{
struct stat st;
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
return -1;
}
if ((sync_ctx->base_uid_validity != 0 &&
sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) ||
(sync_ctx->hdr->uid_validity == 0 && sync_ctx->seen_first_mail)) {
if (sync_ctx->base_uid_validity == 0) {
/* we couldn't rewrite X-IMAPbase because it's
a read-only mbox */
i_assert(sync_ctx->ibox->mbox_readonly);
sync_ctx->base_uid_validity = time(NULL);
}
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, uid_validity),
&sync_ctx->base_uid_validity,
sizeof(sync_ctx->base_uid_validity));
}
if (istream_raw_mbox_is_eof(sync_ctx->input) &&
sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
i_assert(sync_ctx->next_uid != 0);
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, next_uid),
&sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
}
if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
uint32_t sync_stamp = st.st_mtime;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_stamp),
&sync_stamp, sizeof(sync_stamp));
}
if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
uint64_t sync_size = st.st_size;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_size),
&sync_size, sizeof(sync_size));
}
return 0;
}
static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->next_uid = sync_ctx->hdr->next_uid;
sync_ctx->prev_msg_uid = 0;
sync_ctx->seq = 0;
sync_ctx->idx_seq = 1;
sync_ctx->dest_first_mail = TRUE;
sync_ctx->seen_first_mail = FALSE;
}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx, int sync_header)
{
struct mbox_sync_mail_context mail_ctx;
struct stat st;
uint32_t min_msg_count;
int ret;
if (sync_header)
min_msg_count = 1;
else {
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "stat()");
return -1;
}
min_msg_count =
(uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
(uint64_t)st.st_size == sync_ctx->hdr->sync_size ?
0 : (uint32_t)-1;
}
mbox_sync_restart(sync_ctx);
if ((ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count)) == -1)
return -1;
if (ret == -2) {
/* initially we had mbox read-locked, but later we needed a
write-lock. doing it required dropping the read lock.
we're here because mbox was modified before we got the
write-lock. so, restart the whole syncing. */
i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
mail_index_transaction_rollback(sync_ctx->t);
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view,
FALSE);
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, (uint32_t)-1) < 0)
return -1;
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
buffer_set_used_size(sync_ctx->syncs, 0);
memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync_has_changed(struct index_mailbox *ibox)
{
const struct mail_index_header *hdr;
struct stat st;
if (mail_index_get_header(ibox->view, &hdr) < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
if (stat(ibox->path, &st) < 0) {
mbox_set_syscall_error(ibox, "stat()");
return -1;
}
return (uint32_t)st.st_mtime != hdr->sync_stamp ||
(uint64_t)st.st_size != hdr->sync_size;
}
static int mbox_sync_update_imap_base(struct mbox_sync_context *sync_ctx)
{
struct mbox_sync_mail_context mail_ctx;
if (mbox_sync_seek(sync_ctx, 0) < 0)
return -1;
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
sync_ctx->update_base_uid_last = sync_ctx->next_uid-1;
if (mbox_sync_check_excl_lock(sync_ctx) == -1)
return -1;
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, 1) < 0)
return -1;
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync(struct index_mailbox *ibox, int last_commit,
int sync_header, int lock)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
uint32_t seq;
uoff_t offset;
unsigned int lock_id = 0;
int ret, lock_type;
if (lock) {
if (mbox_lock(ibox, F_RDLCK, &lock_id) <= 0)
return -1;
}
if (sync_header)
ret = 0;
else {
if ((ret = mbox_sync_has_changed(ibox)) < 0) {
if (lock)
(void)mbox_unlock(ibox, lock_id);
return -1;
}
}
if (ret == 0 && !last_commit)
return 0;
if (last_commit) {
seq = ibox->commit_log_file_seq;
offset = ibox->commit_log_file_offset;
} else {
seq = (uint32_t)-1;
offset = (uoff_t)-1;
}
ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
seq, offset);
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(ibox);
return ret;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.ibox = ibox;
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.uidl = str_new(default_pool, 128);
sync_ctx.lock_id = lock_id;
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
i_assert(ret == 0);
lock_type = mail_index_sync_have_more(index_sync_ctx) &&
!ibox->mbox_readonly ? F_WRLCK : F_RDLCK;
if (lock_type == F_WRLCK && lock) {
(void)mbox_unlock(ibox, lock_id);
lock_id = 0;
}
if (mbox_sync_lock(&sync_ctx, lock_type) < 0)
ret = -1;
else if (mbox_sync_do(&sync_ctx, sync_header) < 0)
ret = -1;
if (ret < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
} else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
sync_ctx.t = NULL;
if (mail_index_sync_end(index_sync_ctx) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
if (sync_ctx.seen_first_mail &&
sync_ctx.base_uid_last != sync_ctx.next_uid-1 &&
ret == 0 && !ibox->mbox_readonly) {
/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
so previous transactions have been committed. */
/* FIXME: ugly .. */
ret = mail_index_sync_begin(ibox->index,
&sync_ctx.index_sync_ctx,
&sync_ctx.sync_view,
(uint32_t)-1, (uoff_t)-1);
if (ret < 0)
mail_storage_set_index_error(ibox);
else {
(void)mail_index_get_header(sync_ctx.sync_view,
&sync_ctx.hdr);
if ((ret = mbox_sync_update_imap_base(&sync_ctx)) < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t,
&seq,
&offset) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
if (mail_index_sync_end(sync_ctx.index_sync_ctx) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
}
}
}
if (ret == 0 && ibox->mbox_lock_type == F_WRLCK) {
if (fsync(ibox->mbox_fd) < 0) {
mbox_set_syscall_error(ibox, "fsync()");
ret = -1;
}
}
if (sync_ctx.lock_id != 0 && ibox->mbox_lock_type != F_RDLCK) {
/* drop to read lock */
unsigned int lock_id = 0;
if (mbox_lock(ibox, F_RDLCK, &lock_id) <= 0)
ret = -1;
else {
if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
ret = -1;
sync_ctx.lock_id = lock_id;
}
}
if (sync_ctx.lock_id != 0 && !lock) {
/* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
notice changes made by others */
if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
ret = -1;
}
str_free(sync_ctx.uidl);
str_free(sync_ctx.header);
str_free(sync_ctx.from_line);
buffer_free(sync_ctx.mails);
buffer_free(sync_ctx.syncs);
return ret;
}
int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
if (mbox_sync(ibox, FALSE, FALSE, FALSE) < 0)
return -1;
}
return index_storage_sync(box, flags);
}