mbox-sync.c revision c000c8eca8f24b2a0c76393ec4bbf76a505a4983
5a580c3a38ced62d4bcc95b8ac7c4f2935b5d294Timo Sirainen/* Copyright (c) 2004-2010 Dovecot authors, see the included COPYING file */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen/*
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen required disk I/O. We may need to:
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - Write missing X-UID and X-IMAPbase headers
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - Write missing or broken Content-Length header if there's space
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - Expunge specified messages
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen Here's how we do it:
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - Start reading the mails from the beginning
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen of them, remember how much each message has and offset to beginning of the
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen padding
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - If header needs to be rewritten and there's enough space, do it
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - If we didn't have enough space, remember how much was missing
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - Continue reading and counting the padding in each message. If available
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen padding is enough to rewrite all the previous messages needing it, do it
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - When we encounter expunged message, treat all of it as padding and
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen rewrite previous messages if needed (and there's enough space).
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen Afterwards keep moving messages backwards to fill the expunged space.
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen Moving is done by rewriting each message's headers, with possibly adding
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen missing Content-Length header and padding. Message bodies are moved
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen without modifications.
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - If we encounter end of file, grow the file and rewrite needed messages
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen - Rewriting is done by moving message body forward, rewriting message's
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen header and doing the same for previous message, until all of them are
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen rewritten.
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen*/
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "lib.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "ioloop.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "array.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "buffer.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "hostpid.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "istream.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "file-set-size.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "str.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "read-full.h"
063254ba79239f1c0a78ea08d4aa22f8f11d984cTimo Sirainen#include "write-full.h"
063254ba79239f1c0a78ea08d4aa22f8f11d984cTimo Sirainen#include "message-date.h"
063254ba79239f1c0a78ea08d4aa22f8f11d984cTimo Sirainen#include "istream-raw-mbox.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "mbox-storage.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "index-sync-changes.h"
e809db9220c804b16d4d74782433a1075da12274Timo Sirainen#include "mbox-from.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "mbox-file.h"
6ac48a4295edbeb6b962700be3c563ad015c0a4eTimo Sirainen#include "mbox-lock.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include "mbox-sync-private.h"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include <stddef.h>
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include <stdlib.h>
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include <utime.h>
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen#include <sys/stat.h>
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen/* The text below was taken exactly as c-client wrote it to my mailbox,
81d3c215bb1fdbda2cf7ccd9519f6b4fd03c3791Timo Sirainen so it's probably copyrighted by University of Washington. */
8ddc45fe2080010715c212ecbb2b12b6734f6d4bTimo Sirainen#define PSEUDO_MESSAGE_BODY \
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen"This text is part of the internal format of your mail folder, and is not\n" \
8ddc45fe2080010715c212ecbb2b12b6734f6d4bTimo Sirainen"a real message. It is created automatically by the mail system software.\n" \
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen"If deleted, important folder data will be lost, and it will be re-created\n" \
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen"with the data reset to initial values.\n"
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainenvoid mbox_sync_set_critical(struct mbox_sync_context *sync_ctx,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen const char *fmt, ...)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen{
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen va_list va;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->errors = TRUE;
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen if (sync_ctx->ext_modified) {
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen "mbox file %s was modified while we were syncing, "
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen "check your locking settings",
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen mailbox_get_path(&sync_ctx->mbox->box));
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen }
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen va_start(va, fmt);
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen "Sync failed for mbox file %s: %s",
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen mailbox_get_path(&sync_ctx->mbox->box),
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen t_strdup_vprintf(fmt, va));
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen va_end(va);
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen}
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
13f6c879a84b3edd2fcc8f9832812be1f8c5d3b6Timo Sirainen{
8ddc45fe2080010715c212ecbb2b12b6734f6d4bTimo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
8ddc45fe2080010715c212ecbb2b12b6734f6d4bTimo Sirainen mbox_sync_set_critical(sync_ctx,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen from_offset);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return -1;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return 0;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen}
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainenvoid mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx)
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen{
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen struct stat st;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen /* Do this even if ext_modified is already set. Expunging code relies
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen on last_stat being updated. */
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen if (fstat(sync_ctx->write_fd, &st) < 0) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (st.st_size != sync_ctx->last_stat.st_size ||
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen (sync_ctx->last_stat.st_mtime != 0 &&
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen !CMP_ST_MTIME(&st, &sync_ctx->last_stat)))
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->ext_modified = TRUE;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->last_stat = st;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen}
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
81d3c215bb1fdbda2cf7ccd9519f6b4fd03c3791Timo Sirainenvoid mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty)
81d3c215bb1fdbda2cf7ccd9519f6b4fd03c3791Timo Sirainen{
f87844c400cf9741abad57d9815121d0738a738fTimo Sirainen if (dirty) {
81d3c215bb1fdbda2cf7ccd9519f6b4fd03c3791Timo Sirainen /* just mark the stat as dirty. */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->last_stat.st_mtime = 0;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0)
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen i_stream_sync(sync_ctx->input);
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen}
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainenstatic int
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen struct mbox_sync_mail_context *mail_ctx)
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen{
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen /* get EOF */
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input))
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen return 0;
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen p_clear(sync_ctx->mail_keyword_pool);
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mail_ctx->sync_ctx = sync_ctx;
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mail_ctx->seq = ++sync_ctx->seq;
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mail_ctx->header = sync_ctx->header;
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mail_ctx->mail.from_offset =
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mail_ctx->mail.offset =
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->input->eof);
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (istream_raw_mbox_is_corrupted(sync_ctx->input))
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return -1;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mail_ctx->mail.body_size =
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mail_ctx->content_length);
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 &&
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen !mail_ctx->mail.pseudo) {
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen if (!sync_ctx->keep_recent) {
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen /* need to add 'O' flag to Status-header */
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen mail_ctx->need_rewrite = TRUE;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen }
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen mail_ctx->recent = TRUE;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return 1;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen}
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainenstatic void mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen uint32_t uid, bool *sync_expunge_r)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen{
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen uint8_t expunged_guid_128[MAIL_GUID_128_SIZE];
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (uid == 0 || sync_ctx->index_reset) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen /* nothing for this or the future ones */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen uid = (uint32_t)-1;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen index_sync_changes_read(sync_ctx->sync_changes, uid, sync_expunge_r,
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen expunged_guid_128);
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen if (sync_ctx->readonly) {
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen /* we can't expunge anything from read-only mboxes */
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen *sync_expunge_r = FALSE;
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen }
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen}
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainenstatic bool
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen{
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen const struct mail_index_record *rec = NULL;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen uint32_t messages_count;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen bool ret = FALSE;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen if (sync_ctx->index_reset) {
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen *rec_r = NULL;
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen return TRUE;
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen }
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen messages_count =
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen while (sync_ctx->idx_seq <= messages_count) {
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen if (uid <= rec->uid)
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen break;
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen /* externally expunged message, remove from index */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->idx_seq++;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen rec = NULL;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (rec == NULL && uid < sync_ctx->idx_next_uid) {
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen /* this UID was already in index and it was expunged */
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen mbox_sync_set_critical(sync_ctx,
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen "Expunged message reappeared to mailbox "
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen "(UID %u < %u, seq=%u, idx_msgs=%u)",
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen uid, sync_ctx->idx_next_uid,
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen sync_ctx->seq, messages_count);
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen ret = FALSE; rec = NULL;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen } else if (rec != NULL && rec->uid != uid) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mbox_sync_set_critical(sync_ctx,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen "UID inserted in the middle of mailbox "
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen "(%u > %u, seq=%u, idx_msgs=%u)",
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen rec->uid, uid, sync_ctx->seq, messages_count);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen ret = FALSE; rec = NULL;
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen } else {
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen ret = TRUE;
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen }
9ed77dd00248e88d731ec129116db5dddc0dd3b5Timo Sirainen
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen *rec_r = rec;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return ret;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen}
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainenstatic void mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen unsigned char hdr_md5_sum[],
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen const struct mail_index_record **rec_r)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen{
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen const struct mail_index_record *rec = NULL;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen uint32_t messages_count;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen const void *data;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (sync_ctx->index_reset) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen *rec_r = NULL;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen messages_count =
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_index_view_get_messages_count(sync_ctx->sync_view);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen while (sync_ctx->idx_seq <= messages_count) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_index_lookup_ext(sync_ctx->sync_view,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->idx_seq,
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen sync_ctx->mbox->md5hdr_ext_idx,
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen &data, NULL);
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0)
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen break;
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen /* externally expunged message, remove from index */
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen sync_ctx->idx_seq++;
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen rec = NULL;
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen }
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen *rec_r = rec;
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen}
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainenstatic void
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
00bf64c70c231f647c12c2fd49925ef73cb07f07Timo Sirainen struct mbox_sync_mail *mail,
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen bool nocheck)
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen{
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen const void *data;
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen uint64_t offset;
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen if (!nocheck) {
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen /* see if from_offset needs updating */
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen sync_ctx->mbox->mbox_ext_idx,
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen &data, NULL);
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen if (data != NULL &&
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen *((const uint64_t *)data) == mail->from_offset)
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen return;
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen }
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen offset = mail->from_offset;
2ed248fba21fdd3abcc4bb4d07c2822b9ba3f66fTimo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen sync_ctx->mbox->mbox_ext_idx, &offset, NULL);
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen}
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainenstatic void
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainenmbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
2ed248fba21fdd3abcc4bb4d07c2822b9ba3f66fTimo Sirainen{
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen struct mail_index *index = sync_ctx->mbox->box.index;
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen struct mail_keywords *keywords;
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen keywords = !array_is_created(&mail_ctx->mail.keywords) ?
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen mail_index_keywords_create(index, NULL) :
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen mail_index_keywords_create_from_indexes(index,
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen &mail_ctx->mail.keywords);
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
7c3f90095b4168d89a268ac1ec820c5925d48fd3Timo Sirainen MODIFY_REPLACE, keywords);
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen mail_index_keywords_unref(&keywords);
2ed248fba21fdd3abcc4bb4d07c2822b9ba3f66fTimo Sirainen}
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainenstatic void
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainenmbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen{
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen const void *ext_data;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->mbox->md5hdr_ext_idx, &ext_data, NULL);
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen if (ext_data == NULL ||
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->mbox->md5hdr_ext_idx,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_ctx->hdr_md5_sum, NULL);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
2ed248fba21fdd3abcc4bb4d07c2822b9ba3f66fTimo Sirainen}
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainenstatic void mbox_sync_get_dirty_flags(struct mbox_sync_mail_context *mail_ctx,
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen const struct mail_index_record *rec)
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen{
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen ARRAY_TYPE(keyword_indexes) idx_keywords;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen uint8_t idx_flags, mbox_flags;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen /* default to undirtying the message. it gets added back if
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen flags/keywords don't match what is in the index. */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_ctx->mail.flags &= ~MAIL_INDEX_MAIL_FLAG_DIRTY;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen /* replace flags */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen idx_flags = rec->flags & MAIL_FLAGS_NONRECENT;
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen mbox_flags = mail_ctx->mail.flags & MAIL_FLAGS_NONRECENT;
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen if (idx_flags != mbox_flags) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_ctx->need_rewrite = TRUE;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_ctx->mail.flags = (mail_ctx->mail.flags & MAIL_RECENT) |
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen idx_flags | MAIL_INDEX_MAIL_FLAG_DIRTY;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen /* replace keywords */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen t_array_init(&idx_keywords, 32);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_index_lookup_keywords(sync_ctx->sync_view, sync_ctx->idx_seq,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen &idx_keywords);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (!index_keyword_array_cmp(&idx_keywords, &mail_ctx->mail.keywords)) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_ctx->need_rewrite = TRUE;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_ctx->mail.flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (!array_is_created(&mail_ctx->mail.keywords)) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen p_array_init(&mail_ctx->mail.keywords,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->mail_keyword_pool,
b1a2d2042e8c7e99983175eb462b82cc7a8cb70bTimo Sirainen array_count(&idx_keywords));
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
530f80fcbfa8be7378ff0d36ec7ecef41f198e2cTimo Sirainen array_clear(&mail_ctx->mail.keywords);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen array_append_array(&mail_ctx->mail.keywords, &idx_keywords);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen}
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainenstatic void mbox_sync_update_flags(struct mbox_sync_mail_context *mail_ctx,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen const struct mail_index_record *rec)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen{
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen struct mailbox *box = &sync_ctx->mbox->box;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen enum mail_index_sync_type sync_type;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen ARRAY_TYPE(keyword_indexes) orig_keywords = ARRAY_INIT;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen uint8_t flags, orig_flags;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (rec != NULL) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen /* flags and keywords are dirty. replace the current
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen ones from the flags in index file. */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mbox_sync_get_dirty_flags(mail_ctx, rec);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen flags = orig_flags = mail->flags & MAIL_FLAGS_NONRECENT;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (array_is_created(&mail->keywords)) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen t_array_init(&orig_keywords, 32);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen array_append_array(&orig_keywords, &mail->keywords);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen }
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen /* apply new changes */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen index_sync_changes_apply(sync_ctx->sync_changes,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen sync_ctx->mail_keyword_pool,
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen &flags, &mail->keywords, &sync_type);
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen if (flags != orig_flags ||
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen !index_keyword_array_cmp(&mail->keywords, &orig_keywords)) {
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen mail_ctx->need_rewrite = TRUE;
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen mail->flags = flags | (mail->flags & MAIL_RECENT) |
adcb46fe2c6d1139dfbd8ab08a8fbe96e53f8fd6Timo Sirainen MAIL_INDEX_MAIL_FLAG_DIRTY;
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen }
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen if (sync_type != 0 && box->v.sync_notify != NULL) {
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen box->v.sync_notify(box, mail_ctx->mail.uid,
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen index_sync_type_convert(sync_type));
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen }
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen}
df452e9628fe8d3356c42dd644b020ea9733c0c1Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainenstatic void mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen const struct mail_index_record *rec)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen{
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen struct mbox_sync_mail *mail = &mail_ctx->mail;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen ARRAY_TYPE(keyword_indexes) idx_keywords;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen uint8_t mbox_flags;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
mbox_flags = mail->flags & ~MAIL_RECENT;
if (!sync_ctx->delay_writes) {
/* changes are written to the mbox file */
mbox_flags &= ~MAIL_INDEX_MAIL_FLAG_DIRTY;
} else if (mail_ctx->need_rewrite) {
/* make sure this message gets written later */
mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
}
if (rec == NULL) {
/* new message */
mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags);
mbox_sync_update_index_keywords(mail_ctx);
if (sync_ctx->mbox->mbox_save_md5 != 0) {
mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->mbox->md5hdr_ext_idx,
mail_ctx->hdr_md5_sum, NULL);
}
} else {
if ((rec->flags & MAIL_FLAGS_NONRECENT) !=
(mbox_flags & MAIL_FLAGS_NONRECENT)) {
/* flags other than recent/dirty have changed */
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags);
} else if (((rec->flags ^ mbox_flags) &
MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* only dirty flag state changed */
bool dirty;
dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
dirty ? MODIFY_ADD : MODIFY_REMOVE,
(enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY);
}
/* see if keywords changed */
t_array_init(&idx_keywords, 32);
mail_index_lookup_keywords(sync_ctx->sync_view,
sync_ctx->idx_seq, &idx_keywords);
if (!index_keyword_array_cmp(&idx_keywords, &mail->keywords))
mbox_sync_update_index_keywords(mail_ctx);
/* see if we need to update md5 sum. */
if (sync_ctx->mbox->mbox_save_md5 != 0)
mbox_sync_update_md5_if_changed(mail_ctx);
}
if (!mail_ctx->recent) {
/* Mail has "Status: O" header. No messages before this
can be recent. */
sync_ctx->last_nonrecent_uid = mail->uid;
}
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
bool nocheck = rec == NULL || sync_ctx->expunged_space > 0;
mbox_sync_update_from_offset(sync_ctx, mail, nocheck);
}
}
static int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
{
struct istream *input = ctx->sync_ctx->file_input;
const unsigned char *data;
size_t size, from_line_size;
buffer_set_used_size(ctx->sync_ctx->from_line, 0);
from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
i_stream_seek(input, ctx->mail.from_offset);
for (;;) {
data = i_stream_get_data(input, &size);
if (size >= from_line_size)
size = from_line_size;
buffer_append(ctx->sync_ctx->from_line, data, size);
i_stream_skip(input, size);
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx)
{
unsigned char buf[10];
const char *str;
uint32_t uid_last;
unsigned int i;
int ret;
i_assert(sync_ctx->base_uid_last_offset != 0);
/* first check that the 10 bytes are there and they're exactly as
expected. just an extra safety check to make sure we never write
to wrong location in the mbox file. */
ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf),
sync_ctx->base_uid_last_offset);
if (ret < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "pread_full()");
return -1;
}
if (ret == 0) {
mbox_sync_set_critical(sync_ctx,
"X-IMAPbase uid-last offset unexpectedly outside mbox");
return -1;
}
for (i = 0, uid_last = 0; i < sizeof(buf); i++) {
if (buf[i] < '0' || buf[i] > '9') {
uid_last = (uint32_t)-1;
break;
}
uid_last = uid_last * 10 + (buf[i] - '0');
}
if (uid_last != sync_ctx->base_uid_last) {
mbox_sync_set_critical(sync_ctx,
"X-IMAPbase uid-last unexpectedly lost");
return -1;
}
/* and write it */
str = t_strdup_printf("%010u", sync_ctx->next_uid - 1);
if (pwrite_full(sync_ctx->write_fd, str, 10,
sync_ctx->base_uid_last_offset) < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
return -1;
}
mbox_sync_file_updated(sync_ctx, FALSE);
sync_ctx->base_uid_last = sync_ctx->next_uid - 1;
return 0;
}
static int
mbox_write_from_line(struct mbox_sync_mail_context *ctx)
{
string_t *str = ctx->sync_ctx->from_line;
if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str),
ctx->mail.from_offset) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()");
return -1;
}
mbox_sync_file_updated(ctx->sync_ctx, FALSE);
return 0;
}
static void update_from_offsets(struct mbox_sync_context *sync_ctx)
{
const struct mbox_sync_mail *mails;
unsigned int i, count;
uint32_t ext_idx;
uint64_t offset;
ext_idx = sync_ctx->mbox->mbox_ext_idx;
mails = array_get(&sync_ctx->mails, &count);
for (i = 0; i < count; i++) {
if (mails[i].idx_seq == 0 || mails[i].expunged)
continue;
sync_ctx->moved_offsets = TRUE;
offset = mails[i].from_offset;
mail_index_update_ext(sync_ctx->t, mails[i].idx_seq,
ext_idx, &offset, NULL);
}
}
static void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
struct mailbox *box = &sync_ctx->mbox->box;
if (box->v.sync_notify != NULL) {
box->v.sync_notify(box, mail_ctx->mail.uid,
MAILBOX_SYNC_TYPE_EXPUNGE);
}
mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq);
mail_ctx->mail.expunged = TRUE;
mail_ctx->mail.offset = mail_ctx->mail.from_offset;
mail_ctx->mail.space =
mail_ctx->body_offset - mail_ctx->mail.from_offset +
mail_ctx->mail.body_size;
mail_ctx->mail.body_size = 0;
mail_ctx->mail.uid = 0;
if (sync_ctx->seq == 1) {
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
mail_ctx->mail.space++;
if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) {
mail_ctx->mail.space++;
sync_ctx->first_mail_crlf_expunged = TRUE;
}
/* uid-last offset is invalid now */
sync_ctx->base_uid_last_offset = 0;
}
sync_ctx->expunged_space += mail_ctx->mail.space;
}
static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t orig_from_offset;
off_t move_diff;
int ret;
if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
/* move the header backwards to fill expunged space */
move_diff = -sync_ctx->expunged_space;
orig_from_offset = mail_ctx->mail.from_offset;
if (sync_ctx->dest_first_mail) {
/* we're moving this mail to beginning of file.
skip the initial \n (it's already counted in
expunged_space) */
mail_ctx->mail.from_offset++;
if (sync_ctx->first_mail_crlf_expunged)
mail_ctx->mail.from_offset++;
}
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
mbox_sync_update_header(mail_ctx);
ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
if (ret < 0)
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
i_assert(move_diff > 0 ||
(off_t)mail_ctx->mail.from_offset >=
-move_diff);
mail_ctx->mail.from_offset += move_diff;
mail_ctx->mail.offset += move_diff;
if (mbox_write_from_line(mail_ctx) < 0)
return -1;
} else {
if (sync_ctx->dest_first_mail) {
/* didn't have enough space, move the offset
back so seeking into it doesn't fail */
mail_ctx->mail.from_offset = orig_from_offset;
}
}
} else if (mail_ctx->need_rewrite) {
mbox_sync_update_header(mail_ctx);
if (sync_ctx->delay_writes) {
/* mark it dirty and do it later */
mail_ctx->dirty = TRUE;
return 0;
}
if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
return -1;
} else {
/* nothing to do */
return 0;
}
if (ret == 0 && sync_ctx->need_space_seq == 0) {
/* first mail with no space to write it */
sync_ctx->need_space_seq = sync_ctx->seq;
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
memset(&mail, 0, sizeof(mail));
mail.expunged = TRUE;
mail.offset = mail.from_offset =
(sync_ctx->dest_first_mail ? 1 : 0) +
mail_ctx->mail.from_offset -
sync_ctx->expunged_space;
mail.space = sync_ctx->expunged_space;
sync_ctx->space_diff = sync_ctx->expunged_space;
sync_ctx->expunged_space = 0;
i_assert(sync_ctx->space_diff < -mail_ctx->mail.space);
sync_ctx->need_space_seq--;
array_append(&sync_ctx->mails, &mail, 1);
}
}
return 0;
}
static int
mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t end_offset, move_diff, extra_space, needed_space;
uint32_t last_seq;
ARRAY_TYPE(keyword_indexes) keywords_copy;
i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 ||
mail_ctx->mail.offset == mail_ctx->hdr_offset);
if (array_is_created(&mail_ctx->mail.keywords)) {
/* mail's keywords are allocated from a pool that's cleared
for each mail. we'll need to copy it to something more
permanent. */
p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool,
array_count(&mail_ctx->mail.keywords));
array_append_array(&keywords_copy, &mail_ctx->mail.keywords);
mail_ctx->mail.keywords = keywords_copy;
}
array_append(&sync_ctx->mails, &mail_ctx->mail, 1);
sync_ctx->space_diff += mail_ctx->mail.space;
if (sync_ctx->space_diff < 0) {
if (sync_ctx->expunged_space > 0) {
i_assert(sync_ctx->expunged_space ==
mail_ctx->mail.space);
sync_ctx->expunged_space = 0;
}
return 0;
}
/* we have enough space now */
if (mail_ctx->mail.uid == 0) {
/* this message was expunged. fill more or less of the space.
space_diff now consists of a negative "bytes needed" sum,
plus the expunged space of this message. so it contains how
many bytes of _extra_ space we have. */
i_assert(mail_ctx->mail.space >= sync_ctx->space_diff);
extra_space = MBOX_HEADER_PADDING *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
/* don't waste too much on padding */
move_diff = needed_space + extra_space;
sync_ctx->expunged_space =
mail_ctx->mail.space - move_diff;
} else {
move_diff = mail_ctx->mail.space;
extra_space = sync_ctx->space_diff;
sync_ctx->expunged_space = 0;
}
last_seq = sync_ctx->seq - 1;
array_delete(&sync_ctx->mails,
array_count(&sync_ctx->mails) - 1, 1);
end_offset = mail_ctx->mail.from_offset;
} else {
/* this message gave enough space from headers. rewriting stops
at the end of this message's headers. */
sync_ctx->expunged_space = 0;
last_seq = sync_ctx->seq;
end_offset = mail_ctx->body_offset;
move_diff = 0;
extra_space = sync_ctx->space_diff;
}
mbox_sync_file_update_ext_modified(sync_ctx);
if (mbox_sync_rewrite(sync_ctx,
last_seq == sync_ctx->seq ? mail_ctx : NULL,
end_offset, move_diff, extra_space,
sync_ctx->need_space_seq, last_seq) < 0)
return -1;
update_from_offsets(sync_ctx);
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
memset(mail_ctx, 0, sizeof(*mail_ctx));
sync_ctx->need_space_seq = 0;
sync_ctx->space_diff = 0;
array_clear(&sync_ctx->mails);
p_clear(sync_ctx->saved_keywords_pool);
return 0;
}
static int
mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
{
struct mbox_mailbox *mbox = sync_ctx->mbox;
uoff_t old_offset;
uint32_t uid;
int ret;
bool deleted;
if (seq == 0) {
if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) {
mbox->invalid_mbox_file = TRUE;
mail_storage_set_error(&mbox->storage->storage,
MAIL_ERROR_NOTPOSSIBLE,
"Mailbox isn't a valid mbox file");
return -1;
}
seq++;
} else {
old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted);
if (ret < 0) {
if (deleted) {
mbox_sync_set_critical(sync_ctx,
"Message was expunged unexpectedly");
}
return -1;
}
if (ret == 0) {
if (istream_raw_mbox_seek(mbox->mbox_stream,
old_offset) < 0) {
mbox_sync_set_critical(sync_ctx,
"Error seeking back to original "
"offset %s", dec2str(old_offset));
return -1;
}
return 0;
}
}
if (seq <= 1)
uid = 0;
else
mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid);
sync_ctx->prev_msg_uid = uid;
/* set to -1, since it's always increased later */
sync_ctx->seq = seq-1;
if (sync_ctx->seq == 0 &&
istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
/* this mbox has pseudo mail which contains the X-IMAP header */
sync_ctx->seq++;
}
sync_ctx->idx_seq = seq;
sync_ctx->dest_first_mail = sync_ctx->seq == 0;
(void)istream_raw_mbox_get_body_offset(sync_ctx->input);
return 1;
}
static int
mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
{
struct mail_index_view *sync_view = sync_ctx->sync_view;
uint32_t seq1, seq2;
uoff_t size;
int ret;
i_assert(!sync_ctx->index_reset);
if (!mail_index_lookup_seq_range(sync_view, uid, (uint32_t)-1,
&seq1, &seq2)) {
/* doesn't exist anymore, seek to end of file */
ret = i_stream_get_size(sync_ctx->file_input, TRUE, &size);
if (ret < 0) {
mbox_set_syscall_error(sync_ctx->mbox,
"i_stream_get_size()");
return -1;
}
i_assert(ret != 0);
if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream,
size) < 0) {
mbox_sync_set_critical(sync_ctx,
"Error seeking to end of mbox");
return -1;
}
sync_ctx->idx_seq =
mail_index_view_get_messages_count(sync_view) + 1;
return 1;
}
return mbox_sync_seek_to_seq(sync_ctx, seq1);
}
static int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx,
uint32_t next_uid, bool *partial,
bool *skipped_mails)
{
uint32_t messages_count, uid;
int ret;
i_assert(!sync_ctx->index_reset);
/* delete sync records up to next message. so if there's still
something left in array, it means the next message needs modifying */
index_sync_changes_delete_to(sync_ctx->sync_changes, next_uid);
if (index_sync_changes_have(sync_ctx->sync_changes))
return 1;
if (sync_ctx->hdr->first_recent_uid <= next_uid &&
!sync_ctx->keep_recent) {
/* we'll need to rewrite Status: O headers */
return 1;
}
uid = index_sync_changes_get_next_uid(sync_ctx->sync_changes);
if (sync_ctx->hdr->first_recent_uid < sync_ctx->hdr->next_uid &&
(uid > sync_ctx->hdr->first_recent_uid || uid == 0) &&
!sync_ctx->keep_recent) {
/* we'll need to rewrite Status: O headers */
uid = sync_ctx->hdr->first_recent_uid;
}
if (uid != 0) {
/* we can skip forward to next record which needs updating. */
if (uid != next_uid) {
*skipped_mails = TRUE;
next_uid = uid;
}
ret = mbox_sync_seek_to_uid(sync_ctx, next_uid);
} else {
/* if there's no sync records left, we can stop. except if
this is a dirty sync, check if there are new messages. */
if (!sync_ctx->mbox->mbox_hdr.dirty_flag)
return 0;
messages_count =
mail_index_view_get_messages_count(sync_ctx->sync_view);
if (sync_ctx->seq + 1 != messages_count) {
ret = mbox_sync_seek_to_seq(sync_ctx, messages_count);
*skipped_mails = TRUE;
} else {
ret = 1;
}
*partial = FALSE;
}
if (ret == 0) {
/* seek failed because the offset is dirty. just ignore and
continue from where we are now. */
*partial = FALSE;
ret = 1;
}
return ret;
}
static void mbox_sync_hdr_update(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update;
if (update->uid_validity != 0) {
sync_ctx->base_uid_validity = update->uid_validity;
mail_ctx->imapbase_rewrite = TRUE;
mail_ctx->need_rewrite = TRUE;
}
if (update->min_next_uid != 0 &&
sync_ctx->base_uid_last <= update->min_next_uid) {
sync_ctx->base_uid_last = update->min_next_uid-1;
sync_ctx->next_uid = update->min_next_uid;
mail_ctx->imapbase_rewrite = TRUE;
mail_ctx->need_rewrite = TRUE;
}
}
static bool mbox_sync_imapbase(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
if (sync_ctx->base_uid_validity != 0 &&
sync_ctx->hdr->uid_validity != 0 &&
sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s",
sync_ctx->hdr->uid_validity,
sync_ctx->base_uid_validity,
mailbox_get_path(&sync_ctx->mbox->box));
sync_ctx->index_reset = TRUE;
return TRUE;
}
if (sync_ctx->mbox->sync_hdr_update != NULL)
mbox_sync_hdr_update(sync_ctx, mail_ctx);
return FALSE;
}
static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
bool partial)
{
const struct mail_index_record *rec;
uint32_t uid, messages_count;
uoff_t offset;
int ret;
bool expunged, skipped_mails, uids_broken;
messages_count =
mail_index_view_get_messages_count(sync_ctx->sync_view);
/* always start from first message so we can read X-IMAP or
X-IMAPbase header */
ret = mbox_sync_seek_to_seq(sync_ctx, 0);
if (ret <= 0)
return ret;
if (sync_ctx->renumber_uids) {
/* expunge everything */
while (sync_ctx->idx_seq <= messages_count) {
mail_index_expunge(sync_ctx->t,
sync_ctx->idx_seq++);
}
}
skipped_mails = uids_broken = FALSE;
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
uid = mail_ctx->mail.uid;
if (mail_ctx->seq == 1) {
if (mbox_sync_imapbase(sync_ctx, mail_ctx)) {
sync_ctx->mbox->mbox_hdr.dirty_flag = TRUE;
return 0;
}
}
if (mail_ctx->mail.uid_broken && partial) {
/* UID ordering problems, resync everything to make
sure we get everything right */
if (sync_ctx->mbox->mbox_hdr.dirty_flag)
return 0;
mbox_sync_set_critical(sync_ctx,
"UIDs broken with partial sync");
sync_ctx->mbox->mbox_hdr.dirty_flag = TRUE;
return 0;
}
if (mail_ctx->mail.uid_broken)
uids_broken = TRUE;
if (mail_ctx->mail.pseudo)
uid = 0;
rec = NULL; ret = 1;
if (uid != 0) {
if (!mbox_sync_read_index_rec(sync_ctx, uid, &rec))
ret = 0;
}
if (ret == 0) {
/* UID found but it's broken */
uid = 0;
} else if (uid == 0 &&
!mail_ctx->mail.pseudo &&
(sync_ctx->delay_writes ||
sync_ctx->idx_seq <= messages_count)) {
/* If we can't use/store X-UID header, use MD5 sum.
Also check for existing MD5 sums when we're actually
able to write X-UIDs. */
sync_ctx->mbox->mbox_save_md5 = TRUE;
mbox_sync_find_index_md5(sync_ctx,
mail_ctx->hdr_md5_sum, &rec);
if (rec != NULL)
uid = mail_ctx->mail.uid = rec->uid;
}
/* get all sync records related to this message. with pseudo
message just get the first sync record so we can jump to
it with partial seeking. */
mbox_sync_read_index_syncs(sync_ctx,
mail_ctx->mail.pseudo ? 1 : uid,
&expunged);
if (mail_ctx->mail.pseudo) {
/* if it was set, it was for the next message */
expunged = FALSE;
} else {
if (rec == NULL) {
/* message wasn't found from index. we have to
read everything from now on, no skipping */
partial = FALSE;
}
}
if (uid == 0 && !mail_ctx->mail.pseudo) {
/* missing/broken X-UID. all the rest of the mails
need new UIDs. */
while (sync_ctx->idx_seq <= messages_count) {
mail_index_expunge(sync_ctx->t,
sync_ctx->idx_seq++);
}
if (sync_ctx->next_uid == (uint32_t)-1) {
/* oh no, we're out of UIDs. this shouldn't
happen normally, so just try to get it fixed
without crashing. */
mail_storage_set_critical(
&sync_ctx->mbox->storage->storage,
"Out of UIDs, renumbering them in mbox "
"file %s",
mailbox_get_path(&sync_ctx->mbox->box));
sync_ctx->renumber_uids = TRUE;
return 0;
}
mail_ctx->need_rewrite = TRUE;
mail_ctx->mail.uid = sync_ctx->next_uid++;
}
sync_ctx->prev_msg_uid = mail_ctx->mail.uid;
if (!mail_ctx->mail.pseudo)
mail_ctx->mail.idx_seq = sync_ctx->idx_seq;
if (!expunged) {
if (!mail_ctx->mail.pseudo) T_BEGIN {
mbox_sync_update_flags(mail_ctx, rec);
} T_END;
if (mbox_sync_handle_header(mail_ctx) < 0)
return -1;
sync_ctx->dest_first_mail = FALSE;
} else {
mbox_sync_handle_expunge(mail_ctx);
}
if (!mail_ctx->mail.pseudo) {
if (!expunged) T_BEGIN {
mbox_sync_update_index(mail_ctx, rec);
} T_END;
sync_ctx->idx_seq++;
}
istream_raw_mbox_next(sync_ctx->input,
mail_ctx->mail.body_size);
offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
mbox_sync_file_update_ext_modified(sync_ctx);
if (mbox_move(sync_ctx,
mail_ctx->body_offset -
sync_ctx->expunged_space,
mail_ctx->body_offset,
mail_ctx->mail.body_size) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
}
} else if (partial) {
ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1,
&partial,
&skipped_mails);
if (ret <= 0)
break;
}
}
if (ret < 0)
return -1;
if (istream_raw_mbox_is_eof(sync_ctx->input)) {
/* rest of the messages in index don't exist -> expunge them */
while (sync_ctx->idx_seq <= messages_count)
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
}
if (!skipped_mails)
sync_ctx->mbox->mbox_hdr.dirty_flag = FALSE;
sync_ctx->mbox->mbox_broken_offsets = FALSE;
if (uids_broken && sync_ctx->delay_writes) {
/* once we get around to writing the changes, we'll need to do
a full sync to avoid the "UIDs broken in partial sync"
error */
sync_ctx->mbox->mbox_hdr.dirty_flag = TRUE;
}
return 1;
}
static int mbox_write_pseudo(struct mbox_sync_context *sync_ctx)
{
string_t *str;
unsigned int uid_validity;
i_assert(sync_ctx->write_fd != -1);
if (sync_ctx->mbox->sync_hdr_update != NULL) {
const struct mailbox_update *update =
sync_ctx->mbox->sync_hdr_update;
if (update->uid_validity != 0)
sync_ctx->base_uid_validity = update->uid_validity;
if (update->min_next_uid != 0)
sync_ctx->base_uid_last = update->min_next_uid-1;
}
uid_validity = sync_ctx->base_uid_validity != 0 ?
sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity;
i_assert(uid_validity != 0);
str = t_str_new(1024);
str_printfa(str, "%sDate: %s\n"
"From: Mail System Internal Data <MAILER-DAEMON@%s>\n"
"Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA"
"\nMessage-ID: <%s@%s>\n"
"X-IMAP: %u %010u\n"
"Status: RO\n"
"\n"
PSEUDO_MESSAGE_BODY
"\n",
mbox_from_create("MAILER_DAEMON", ioloop_time),
message_date_create(ioloop_time),
my_hostname, dec2str(ioloop_time), my_hostname,
uid_validity, sync_ctx->next_uid-1);
if (pwrite_full(sync_ctx->write_fd,
str_data(str), str_len(str), 0) < 0) {
if (!ENOSPACE(errno)) {
mbox_set_syscall_error(sync_ctx->mbox,
"pwrite_full()");
return -1;
}
/* out of disk space, truncate to empty */
if (ftruncate(sync_ctx->write_fd, 0) < 0)
mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
}
sync_ctx->base_uid_validity = uid_validity;
sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */
sync_ctx->base_uid_last = sync_ctx->next_uid-1;
return 0;
}
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
uoff_t file_size, offset, padding, trailer_size;
int ret;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
i_assert(sync_ctx->expunged_space == 0);
return 0;
}
ret = i_stream_get_size(sync_ctx->file_input, TRUE, &file_size);
if (ret < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "i_stream_get_size()");
return -1;
}
if (ret == 0) {
/* Not a file - allow anyway */
return 0;
}
if (file_size < sync_ctx->file_input->v_offset) {
mbox_sync_set_critical(sync_ctx,
"file size unexpectedly shrank "
"(%"PRIuUOFF_T" vs %"PRIuUOFF_T")", file_size,
sync_ctx->file_input->v_offset);
return -1;
}
trailer_size = file_size - sync_ctx->file_input->v_offset;
i_assert(trailer_size <= 2);
if (sync_ctx->need_space_seq != 0) {
i_assert(sync_ctx->write_fd != -1);
i_assert(sync_ctx->space_diff < 0);
padding = MBOX_HEADER_PADDING *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
sync_ctx->space_diff -= padding;
i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff);
sync_ctx->space_diff += sync_ctx->expunged_space;
sync_ctx->expunged_space = 0;
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
i_assert(sync_ctx->space_diff < 0);
if (file_set_size(sync_ctx->write_fd,
file_size + -sync_ctx->space_diff) < 0) {
mbox_set_syscall_error(sync_ctx->mbox,
"file_set_size()");
if (ftruncate(sync_ctx->write_fd, file_size) < 0) {
mbox_set_syscall_error(sync_ctx->mbox,
"ftruncate()");
}
return -1;
}
mbox_sync_file_updated(sync_ctx, FALSE);
if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size,
-sync_ctx->space_diff, padding,
sync_ctx->need_space_seq,
sync_ctx->seq) < 0)
return -1;
update_from_offsets(sync_ctx);
sync_ctx->need_space_seq = 0;
array_clear(&sync_ctx->mails);
p_clear(sync_ctx->saved_keywords_pool);
}
if (sync_ctx->expunged_space > 0) {
i_assert(sync_ctx->write_fd != -1);
mbox_sync_file_update_ext_modified(sync_ctx);
/* copy trailer, then truncate the file */
file_size = sync_ctx->last_stat.st_size;
if (file_size == (uoff_t)sync_ctx->expunged_space) {
/* everything deleted, the trailer_size still contains
the \n trailer though */
trailer_size = 0;
} else if (sync_ctx->expunged_space == (off_t)file_size + 1 ||
sync_ctx->expunged_space == (off_t)file_size + 2) {
/* everything deleted and we didn't have a proper
trailer. */
trailer_size = 0;
sync_ctx->expunged_space = file_size;
}
i_assert(file_size >= sync_ctx->expunged_space + trailer_size);
offset = file_size - sync_ctx->expunged_space - trailer_size;
i_assert(offset == 0 || offset > 31);
if (mbox_move(sync_ctx, offset,
offset + sync_ctx->expunged_space,
trailer_size) < 0)
return -1;
if (ftruncate(sync_ctx->write_fd,
offset + trailer_size) < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
return -1;
}
if (offset == 0) {
if (mbox_write_pseudo(sync_ctx) < 0)
return -1;
}
sync_ctx->expunged_space = 0;
mbox_sync_file_updated(sync_ctx, FALSE);
} else {
if (file_size == 0 && sync_ctx->mbox->sync_hdr_update != NULL) {
if (mbox_write_pseudo(sync_ctx) < 0)
return -1;
}
}
return 0;
}
static void
mbox_sync_index_update_ext_header(struct mbox_sync_context *sync_ctx)
{
const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update;
struct mbox_mailbox *mbox = sync_ctx->mbox;
const void *data;
size_t data_size;
if (update != NULL && !mail_guid_128_is_empty(update->mailbox_guid)) {
memcpy(mbox->mbox_hdr.mailbox_guid, update->mailbox_guid,
sizeof(mbox->mbox_hdr.mailbox_guid));
} else if (mail_guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
mail_generate_guid_128(mbox->mbox_hdr.mailbox_guid);
}
mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx,
&data, &data_size);
if (data_size != sizeof(mbox->mbox_hdr) ||
memcmp(data, &mbox->mbox_hdr, data_size) != 0) {
mail_index_update_header_ext(sync_ctx->t, mbox->mbox_ext_idx,
0, &mbox->mbox_hdr,
sizeof(mbox->mbox_hdr));
}
}
static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
{
struct mail_index_view *view;
const struct stat *st;
uint32_t first_recent_uid, seq, seq2;
st = i_stream_stat(sync_ctx->file_input, FALSE);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
return -1;
}
if (sync_ctx->moved_offsets &&
((uint64_t)st->st_size == sync_ctx->mbox->mbox_hdr.sync_size ||
(uint64_t)st->st_size == sync_ctx->orig_size)) {
/* We moved messages inside the mbox file without changing
the file's size. If mtime doesn't change, another process
not using the same index file as us can't know that the file
was changed. So make sure the mtime changes. This should
happen rarely enough that the sleeping doesn't become a
performance problem.
Note that to do this perfectly safe we should do this wait
whenever mails are moved or expunged, regardless of whether
the file's size changed. That however could become a
performance problem and the consequences of being wrong are
quite minimal (an extra logged error message). */
while (sync_ctx->orig_mtime == st->st_mtime) {
usleep(500000);
if (utime(mailbox_get_path(&sync_ctx->mbox->box), NULL) < 0) {
mbox_set_syscall_error(sync_ctx->mbox,
"utime()");
return -1;
}
st = i_stream_stat(sync_ctx->file_input, FALSE);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox,
"i_stream_stat()");
return -1;
}
}
}
sync_ctx->mbox->mbox_hdr.sync_mtime = st->st_mtime;
sync_ctx->mbox->mbox_hdr.sync_size = st->st_size;
mbox_sync_index_update_ext_header(sync_ctx);
/* only reason not to have UID validity at this point is if the file
is entirely empty. In that case just make up a new one if needed. */
i_assert(sync_ctx->base_uid_validity != 0 || st->st_size <= 0);
if (sync_ctx->base_uid_validity == 0) {
sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ?
sync_ctx->hdr->uid_validity :
I_MAX((unsigned int)ioloop_time, 1);
}
if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, uid_validity),
&sync_ctx->base_uid_validity,
sizeof(sync_ctx->base_uid_validity), TRUE);
}
if (istream_raw_mbox_is_eof(sync_ctx->input) &&
sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
i_assert(sync_ctx->next_uid != 0);
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, next_uid),
&sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
}
if (sync_ctx->last_nonrecent_uid < sync_ctx->hdr->first_recent_uid) {
/* other sessions have already marked more messages as
recent. */
sync_ctx->last_nonrecent_uid =
sync_ctx->hdr->first_recent_uid - 1;
}
/* mark recent messages */
view = mail_index_transaction_open_updated_view(sync_ctx->t);
if (mail_index_lookup_seq_range(view, sync_ctx->last_nonrecent_uid + 1,
(uint32_t)-1, &seq, &seq2)) {
index_mailbox_set_recent_seq(&sync_ctx->mbox->box,
view, seq, seq2);
}
mail_index_view_close(&view);
first_recent_uid = !sync_ctx->keep_recent ?
sync_ctx->next_uid : sync_ctx->last_nonrecent_uid + 1;
if (sync_ctx->hdr->first_recent_uid < first_recent_uid) {
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, first_recent_uid),
&first_recent_uid, sizeof(first_recent_uid), FALSE);
}
return 0;
}
static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->base_uid_last_offset = 0;
array_clear(&sync_ctx->mails);
p_clear(sync_ctx->saved_keywords_pool);
index_sync_changes_reset(sync_ctx->sync_changes);
mail_index_sync_reset(sync_ctx->index_sync_ctx);
mail_index_transaction_reset(sync_ctx->t);
if (sync_ctx->index_reset) {
mail_index_reset(sync_ctx->t);
sync_ctx->reset_hdr.next_uid = 1;
sync_ctx->hdr = &sync_ctx->reset_hdr;
index_mailbox_reset_uidvalidity(&sync_ctx->mbox->box);
}
sync_ctx->prev_msg_uid = 0;
sync_ctx->next_uid = sync_ctx->hdr->next_uid;
sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
sync_ctx->seq = 0;
sync_ctx->idx_seq = 1;
sync_ctx->need_space_seq = 0;
sync_ctx->expunged_space = 0;
sync_ctx->space_diff = 0;
sync_ctx->dest_first_mail = TRUE;
sync_ctx->ext_modified = FALSE;
sync_ctx->errors = FALSE;
}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx,
enum mbox_sync_flags flags)
{
struct mbox_index_header *mbox_hdr = &sync_ctx->mbox->mbox_hdr;
struct mbox_sync_mail_context mail_ctx;
const struct stat *st;
unsigned int i;
int ret, partial;
st = i_stream_stat(sync_ctx->file_input, FALSE);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
return -1;
}
sync_ctx->last_stat = *st;
sync_ctx->orig_size = st->st_size;
sync_ctx->orig_atime = st->st_atime;
sync_ctx->orig_mtime = st->st_mtime;
if ((flags & MBOX_SYNC_FORCE_SYNC) != 0) {
/* forcing a full sync. assume file has changed. */
partial = FALSE;
mbox_hdr->dirty_flag = TRUE;
} else if ((uint32_t)st->st_mtime == mbox_hdr->sync_mtime &&
(uint64_t)st->st_size == mbox_hdr->sync_size) {
/* file is fully synced */
if (mbox_hdr->dirty_flag && (flags & MBOX_SYNC_UNDIRTY) != 0)
partial = FALSE;
else
partial = TRUE;
} else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
(uint64_t)st->st_size == mbox_hdr->sync_size) {
/* we want to do full syncing. always do this if
file size hasn't changed but timestamp has. it most
likely means that someone had modified some header
and we probably want to know about it */
partial = FALSE;
sync_ctx->mbox->mbox_hdr.dirty_flag = TRUE;
} else {
/* see if we can delay syncing the whole file.
normally we only notice expunges and appends
in partial syncing. */
partial = TRUE;
sync_ctx->mbox->mbox_hdr.dirty_flag = TRUE;
}
mbox_sync_restart(sync_ctx);
for (i = 0;;) {
ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial);
if (ret > 0 && !sync_ctx->errors)
break;
if (ret < 0)
return -1;
/* a) partial sync didn't work
b) we ran out of UIDs
c) syncing had errors */
if (sync_ctx->delay_writes &&
(sync_ctx->errors || sync_ctx->renumber_uids)) {
/* fixing a broken mbox state, be sure to write
the changes (except if we're readonly). */
if (!sync_ctx->readonly)
sync_ctx->delay_writes = FALSE;
}
if (++i == 3)
break;
mbox_sync_restart(sync_ctx);
partial = FALSE;
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
index_sync_changes_reset(sync_ctx->sync_changes);
if (sync_ctx->base_uid_last != sync_ctx->next_uid-1 &&
ret > 0 && !sync_ctx->delay_writes &&
sync_ctx->base_uid_last_offset != 0) {
/* Rewrite uid_last in X-IMAPbase header if we've seen it
(ie. the file isn't empty) */
ret = mbox_rewrite_base_uid_last(sync_ctx);
} else {
ret = 0;
}
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return ret;
}
int mbox_sync_header_refresh(struct mbox_mailbox *mbox)
{
const struct mail_index_header *hdr;
const void *data;
size_t data_size;
if (mail_index_refresh(mbox->box.index) < 0) {
mail_storage_set_index_error(&mbox->box);
return -1;
}
mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx,
&data, &data_size);
if (data_size == 0) {
/* doesn't exist. FIXME: backwards compatibility copying */
hdr = mail_index_get_header(mbox->box.view);
mbox->mbox_hdr.sync_mtime = hdr->sync_stamp;
mbox->mbox_hdr.sync_size = hdr->sync_size;
return 0;
}
memcpy(&mbox->mbox_hdr, data, I_MIN(sizeof(mbox->mbox_hdr), data_size));
if (mbox->mbox_broken_offsets)
mbox->mbox_hdr.dirty_flag = TRUE;
return 0;
}
int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty)
{
bool empty;
return mbox_sync_has_changed_full(mbox, leave_dirty, &empty);
}
int mbox_sync_has_changed_full(struct mbox_mailbox *mbox, bool leave_dirty,
bool *empty_r)
{
const struct stat *st;
struct stat statbuf;
if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) {
/* read-only stream */
st = i_stream_stat(mbox->mbox_file_stream, FALSE);
if (st == NULL) {
if (errno == ENOENT) {
mailbox_set_deleted(&mbox->box);
return 0;
}
mbox_set_syscall_error(mbox, "i_stream_stat()");
return -1;
}
} else {
if (stat(mailbox_get_path(&mbox->box), &statbuf) < 0) {
if (errno == ENOENT) {
mailbox_set_deleted(&mbox->box);
return 0;
}
mbox_set_syscall_error(mbox, "stat()");
return -1;
}
st = &statbuf;
}
*empty_r = st->st_size == 0;
if (mbox_sync_header_refresh(mbox) < 0)
return -1;
if (mail_guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
/* need to assign mailbox GUID */
return 1;
}
if ((uint32_t)st->st_mtime == mbox->mbox_hdr.sync_mtime &&
(uint64_t)st->st_size == mbox->mbox_hdr.sync_size) {
/* fully synced */
if (!mbox->mbox_hdr.dirty_flag || leave_dirty)
return 0;
/* flushing dirtyness */
}
/* file changed */
return 1;
}
static void mbox_sync_context_free(struct mbox_sync_context *sync_ctx)
{
index_sync_changes_deinit(&sync_ctx->sync_changes);
if (sync_ctx->index_sync_ctx != NULL)
mail_index_sync_rollback(&sync_ctx->index_sync_ctx);
pool_unref(&sync_ctx->mail_keyword_pool);
pool_unref(&sync_ctx->saved_keywords_pool);
str_free(&sync_ctx->header);
str_free(&sync_ctx->from_line);
array_free(&sync_ctx->mails);
}
static int mbox_sync_int(struct mbox_mailbox *mbox, enum mbox_sync_flags flags,
unsigned int *lock_id)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mail_index_transaction *trans;
struct mbox_sync_context sync_ctx;
enum mail_index_sync_flags sync_flags;
int ret, changed;
bool delay_writes, readonly;
readonly = mbox_is_backend_readonly(mbox) ||
(flags & MBOX_SYNC_READONLY) != 0;
delay_writes = readonly ||
((flags & MBOX_SYNC_REWRITE) == 0 &&
mbox->storage->set->mbox_lazy_writes);
if (!mbox->storage->set->mbox_dirty_syncs &&
!mbox->storage->set->mbox_very_dirty_syncs)
flags |= MBOX_SYNC_UNDIRTY;
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0)
return -1;
}
if ((flags & MBOX_SYNC_HEADER) != 0 ||
(flags & MBOX_SYNC_FORCE_SYNC) != 0) {
if (mbox_sync_header_refresh(mbox) < 0)
return -1;
changed = 1;
} else {
bool leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
if ((changed = mbox_sync_has_changed(mbox, leave_dirty)) < 0)
return -1;
}
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
/* we just want to lock it for reading. if mbox hasn't been
modified don't do any syncing. */
if (!changed)
return 0;
/* have to sync to make sure offsets have stayed the same */
(void)mbox_unlock(mbox, *lock_id);
*lock_id = 0;
}
/* flush input streams' buffers */
if (mbox->mbox_stream != NULL)
i_stream_sync(mbox->mbox_stream);
if (mbox->mbox_file_stream != NULL)
i_stream_sync(mbox->mbox_file_stream);
again:
if (changed) {
/* we're most likely modifying the mbox while syncing, just
lock it for writing immediately. the mbox must be locked
before index syncing is started to avoid deadlocks, so we
don't have much choice either (well, easy ones anyway). */
int lock_type = readonly ? F_RDLCK : F_WRLCK;
if ((ret = mbox_lock(mbox, lock_type, lock_id)) <= 0) {
if (ret == 0 || lock_type == F_RDLCK)
return -1;
/* try as read-only */
if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0)
return -1;
mbox->backend_readonly = readonly = TRUE;
mbox->backend_readonly_set = TRUE;
delay_writes = TRUE;
}
}
sync_flags = index_storage_get_sync_flags(&mbox->box);
if ((flags & MBOX_SYNC_REWRITE) != 0)
sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
ret = mail_index_sync_begin(mbox->box.index, &index_sync_ctx,
&sync_view, &trans, sync_flags);
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(&mbox->box);
return ret;
}
if ((mbox->box.flags & MAILBOX_FLAG_KEEP_RECENT) == 0) {
/* see if we need to drop recent flags */
sync_ctx.hdr = mail_index_get_header(sync_view);
if (sync_ctx.hdr->first_recent_uid < sync_ctx.hdr->next_uid)
changed = 1;
}
if (!changed && !mail_index_sync_have_more(index_sync_ctx)) {
/* nothing to do */
nothing_to_do:
/* index may need to do internal syncing though, so commit
instead of rollbacking. */
if (mail_index_sync_commit(&index_sync_ctx) < 0) {
mail_storage_set_index_error(&mbox->box);
return -1;
}
return 0;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.mbox = mbox;
sync_ctx.keep_recent =
(mbox->box.flags & MAILBOX_FLAG_KEEP_RECENT) != 0;
sync_ctx.hdr = mail_index_get_header(sync_view);
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = trans;
sync_ctx.mail_keyword_pool =
pool_alloconly_create("mbox keywords", 512);
sync_ctx.saved_keywords_pool =
pool_alloconly_create("mbox saved keywords", 4096);
/* make sure we've read the latest keywords in index */
(void)mail_index_get_keywords(mbox->box.index);
i_array_init(&sync_ctx.mails, 64);
sync_ctx.flags = flags;
sync_ctx.readonly = readonly;
sync_ctx.delay_writes = delay_writes;
sync_ctx.sync_changes =
index_sync_changes_init(index_sync_ctx, sync_view, trans,
sync_ctx.delay_writes);
if (!changed && delay_writes) {
/* if we have only flag changes, we don't need to open the
mbox file */
bool expunged;
uint32_t uid;
mbox_sync_read_index_syncs(&sync_ctx, 1, &expunged);
uid = expunged ? 1 :
index_sync_changes_get_next_uid(sync_ctx.sync_changes);
if (uid == 0) {
sync_ctx.index_sync_ctx = NULL;
mbox_sync_context_free(&sync_ctx);
goto nothing_to_do;
}
}
if (*lock_id == 0) {
/* ok, we have something to do but no locks. we'll have to
restart syncing to avoid deadlocking. */
mbox_sync_context_free(&sync_ctx);
changed = 1;
goto again;
}
if (mbox_file_open_stream(mbox) < 0) {
mbox_sync_context_free(&sync_ctx);
return -1;
}
sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream;
sync_ctx.input = sync_ctx.mbox->mbox_stream;
sync_ctx.write_fd = sync_ctx.mbox->mbox_lock_type != F_WRLCK ? -1 :
sync_ctx.mbox->mbox_fd;
ret = mbox_sync_do(&sync_ctx, flags);
if (ret < 0)
mail_index_sync_rollback(&index_sync_ctx);
else if (mail_index_sync_commit(&index_sync_ctx) < 0) {
mail_storage_set_index_error(&mbox->box);
ret = -1;
}
sync_ctx.t = NULL;
sync_ctx.index_sync_ctx = NULL;
if (ret == 0 && mbox->mbox_fd != -1 && sync_ctx.keep_recent &&
!readonly) {
/* try to set atime back to its original value.
(it'll fail with EPERM for shared mailboxes where we aren't
the file's owner) */
struct utimbuf buf;
struct stat st;
if (fstat(mbox->mbox_fd, &st) < 0)
mbox_set_syscall_error(mbox, "fstat()");
else {
buf.modtime = st.st_mtime;
buf.actime = sync_ctx.orig_atime;
if (utime(mailbox_get_path(&mbox->box), &buf) < 0 &&
errno != EPERM)
mbox_set_syscall_error(mbox, "utime()");
}
}
i_assert(*lock_id != 0);
if (mbox->storage->storage.set->mail_nfs_storage &&
mbox->mbox_fd != -1) {
if (fdatasync(mbox->mbox_fd) < 0) {
mbox_set_syscall_error(mbox, "fdatasync()");
ret = -1;
}
}
mbox_sync_context_free(&sync_ctx);
return ret;
}
int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
{
unsigned int lock_id = 0;
int ret;
i_assert(mbox->mbox_lock_type != F_RDLCK ||
(flags & MBOX_SYNC_READONLY) != 0);
mbox->syncing = TRUE;
ret = mbox_sync_int(mbox, flags, &lock_id);
mbox->syncing = FALSE;
if (lock_id != 0) {
if (ret < 0) {
/* syncing failed, don't leave it locked */
(void)mbox_unlock(mbox, lock_id);
} else if ((flags & MBOX_SYNC_LOCK_READING) == 0) {
if (mbox_unlock(mbox, lock_id) < 0)
ret = -1;
} else if (mbox->mbox_lock_type != F_RDLCK) {
/* drop to read lock */
unsigned int read_lock_id = 0;
if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0)
ret = -1;
if (mbox_unlock(mbox, lock_id) < 0)
ret = -1;
}
}
if (mbox->box.v.sync_notify != NULL)
mbox->box.v.sync_notify(&mbox->box, 0, 0);
return ret;
}
struct mailbox_sync_context *
mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
enum mbox_sync_flags mbox_sync_flags = 0;
int ret = 0;
if (!box->opened) {
if (mailbox_open(box) < 0)
ret = -1;
}
if (index_mailbox_want_full_sync(&mbox->box, flags) && ret == 0) {
if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
!mbox->storage->set->mbox_very_dirty_syncs)
mbox_sync_flags |= MBOX_SYNC_UNDIRTY;
if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0)
mbox_sync_flags |= MBOX_SYNC_REWRITE;
if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) {
mbox_sync_flags |= MBOX_SYNC_UNDIRTY |
MBOX_SYNC_REWRITE | MBOX_SYNC_FORCE_SYNC;
}
ret = mbox_sync(mbox, mbox_sync_flags);
}
return index_mailbox_sync_init(box, flags, ret < 0);
}