mbox-sync.c revision 145ecefd208f5d7f6a52cb326a85f28d2154e217
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa/* Copyright (c) 2004-2008 Dovecot authors, see the included COPYING file */
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa/*
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa Modifying mbox can be slow, so we try to do it all at once minimizing the
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa required disk I/O. We may need to:
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa - Update message flags in Status, X-Status and X-Keywords headers
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa - Write missing X-UID and X-IMAPbase headers
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa - Write missing or broken Content-Length header if there's space
b1fe9054ad7c7192fe4c474363247dad15963e99Eugen Kuksa - Expunge specified messages
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa Here's how we do it:
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa - Start reading the mails from the beginning
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa of them, remember how much each message has and offset to beginning of the
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa padding
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa - If header needs to be rewritten and there's enough space, do it
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa - If we didn't have enough space, remember how much was missing
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa - Continue reading and counting the padding in each message. If available
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa padding is enough to rewrite all the previous messages needing it, do it
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa - When we encounter expunged message, treat all of it as padding and
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa rewrite previous messages if needed (and there's enough space).
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa Afterwards keep moving messages backwards to fill the expunged space.
366ce8d807067a97613cb23d49105d8a093c5015Eugen Kuksa Moving is done by rewriting each message's headers, with possibly adding
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa missing Content-Length header and padding. Message bodies are moved
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa without modifications.
366ce8d807067a97613cb23d49105d8a093c5015Eugen Kuksa - If we encounter end of file, grow the file and rewrite needed messages
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa - Rewriting is done by moving message body forward, rewriting message's
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa header and doing the same for previous message, until all of them are
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa rewritten.
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa*/
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "lib.h"
366ce8d807067a97613cb23d49105d8a093c5015Eugen Kuksa#include "ioloop.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "array.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "buffer.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "hostpid.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "istream.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "file-set-size.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "str.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "read-full.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "write-full.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "message-date.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "istream-raw-mbox.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "mbox-storage.h"
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa#include "index-sync-changes.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "mbox-from.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "mbox-file.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "mbox-lock.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include "mbox-sync-private.h"
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include <stddef.h>
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include <stdlib.h>
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include <utime.h>
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include <sys/stat.h>
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa/* The text below was taken exactly as c-client wrote it to my mailbox,
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa so it's probably copyrighted by University of Washington. */
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa#define PSEUDO_MESSAGE_BODY \
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa"This text is part of the internal format of your mail folder, and is not\n" \
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa"a real message. It is created automatically by the mail system software.\n" \
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa"If deleted, important folder data will be lost, and it will be re-created\n" \
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa"with the data reset to initial values.\n"
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksavoid mbox_sync_set_critical(struct mbox_sync_context *sync_ctx,
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa const char *fmt, ...)
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa{
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa va_list va;
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa sync_ctx->errors = TRUE;
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa if (sync_ctx->ext_modified) {
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa "mbox file %s was modified while we were syncing, "
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa "check your locking settings", sync_ctx->mbox->path);
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa }
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa va_start(va, fmt);
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa mail_storage_set_critical(&sync_ctx->mbox->storage->storage,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa "%s", t_strdup_vprintf(fmt, va));
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa va_end(va);
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa}
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksaint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa{
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa mbox_sync_set_critical(sync_ctx,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa "Unexpectedly lost From-line at offset %"PRIuUOFF_T
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa " from mbox file %s", from_offset,
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa sync_ctx->mbox->path);
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return -1;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa }
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return 0;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa}
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksavoid mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx)
284432981d641cf3d679841f75acbcf039d83062Eugen Kuksa{
284432981d641cf3d679841f75acbcf039d83062Eugen Kuksa struct stat st;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa /* Do this even if ext_modified is already set. Expunging code relies
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa on last_stat being updated. */
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (fstat(sync_ctx->write_fd, &st) < 0) {
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return;
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa }
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (st.st_size != sync_ctx->last_stat.st_size ||
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa (sync_ctx->last_stat.st_mtime != 0 &&
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa !CMP_ST_MTIME(&st, &sync_ctx->last_stat)))
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa sync_ctx->ext_modified = TRUE;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa sync_ctx->last_stat = st;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa}
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksavoid mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa{
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (dirty) {
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa /* just mark the stat as dirty. */
2dbc668d1e44c95db1857d3968bcde7517852beaEugen Kuksa sync_ctx->last_stat.st_mtime = 0;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa }
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa i_stream_sync(sync_ctx->input);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa}
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksastatic int
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksambox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa struct mbox_sync_mail_context *mail_ctx)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa{
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa /* get EOF */
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (istream_raw_mbox_is_eof(sync_ctx->input))
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return 0;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa p_clear(sync_ctx->mail_keyword_pool);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa memset(mail_ctx, 0, sizeof(*mail_ctx));
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_ctx->sync_ctx = sync_ctx;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_ctx->seq = ++sync_ctx->seq;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_ctx->header = sync_ctx->header;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_ctx->mail.from_offset =
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa istream_raw_mbox_get_start_offset(sync_ctx->input);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_ctx->mail.offset =
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa istream_raw_mbox_get_header_offset(sync_ctx->input);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa sync_ctx->input->eof);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
284432981d641cf3d679841f75acbcf039d83062Eugen Kuksa mail_ctx->mail.body_size =
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa istream_raw_mbox_get_body_size(sync_ctx->input,
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_ctx->content_length);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 &&
284432981d641cf3d679841f75acbcf039d83062Eugen Kuksa !mail_ctx->mail.pseudo) {
284432981d641cf3d679841f75acbcf039d83062Eugen Kuksa if (!sync_ctx->mbox->ibox.keep_recent) {
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa /* need to add 'O' flag to Status-header */
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_ctx->need_rewrite = TRUE;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa }
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_ctx->recent = TRUE;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa }
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return 1;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa}
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksastatic void mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa uint32_t uid, bool *sync_expunge_r)
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa{
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if (uid == 0 || sync_ctx->index_reset) {
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa /* nothing for this or the future ones */
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa uid = (uint32_t)-1;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa }
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa index_sync_changes_read(sync_ctx->sync_changes, uid, sync_expunge_r);
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (sync_ctx->mbox->mbox_readonly) {
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa /* we can't expunge anything from read-only mboxes */
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa *sync_expunge_r = FALSE;
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa }
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa}
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksastatic bool
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksambox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uint32_t uid, const struct mail_index_record **rec_r)
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa{
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa const struct mail_index_record *rec = NULL;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa uint32_t messages_count;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa bool ret = FALSE;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa if (sync_ctx->index_reset) {
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa *rec_r = NULL;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa return TRUE;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa }
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa messages_count =
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa mail_index_view_get_messages_count(sync_ctx->sync_view);
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa while (sync_ctx->idx_seq <= messages_count) {
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if (uid <= rec->uid)
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa break;
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa /* externally expunged message, remove from index */
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa sync_ctx->idx_seq++;
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa rec = NULL;
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa }
486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa if (rec == NULL && uid < sync_ctx->idx_next_uid) {
6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksa /* this UID was already in index and it was expunged */
6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksa mbox_sync_set_critical(sync_ctx,
6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksa "mbox sync: Expunged message reappeared in mailbox %s "
6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksa "(UID %u < %u, seq=%u, idx_msgs=%u)",
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa sync_ctx->mbox->path, uid, sync_ctx->idx_next_uid,
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa sync_ctx->seq, messages_count);
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa ret = FALSE; rec = NULL;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa } else if (rec != NULL && rec->uid != uid) {
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa /* new UID in the middle of the mailbox - shouldn't happen */
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa mbox_sync_set_critical(sync_ctx,
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa "mbox sync: UID inserted in the middle of mailbox %s "
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa "(%u > %u, seq=%u, idx_msgs=%u)", sync_ctx->mbox->path,
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa rec->uid, uid, sync_ctx->seq, messages_count);
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa ret = FALSE; rec = NULL;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa } else {
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa ret = TRUE;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa }
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa *rec_r = rec;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa return ret;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa}
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksastatic void mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa unsigned char hdr_md5_sum[],
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa const struct mail_index_record **rec_r)
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa{
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa const struct mail_index_record *rec = NULL;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa uint32_t messages_count;
28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa const void *data;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (sync_ctx->index_reset) {
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa *rec_r = NULL;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa }
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa messages_count =
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa mail_index_view_get_messages_count(sync_ctx->sync_view);
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa while (sync_ctx->idx_seq <= messages_count) {
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa mail_index_lookup_ext(sync_ctx->sync_view,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa sync_ctx->idx_seq,
0b99be5ced371cca00283694e1bd53a8ac0d7b5dEugen Kuksa sync_ctx->mbox->ibox.md5hdr_ext_idx,
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa &data, NULL);
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0)
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa break;
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa /* externally expunged message, remove from index */
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa sync_ctx->idx_seq++;
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa rec = NULL;
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa }
df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa *rec_r = rec;
b1fe9054ad7c7192fe4c474363247dad15963e99Eugen Kuksa}
366ce8d807067a97613cb23d49105d8a093c5015Eugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksastatic void
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksambox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa struct mbox_sync_mail *mail,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa bool nocheck)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa{
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa const void *data;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uint64_t offset;
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa if (!nocheck) {
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa /* see if from_offset needs updating */
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa sync_ctx->mbox->mbox_ext_idx,
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa &data, NULL);
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa if (data != NULL &&
5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa *((const uint64_t *)data) == mail->from_offset)
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return;
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa }
923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa offset = mail->from_offset;
daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->mbox->mbox_ext_idx, &offset, NULL);
}
static void
mbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
struct mail_keywords *keywords;
keywords = !array_is_created(&mail_ctx->mail.keywords) ?
mail_index_keywords_create(sync_ctx->mbox->ibox.index, NULL) :
mail_index_keywords_create_from_indexes(
sync_ctx->mbox->ibox.index, &mail_ctx->mail.keywords);
mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, keywords);
mail_index_keywords_free(&keywords);
}
static void
mbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
const void *ext_data;
mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
sync_ctx->mbox->ibox.md5hdr_ext_idx,
&ext_data, NULL);
if (ext_data == NULL ||
memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) {
mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->mbox->ibox.md5hdr_ext_idx,
mail_ctx->hdr_md5_sum, NULL);
}
}
static void mbox_sync_get_dirty_flags(struct mbox_sync_mail_context *mail_ctx,
const struct mail_index_record *rec)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
ARRAY_TYPE(keyword_indexes) idx_keywords;
uint8_t idx_flags, mbox_flags;
/* default to undirtying the message. it gets added back if
flags/keywords don't match what is in the index. */
mail_ctx->mail.flags &= ~MAIL_INDEX_MAIL_FLAG_DIRTY;
/* replace flags */
idx_flags = rec->flags & MAIL_FLAGS_NONRECENT;
mbox_flags = mail_ctx->mail.flags & MAIL_FLAGS_NONRECENT;
if (idx_flags != mbox_flags) {
mail_ctx->need_rewrite = TRUE;
mail_ctx->mail.flags = (mail_ctx->mail.flags & MAIL_RECENT) |
idx_flags | MAIL_INDEX_MAIL_FLAG_DIRTY;
}
/* replace keywords */
t_array_init(&idx_keywords, 32);
mail_index_lookup_keywords(sync_ctx->sync_view, sync_ctx->idx_seq,
&idx_keywords);
if (!index_keyword_array_cmp(&idx_keywords, &mail_ctx->mail.keywords)) {
mail_ctx->need_rewrite = TRUE;
mail_ctx->mail.flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
if (!array_is_created(&mail_ctx->mail.keywords)) {
p_array_init(&mail_ctx->mail.keywords,
sync_ctx->mail_keyword_pool,
array_count(&idx_keywords));
}
array_clear(&mail_ctx->mail.keywords);
array_append_array(&mail_ctx->mail.keywords, &idx_keywords);
}
}
static void mbox_sync_update_flags(struct mbox_sync_mail_context *mail_ctx,
const struct mail_index_record *rec)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
struct mailbox *box = &sync_ctx->mbox->ibox.box;
struct mbox_sync_mail *mail = &mail_ctx->mail;
enum mail_index_sync_type sync_type;
ARRAY_TYPE(keyword_indexes) orig_keywords = ARRAY_INIT;
uint8_t flags, orig_flags;
if (rec != NULL) {
if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* flags and keywords are dirty. replace the current
ones from the flags in index file. */
mbox_sync_get_dirty_flags(mail_ctx, rec);
}
}
flags = orig_flags = mail->flags & MAIL_FLAGS_NONRECENT;
if (array_is_created(&mail->keywords)) {
t_array_init(&orig_keywords, 32);
array_append_array(&orig_keywords, &mail->keywords);
}
/* apply new changes */
index_sync_changes_apply(sync_ctx->sync_changes,
sync_ctx->mail_keyword_pool,
&flags, &mail->keywords, &sync_type);
if (flags != orig_flags ||
!index_keyword_array_cmp(&mail->keywords, &orig_keywords)) {
mail_ctx->need_rewrite = TRUE;
mail->flags = flags | (mail->flags & MAIL_RECENT) |
MAIL_INDEX_MAIL_FLAG_DIRTY;
}
if (sync_type != 0 && box->v.sync_notify != NULL) {
box->v.sync_notify(box, rec->uid,
index_sync_type_convert(sync_type));
}
}
static void mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
const struct mail_index_record *rec)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
struct mbox_sync_mail *mail = &mail_ctx->mail;
ARRAY_TYPE(keyword_indexes) idx_keywords;
uint8_t mbox_flags;
mbox_flags = mail->flags & ~MAIL_RECENT;
if (!sync_ctx->delay_writes) {
/* changes are written to the mbox file */
mbox_flags &= ~MAIL_INDEX_MAIL_FLAG_DIRTY;
} else if (mail_ctx->need_rewrite) {
/* make sure this message gets written later */
mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
}
if (rec == NULL) {
/* new message */
mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags);
mbox_sync_update_index_keywords(mail_ctx);
if (sync_ctx->mbox->mbox_save_md5 != 0) {
mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->mbox->ibox.md5hdr_ext_idx,
mail_ctx->hdr_md5_sum, NULL);
}
} else {
if ((rec->flags & MAIL_FLAGS_NONRECENT) !=
(mbox_flags & MAIL_FLAGS_NONRECENT)) {
/* flags other than recent/dirty have changed */
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags);
} else if (((rec->flags ^ mbox_flags) &
MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* only dirty flag state changed */
bool dirty;
dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
dirty ? MODIFY_ADD : MODIFY_REMOVE,
(enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY);
}
/* see if keywords changed */
t_array_init(&idx_keywords, 32);
mail_index_lookup_keywords(sync_ctx->sync_view,
sync_ctx->idx_seq, &idx_keywords);
if (!index_keyword_array_cmp(&idx_keywords, &mail->keywords))
mbox_sync_update_index_keywords(mail_ctx);
/* see if we need to update md5 sum. */
if (sync_ctx->mbox->mbox_save_md5 != 0)
mbox_sync_update_md5_if_changed(mail_ctx);
}
if (!mail_ctx->recent) {
/* Mail has "Status: O" header. No messages before this
can be recent. */
sync_ctx->last_nonrecent_uid = mail->uid;
}
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
bool nocheck = rec == NULL || sync_ctx->expunged_space > 0;
mbox_sync_update_from_offset(sync_ctx, mail, nocheck);
}
}
static int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
{
struct istream *input = ctx->sync_ctx->file_input;
const unsigned char *data;
size_t size, from_line_size;
buffer_set_used_size(ctx->sync_ctx->from_line, 0);
from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
i_stream_seek(input, ctx->mail.from_offset);
for (;;) {
data = i_stream_get_data(input, &size);
if (size >= from_line_size)
size = from_line_size;
buffer_append(ctx->sync_ctx->from_line, data, size);
i_stream_skip(input, size);
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx)
{
unsigned char buf[10];
const char *str;
uint32_t uid_last;
unsigned int i;
int ret;
i_assert(sync_ctx->base_uid_last_offset != 0);
/* first check that the 10 bytes are there and they're exactly as
expected. just an extra safety check to make sure we never write
to wrong location in the mbox file. */
ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf),
sync_ctx->base_uid_last_offset);
if (ret < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "pread_full()");
return -1;
}
if (ret == 0) {
mbox_sync_set_critical(sync_ctx,
"X-IMAPbase uid-last unexpectedly points outside "
"mbox file %s", sync_ctx->mbox->path);
return -1;
}
for (i = 0, uid_last = 0; i < sizeof(buf); i++) {
if (buf[i] < '0' || buf[i] > '9') {
uid_last = (uint32_t)-1;
break;
}
uid_last = uid_last * 10 + (buf[i] - '0');
}
if (uid_last != sync_ctx->base_uid_last) {
mbox_sync_set_critical(sync_ctx,
"X-IMAPbase uid-last unexpectedly lost in mbox file %s",
sync_ctx->mbox->path);
return -1;
}
/* and write it */
str = t_strdup_printf("%010u", sync_ctx->next_uid - 1);
if (pwrite_full(sync_ctx->write_fd, str, 10,
sync_ctx->base_uid_last_offset) < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
return -1;
}
mbox_sync_file_updated(sync_ctx, FALSE);
sync_ctx->base_uid_last = sync_ctx->next_uid - 1;
return 0;
}
static int
mbox_write_from_line(struct mbox_sync_mail_context *ctx)
{
string_t *str = ctx->sync_ctx->from_line;
if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str),
ctx->mail.from_offset) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()");
return -1;
}
mbox_sync_file_updated(ctx->sync_ctx, FALSE);
return 0;
}
static void update_from_offsets(struct mbox_sync_context *sync_ctx)
{
const struct mbox_sync_mail *mails;
unsigned int i, count;
uint32_t ext_idx;
uint64_t offset;
ext_idx = sync_ctx->mbox->mbox_ext_idx;
mails = array_get(&sync_ctx->mails, &count);
for (i = 0; i < count; i++) {
if (mails[i].idx_seq == 0 || mails[i].expunged)
continue;
sync_ctx->moved_offsets = TRUE;
offset = mails[i].from_offset;
mail_index_update_ext(sync_ctx->t, mails[i].idx_seq,
ext_idx, &offset, NULL);
}
}
static void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
struct mailbox *box = &sync_ctx->mbox->ibox.box;
if (box->v.sync_notify != NULL) {
box->v.sync_notify(box, mail_ctx->mail.uid,
MAILBOX_SYNC_TYPE_EXPUNGE);
}
mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq);
mail_ctx->mail.expunged = TRUE;
mail_ctx->mail.offset = mail_ctx->mail.from_offset;
mail_ctx->mail.space =
mail_ctx->body_offset - mail_ctx->mail.from_offset +
mail_ctx->mail.body_size;
mail_ctx->mail.body_size = 0;
mail_ctx->mail.uid = 0;
if (sync_ctx->seq == 1) {
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
mail_ctx->mail.space++;
if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) {
mail_ctx->mail.space++;
sync_ctx->first_mail_crlf_expunged = TRUE;
}
/* uid-last offset is invalid now */
sync_ctx->base_uid_last_offset = 0;
}
sync_ctx->expunged_space += mail_ctx->mail.space;
}
static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t orig_from_offset;
off_t move_diff;
int ret;
if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
/* move the header backwards to fill expunged space */
move_diff = -sync_ctx->expunged_space;
orig_from_offset = mail_ctx->mail.from_offset;
if (sync_ctx->dest_first_mail) {
/* we're moving this mail to beginning of file.
skip the initial \n (it's already counted in
expunged_space) */
mail_ctx->mail.from_offset++;
if (sync_ctx->first_mail_crlf_expunged)
mail_ctx->mail.from_offset++;
}
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
mbox_sync_update_header(mail_ctx);
ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
if (ret < 0)
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
i_assert(move_diff > 0 ||
(off_t)mail_ctx->mail.from_offset >=
-move_diff);
mail_ctx->mail.from_offset += move_diff;
mail_ctx->mail.offset += move_diff;
if (mbox_write_from_line(mail_ctx) < 0)
return -1;
} else {
if (sync_ctx->dest_first_mail) {
/* didn't have enough space, move the offset
back so seeking into it doesn't fail */
mail_ctx->mail.from_offset = orig_from_offset;
}
}
} else if (mail_ctx->need_rewrite) {
mbox_sync_update_header(mail_ctx);
if (sync_ctx->delay_writes) {
/* mark it dirty and do it later */
mail_ctx->dirty = TRUE;
return 0;
}
if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
return -1;
} else {
/* nothing to do */
return 0;
}
if (ret == 0 && sync_ctx->need_space_seq == 0) {
/* first mail with no space to write it */
sync_ctx->need_space_seq = sync_ctx->seq;
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
memset(&mail, 0, sizeof(mail));
mail.expunged = TRUE;
mail.offset = mail.from_offset =
(sync_ctx->dest_first_mail ? 1 : 0) +
mail_ctx->mail.from_offset -
sync_ctx->expunged_space;
mail.space = sync_ctx->expunged_space;
sync_ctx->space_diff = sync_ctx->expunged_space;
sync_ctx->expunged_space = 0;
i_assert(sync_ctx->space_diff < -mail_ctx->mail.space);
sync_ctx->need_space_seq--;
array_append(&sync_ctx->mails, &mail, 1);
}
}
return 0;
}
static int
mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t end_offset, move_diff, extra_space, needed_space;
uint32_t last_seq;
ARRAY_TYPE(keyword_indexes) keywords_copy;
i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 ||
mail_ctx->mail.offset == mail_ctx->hdr_offset);
if (array_is_created(&mail_ctx->mail.keywords)) {
/* mail's keywords are allocated from a pool that's cleared
for each mail. we'll need to copy it to something more
permanent. */
p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool,
array_count(&mail_ctx->mail.keywords));
array_append_array(&keywords_copy, &mail_ctx->mail.keywords);
mail_ctx->mail.keywords = keywords_copy;
}
array_append(&sync_ctx->mails, &mail_ctx->mail, 1);
sync_ctx->space_diff += mail_ctx->mail.space;
if (sync_ctx->space_diff < 0) {
if (sync_ctx->expunged_space > 0) {
i_assert(sync_ctx->expunged_space ==
mail_ctx->mail.space);
sync_ctx->expunged_space = 0;
}
return 0;
}
/* we have enough space now */
if (mail_ctx->mail.uid == 0) {
/* this message was expunged. fill more or less of the space.
space_diff now consists of a negative "bytes needed" sum,
plus the expunged space of this message. so it contains how
many bytes of _extra_ space we have. */
i_assert(mail_ctx->mail.space >= sync_ctx->space_diff);
extra_space = MBOX_HEADER_PADDING *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
/* don't waste too much on padding */
move_diff = needed_space + extra_space;
sync_ctx->expunged_space =
mail_ctx->mail.space - move_diff;
} else {
move_diff = mail_ctx->mail.space;
extra_space = sync_ctx->space_diff;
sync_ctx->expunged_space = 0;
}
last_seq = sync_ctx->seq - 1;
array_delete(&sync_ctx->mails,
array_count(&sync_ctx->mails) - 1, 1);
end_offset = mail_ctx->mail.from_offset;
} else {
/* this message gave enough space from headers. rewriting stops
at the end of this message's headers. */
sync_ctx->expunged_space = 0;
last_seq = sync_ctx->seq;
end_offset = mail_ctx->body_offset;
move_diff = 0;
extra_space = sync_ctx->space_diff;
}
mbox_sync_file_update_ext_modified(sync_ctx);
if (mbox_sync_rewrite(sync_ctx,
last_seq == sync_ctx->seq ? mail_ctx : NULL,
end_offset, move_diff, extra_space,
sync_ctx->need_space_seq, last_seq) < 0)
return -1;
update_from_offsets(sync_ctx);
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
memset(mail_ctx, 0, sizeof(*mail_ctx));
sync_ctx->need_space_seq = 0;
sync_ctx->space_diff = 0;
array_clear(&sync_ctx->mails);
p_clear(sync_ctx->saved_keywords_pool);
return 0;
}
static int
mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
{
struct mbox_mailbox *mbox = sync_ctx->mbox;
uoff_t old_offset;
uint32_t uid;
int ret;
bool deleted;
if (seq == 0) {
if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) {
mbox->invalid_mbox_file = TRUE;
mail_storage_set_error(&mbox->storage->storage,
MAIL_ERROR_NOTPOSSIBLE,
"Mailbox isn't a valid mbox file");
return -1;
}
seq++;
} else {
old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted);
if (ret < 0) {
if (deleted) {
mbox_sync_set_critical(sync_ctx,
"Message was expunged unexpectedly "
"in mbox file %s", mbox->path);
}
return -1;
}
if (ret == 0) {
if (istream_raw_mbox_seek(mbox->mbox_stream,
old_offset) < 0) {
mbox_sync_set_critical(sync_ctx,
"Error seeking back to original "
"offset %s in mbox file %s",
dec2str(old_offset), mbox->path);
return -1;
}
return 0;
}
}
if (seq <= 1)
uid = 0;
else
mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid);
sync_ctx->prev_msg_uid = uid;
/* set to -1, since it's always increased later */
sync_ctx->seq = seq-1;
if (sync_ctx->seq == 0 &&
istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
/* this mbox has pseudo mail which contains the X-IMAP header */
sync_ctx->seq++;
}
sync_ctx->idx_seq = seq;
sync_ctx->dest_first_mail = sync_ctx->seq == 0;
(void)istream_raw_mbox_get_body_offset(sync_ctx->input);
return 1;
}
static int
mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
{
struct mail_index_view *sync_view = sync_ctx->sync_view;
uint32_t seq1, seq2;
const struct stat *st;
i_assert(!sync_ctx->index_reset);
if (!mail_index_lookup_seq_range(sync_view, uid, (uint32_t)-1,
&seq1, &seq2)) {
/* doesn't exist anymore, seek to end of file */
st = i_stream_stat(sync_ctx->file_input, TRUE);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox,
"i_stream_stat()");
return -1;
}
if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream,
st->st_size) < 0) {
mbox_sync_set_critical(sync_ctx,
"Error seeking to end of mbox file %s",
sync_ctx->mbox->path);
return -1;
}
sync_ctx->idx_seq =
mail_index_view_get_messages_count(sync_view) + 1;
return 1;
}
return mbox_sync_seek_to_seq(sync_ctx, seq1);
}
static int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx,
uint32_t next_uid, bool *partial,
bool *skipped_mails)
{
uint32_t messages_count, uid;
int ret;
i_assert(!sync_ctx->index_reset);
/* delete sync records up to next message. so if there's still
something left in array, it means the next message needs modifying */
index_sync_changes_delete_to(sync_ctx->sync_changes, next_uid);
if (index_sync_changes_have(sync_ctx->sync_changes))
return 1;
if (sync_ctx->hdr->first_recent_uid <= next_uid &&
!sync_ctx->mbox->ibox.keep_recent) {
/* we'll need to rewrite Status: O headers */
return 1;
}
uid = index_sync_changes_get_next_uid(sync_ctx->sync_changes);
if (sync_ctx->hdr->first_recent_uid < sync_ctx->hdr->next_uid &&
(uid > sync_ctx->hdr->first_recent_uid || uid == 0) &&
!sync_ctx->mbox->ibox.keep_recent) {
/* we'll need to rewrite Status: O headers */
uid = sync_ctx->hdr->first_recent_uid;
}
if (uid != 0) {
/* we can skip forward to next record which needs updating. */
if (uid != next_uid) {
*skipped_mails = TRUE;
next_uid = uid;
}
ret = mbox_sync_seek_to_uid(sync_ctx, next_uid);
} else {
/* if there's no sync records left, we can stop. except if
this is a dirty sync, check if there are new messages. */
if (!sync_ctx->mbox->mbox_sync_dirty)
return 0;
messages_count =
mail_index_view_get_messages_count(sync_ctx->sync_view);
if (sync_ctx->seq + 1 != messages_count) {
ret = mbox_sync_seek_to_seq(sync_ctx, messages_count);
*skipped_mails = TRUE;
} else {
ret = 1;
}
*partial = FALSE;
}
if (ret == 0) {
/* seek failed because the offset is dirty. just ignore and
continue from where we are now. */
*partial = FALSE;
ret = 1;
}
return ret;
}
static bool mbox_sync_uidvalidity_changed(struct mbox_sync_context *sync_ctx)
{
if (sync_ctx->base_uid_validity != 0 &&
sync_ctx->hdr->uid_validity != 0 &&
sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s",
sync_ctx->hdr->uid_validity,
sync_ctx->base_uid_validity,
sync_ctx->mbox->path);
sync_ctx->index_reset = TRUE;
return TRUE;
}
return FALSE;
}
static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
bool partial)
{
const struct mail_index_record *rec;
uint32_t uid, messages_count;
uoff_t offset;
int ret;
bool expunged, skipped_mails, uids_broken;
messages_count =
mail_index_view_get_messages_count(sync_ctx->sync_view);
/* always start from first message so we can read X-IMAP or
X-IMAPbase header */
ret = mbox_sync_seek_to_seq(sync_ctx, 0);
if (ret <= 0)
return ret;
if (sync_ctx->renumber_uids) {
/* expunge everything */
while (sync_ctx->idx_seq <= messages_count) {
mail_index_expunge(sync_ctx->t,
sync_ctx->idx_seq++);
}
}
skipped_mails = uids_broken = FALSE;
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
uid = mail_ctx->mail.uid;
if (mail_ctx->seq == 1) {
if (mbox_sync_uidvalidity_changed(sync_ctx)) {
sync_ctx->mbox->mbox_sync_dirty = TRUE;
return 0;
}
}
if (mail_ctx->mail.uid_broken && partial) {
/* UID ordering problems, resync everything to make
sure we get everything right */
if (sync_ctx->mbox->mbox_sync_dirty)
return 0;
mbox_sync_set_critical(sync_ctx,
"UIDs broken with partial sync in mbox file %s",
sync_ctx->mbox->path);
sync_ctx->mbox->mbox_sync_dirty = TRUE;
return 0;
}
if (mail_ctx->mail.uid_broken)
uids_broken = TRUE;
if (mail_ctx->mail.pseudo)
uid = 0;
rec = NULL; ret = 1;
if (uid != 0) {
if (!mbox_sync_read_index_rec(sync_ctx, uid, &rec))
ret = 0;
}
if (ret == 0) {
/* UID found but it's broken */
uid = 0;
} else if (uid == 0 &&
!mail_ctx->mail.pseudo &&
(sync_ctx->delay_writes ||
sync_ctx->idx_seq <= messages_count)) {
/* If we can't use/store X-UID header, use MD5 sum.
Also check for existing MD5 sums when we're actually
able to write X-UIDs. */
sync_ctx->mbox->mbox_save_md5 = TRUE;
mbox_sync_find_index_md5(sync_ctx,
mail_ctx->hdr_md5_sum, &rec);
if (rec != NULL)
uid = mail_ctx->mail.uid = rec->uid;
}
/* get all sync records related to this message. with pseudo
message just get the first sync record so we can jump to
it with partial seeking. */
mbox_sync_read_index_syncs(sync_ctx,
mail_ctx->mail.pseudo ? 1 : uid,
&expunged);
if (mail_ctx->mail.pseudo) {
/* if it was set, it was for the next message */
expunged = FALSE;
} else {
if (rec == NULL) {
/* message wasn't found from index. we have to
read everything from now on, no skipping */
partial = FALSE;
}
}
if (uid == 0 && !mail_ctx->mail.pseudo) {
/* missing/broken X-UID. all the rest of the mails
need new UIDs. */
while (sync_ctx->idx_seq <= messages_count) {
mail_index_expunge(sync_ctx->t,
sync_ctx->idx_seq++);
}
if (sync_ctx->next_uid == (uint32_t)-1) {
/* oh no, we're out of UIDs. this shouldn't
happen normally, so just try to get it fixed
without crashing. */
mail_storage_set_critical(
&sync_ctx->mbox->storage->storage,
"Out of UIDs, renumbering them in mbox "
"file %s", sync_ctx->mbox->path);
sync_ctx->renumber_uids = TRUE;
return 0;
}
mail_ctx->need_rewrite = TRUE;
mail_ctx->mail.uid = sync_ctx->next_uid++;
}
sync_ctx->prev_msg_uid = mail_ctx->mail.uid;
if (!mail_ctx->mail.pseudo)
mail_ctx->mail.idx_seq = sync_ctx->idx_seq;
if (!expunged) {
if (!mail_ctx->mail.pseudo) T_BEGIN {
mbox_sync_update_flags(mail_ctx, rec);
} T_END;
if (mbox_sync_handle_header(mail_ctx) < 0)
return -1;
sync_ctx->dest_first_mail = FALSE;
} else {
mbox_sync_handle_expunge(mail_ctx);
}
if (!mail_ctx->mail.pseudo) {
if (!expunged) T_BEGIN {
mbox_sync_update_index(mail_ctx, rec);
} T_END;
sync_ctx->idx_seq++;
}
istream_raw_mbox_next(sync_ctx->input,
mail_ctx->mail.body_size);
offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
mbox_sync_file_update_ext_modified(sync_ctx);
if (mbox_move(sync_ctx,
mail_ctx->body_offset -
sync_ctx->expunged_space,
mail_ctx->body_offset,
mail_ctx->mail.body_size) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
}
} else if (partial) {
ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1,
&partial,
&skipped_mails);
if (ret <= 0) {
if (ret < 0)
return -1;
break;
}
}
}
if (istream_raw_mbox_is_eof(sync_ctx->input)) {
/* rest of the messages in index don't exist -> expunge them */
while (sync_ctx->idx_seq <= messages_count)
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
}
if (!skipped_mails)
sync_ctx->mbox->mbox_sync_dirty = FALSE;
if (uids_broken && sync_ctx->delay_writes) {
/* once we get around to writing the changes, we'll need to do
a full sync to avoid the "UIDs broken in partial sync"
error */
sync_ctx->mbox->mbox_sync_dirty = TRUE;
}
return 1;
}
static int mbox_write_pseudo(struct mbox_sync_context *sync_ctx)
{
string_t *str;
unsigned int uid_validity;
i_assert(sync_ctx->write_fd != -1);
uid_validity = sync_ctx->base_uid_validity != 0 ?
sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity;
i_assert(uid_validity != 0);
str = t_str_new(1024);
str_printfa(str, "%sDate: %s\n"
"From: Mail System Internal Data <MAILER-DAEMON@%s>\n"
"Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA"
"\nMessage-ID: <%s@%s>\n"
"X-IMAP: %u %010u\n"
"Status: RO\n"
"\n"
PSEUDO_MESSAGE_BODY
"\n",
mbox_from_create("MAILER_DAEMON", ioloop_time),
message_date_create(ioloop_time),
my_hostname, dec2str(ioloop_time), my_hostname,
uid_validity, sync_ctx->next_uid-1);
if (pwrite_full(sync_ctx->write_fd,
str_data(str), str_len(str), 0) < 0) {
if (!ENOSPACE(errno)) {
mbox_set_syscall_error(sync_ctx->mbox,
"pwrite_full()");
return -1;
}
/* out of disk space, truncate to empty */
if (ftruncate(sync_ctx->write_fd, 0) < 0)
mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
}
sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */
sync_ctx->base_uid_last = sync_ctx->next_uid-1;
return 0;
}
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
const struct stat *st;
uoff_t file_size, offset, padding, trailer_size;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
i_assert(sync_ctx->expunged_space == 0);
return 0;
}
st = i_stream_stat(sync_ctx->file_input, TRUE);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
return -1;
}
if (st->st_size < 0) {
/* Not a file - allow anyway */
return 0;
}
file_size = st->st_size;
if (file_size < sync_ctx->file_input->v_offset) {
mbox_sync_set_critical(sync_ctx,
"file size unexpectedly shrank in mbox file %s "
"(%"PRIuUOFF_T" vs %"PRIuUOFF_T")",
sync_ctx->mbox->path, file_size,
sync_ctx->file_input->v_offset);
return -1;
}
trailer_size = file_size - sync_ctx->file_input->v_offset;
i_assert(trailer_size <= 2);
if (sync_ctx->need_space_seq != 0) {
i_assert(sync_ctx->write_fd != -1);
i_assert(sync_ctx->space_diff < 0);
padding = MBOX_HEADER_PADDING *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
sync_ctx->space_diff -= padding;
i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff);
sync_ctx->space_diff += sync_ctx->expunged_space;
sync_ctx->expunged_space = 0;
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
i_assert(sync_ctx->space_diff < 0);
if (file_set_size(sync_ctx->write_fd,
file_size + -sync_ctx->space_diff) < 0) {
mbox_set_syscall_error(sync_ctx->mbox,
"file_set_size()");
if (ftruncate(sync_ctx->write_fd, file_size) < 0) {
mbox_set_syscall_error(sync_ctx->mbox,
"ftruncate()");
}
return -1;
}
mbox_sync_file_updated(sync_ctx, FALSE);
if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size,
-sync_ctx->space_diff, padding,
sync_ctx->need_space_seq,
sync_ctx->seq) < 0)
return -1;
update_from_offsets(sync_ctx);
sync_ctx->need_space_seq = 0;
array_clear(&sync_ctx->mails);
p_clear(sync_ctx->saved_keywords_pool);
}
if (sync_ctx->expunged_space > 0) {
i_assert(sync_ctx->write_fd != -1);
mbox_sync_file_update_ext_modified(sync_ctx);
/* copy trailer, then truncate the file */
file_size = sync_ctx->last_stat.st_size;
if (file_size == (uoff_t)sync_ctx->expunged_space) {
/* everything deleted, the trailer_size still contains
the \n trailer though */
trailer_size = 0;
}
i_assert(file_size >= sync_ctx->expunged_space + trailer_size);
offset = file_size - sync_ctx->expunged_space - trailer_size;
i_assert(offset == 0 || offset > 31);
if (mbox_move(sync_ctx, offset,
offset + sync_ctx->expunged_space,
trailer_size) < 0)
return -1;
if (ftruncate(sync_ctx->write_fd,
offset + trailer_size) < 0) {
mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
return -1;
}
if (offset == 0) {
if (mbox_write_pseudo(sync_ctx) < 0)
return -1;
}
sync_ctx->expunged_space = 0;
mbox_sync_file_updated(sync_ctx, FALSE);
}
return 0;
}
static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
{
struct mail_index_view *view;
const struct stat *st;
uint32_t first_recent_uid, seq, seq2;
st = i_stream_stat(sync_ctx->file_input, FALSE);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
return -1;
}
if (sync_ctx->moved_offsets &&
((uint64_t)st->st_size == sync_ctx->hdr->sync_size ||
(uint64_t)st->st_size == sync_ctx->orig_size)) {
/* We moved messages inside the mbox file without changing
the file's size. If mtime doesn't change, another process
not using the same index file as us can't know that the file
was changed. So make sure the mtime changes. This should
happen rarely enough that the sleeping doesn't become a
performance problem.
Note that to do this perfectly safe we should do this wait
whenever mails are moved or expunged, regardless of whether
the file's size changed. That however could become a
performance problem and the consequences of being wrong are
quite minimal (an extra logged error message). */
while (sync_ctx->orig_mtime == st->st_mtime) {
usleep(500000);
if (utime(sync_ctx->mbox->path, NULL) < 0) {
mbox_set_syscall_error(sync_ctx->mbox,
"utime()");
return -1;
}
st = i_stream_stat(sync_ctx->file_input, FALSE);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox,
"i_stream_stat()");
return -1;
}
}
}
/* only reason not to have UID validity at this point is if the file
is entirely empty. In that case just make up a new one if needed. */
i_assert(sync_ctx->base_uid_validity != 0 || st->st_size <= 0);
if (sync_ctx->base_uid_validity == 0) {
sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ?
sync_ctx->hdr->uid_validity :
I_MIN((unsigned int)ioloop_time, 1);
}
if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, uid_validity),
&sync_ctx->base_uid_validity,
sizeof(sync_ctx->base_uid_validity), TRUE);
}
if (istream_raw_mbox_is_eof(sync_ctx->input) &&
sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
i_assert(sync_ctx->next_uid != 0);
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, next_uid),
&sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
}
if ((uint32_t)st->st_mtime != sync_ctx->hdr->sync_stamp &&
!sync_ctx->mbox->mbox_sync_dirty) {
uint32_t sync_stamp = st->st_mtime;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_stamp),
&sync_stamp, sizeof(sync_stamp), TRUE);
}
if ((uint64_t)st->st_size != sync_ctx->hdr->sync_size &&
!sync_ctx->mbox->mbox_sync_dirty) {
uint64_t sync_size = st->st_size;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_size),
&sync_size, sizeof(sync_size), TRUE);
}
if (sync_ctx->last_nonrecent_uid < sync_ctx->hdr->first_recent_uid) {
/* other sessions have already marked more messages as
recent. */
sync_ctx->last_nonrecent_uid =
sync_ctx->hdr->first_recent_uid - 1;
}
/* mark recent messages */
view = mail_index_transaction_open_updated_view(sync_ctx->t);
if (mail_index_lookup_seq_range(view, sync_ctx->last_nonrecent_uid + 1,
(uint32_t)-1, &seq, &seq2)) {
index_mailbox_set_recent_seq(&sync_ctx->mbox->ibox,
view, seq, seq2);
}
mail_index_view_close(&view);
first_recent_uid = !sync_ctx->mbox->ibox.keep_recent ?
sync_ctx->next_uid : sync_ctx->last_nonrecent_uid + 1;
if (sync_ctx->hdr->first_recent_uid < first_recent_uid) {
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, first_recent_uid),
&first_recent_uid, sizeof(first_recent_uid), FALSE);
}
sync_ctx->mbox->mbox_dirty_stamp = st->st_mtime;
sync_ctx->mbox->mbox_dirty_size = st->st_size;
return 0;
}
static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->base_uid_last_offset = 0;
array_clear(&sync_ctx->mails);
p_clear(sync_ctx->saved_keywords_pool);
index_sync_changes_reset(sync_ctx->sync_changes);
mail_index_sync_reset(sync_ctx->index_sync_ctx);
mail_index_transaction_reset(sync_ctx->t);
if (sync_ctx->index_reset) {
mail_index_reset(sync_ctx->t);
sync_ctx->reset_hdr.next_uid = 1;
sync_ctx->hdr = &sync_ctx->reset_hdr;
index_mailbox_reset_uidvalidity(&sync_ctx->mbox->ibox);
}
sync_ctx->prev_msg_uid = 0;
sync_ctx->next_uid = sync_ctx->hdr->next_uid;
sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
sync_ctx->seq = 0;
sync_ctx->idx_seq = 1;
sync_ctx->need_space_seq = 0;
sync_ctx->expunged_space = 0;
sync_ctx->space_diff = 0;
sync_ctx->dest_first_mail = TRUE;
sync_ctx->ext_modified = FALSE;
sync_ctx->errors = FALSE;
}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx,
enum mbox_sync_flags flags)
{
struct mbox_sync_mail_context mail_ctx;
const struct stat *st;
unsigned int i;
int ret, partial;
st = i_stream_stat(sync_ctx->file_input, FALSE);
if (st == NULL) {
mbox_set_syscall_error(sync_ctx->mbox, "i_stream_stat()");
return -1;
}
sync_ctx->last_stat = *st;
sync_ctx->orig_size = st->st_size;
sync_ctx->orig_atime = st->st_atime;
sync_ctx->orig_mtime = st->st_mtime;
if ((flags & MBOX_SYNC_FORCE_SYNC) != 0) {
/* forcing a full sync. assume file has changed. */
partial = FALSE;
sync_ctx->mbox->mbox_sync_dirty = TRUE;
} else if ((uint32_t)st->st_mtime == sync_ctx->hdr->sync_stamp &&
(uint64_t)st->st_size == sync_ctx->hdr->sync_size) {
/* file is fully synced */
partial = TRUE;
sync_ctx->mbox->mbox_sync_dirty = FALSE;
} else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
(uint64_t)st->st_size == sync_ctx->hdr->sync_size) {
/* we want to do full syncing. always do this if
file size hasn't changed but timestamp has. it most
likely means that someone had modified some header
and we probably want to know about it */
partial = FALSE;
sync_ctx->mbox->mbox_sync_dirty = TRUE;
} else {
/* see if we can delay syncing the whole file.
normally we only notice expunges and appends
in partial syncing. */
partial = TRUE;
sync_ctx->mbox->mbox_sync_dirty = TRUE;
}
mbox_sync_restart(sync_ctx);
for (i = 0;;) {
ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial);
if (ret > 0 && !sync_ctx->errors)
break;
if (ret < 0)
return -1;
/* a) partial sync didn't work
b) we ran out of UIDs
c) syncing had errors */
if (sync_ctx->delay_writes && !sync_ctx->mbox->mbox_readonly &&
(sync_ctx->errors || sync_ctx->renumber_uids)) {
/* fixing a broken mbox state, be sure to write
the changes. */
sync_ctx->delay_writes = FALSE;
}
if (++i == 3)
break;
mbox_sync_restart(sync_ctx);
partial = FALSE;
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
index_sync_changes_reset(sync_ctx->sync_changes);
if (sync_ctx->base_uid_last != sync_ctx->next_uid-1 &&
ret > 0 && !sync_ctx->delay_writes &&
sync_ctx->base_uid_last_offset != 0) {
/* Rewrite uid_last in X-IMAPbase header if we've seen it
(ie. the file isn't empty) */
ret = mbox_rewrite_base_uid_last(sync_ctx);
}
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty)
{
bool empty;
return mbox_sync_has_changed_full(mbox, leave_dirty, &empty);
}
int mbox_sync_has_changed_full(struct mbox_mailbox *mbox, bool leave_dirty,
bool *empty_r)
{
const struct mail_index_header *hdr;
const struct stat *st;
struct stat statbuf;
if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) {
/* read-only stream */
st = i_stream_stat(mbox->mbox_file_stream, FALSE);
if (st == NULL) {
if (errno == ENOENT) {
mailbox_set_deleted(&mbox->ibox.box);
return 0;
}
mbox_set_syscall_error(mbox, "i_stream_stat()");
return -1;
}
} else {
if (stat(mbox->path, &statbuf) < 0) {
if (errno == ENOENT) {
mailbox_set_deleted(&mbox->ibox.box);
return 0;
}
mbox_set_syscall_error(mbox, "stat()");
return -1;
}
st = &statbuf;
}
*empty_r = st->st_size == 0;
hdr = mail_index_get_header(mbox->ibox.view);
if ((uint32_t)st->st_mtime == hdr->sync_stamp &&
(uint64_t)st->st_size == hdr->sync_size) {
/* fully synced */
mbox->mbox_sync_dirty = FALSE;
return 0;
}
if (!mbox->mbox_sync_dirty || !leave_dirty) {
mbox->mbox_sync_dirty = TRUE;
return 1;
}
return st->st_mtime != mbox->mbox_dirty_stamp ||
st->st_size != mbox->mbox_dirty_size;
}
static void mbox_sync_context_free(struct mbox_sync_context *sync_ctx)
{
index_sync_changes_deinit(&sync_ctx->sync_changes);
if (sync_ctx->index_sync_ctx != NULL)
mail_index_sync_rollback(&sync_ctx->index_sync_ctx);
pool_unref(&sync_ctx->mail_keyword_pool);
pool_unref(&sync_ctx->saved_keywords_pool);
str_free(&sync_ctx->header);
str_free(&sync_ctx->from_line);
array_free(&sync_ctx->mails);
}
static int mbox_sync_int(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mail_index_transaction *trans;
struct mbox_sync_context sync_ctx;
enum mail_index_sync_flags sync_flags;
unsigned int lock_id = 0;
int ret, changed;
bool delay_writes;
delay_writes = mbox->mbox_readonly ||
((flags & MBOX_SYNC_REWRITE) == 0 &&
getenv("MBOX_LAZY_WRITES") != NULL);
if (!mbox->mbox_do_dirty_syncs)
flags |= MBOX_SYNC_UNDIRTY;
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
if (mbox_lock(mbox, F_RDLCK, &lock_id) <= 0)
return -1;
}
if ((flags & MBOX_SYNC_HEADER) != 0 ||
(flags & MBOX_SYNC_FORCE_SYNC) != 0)
changed = 1;
else {
bool leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
if ((changed = mbox_sync_has_changed(mbox, leave_dirty)) < 0) {
if ((flags & MBOX_SYNC_LOCK_READING) != 0)
(void)mbox_unlock(mbox, lock_id);
return -1;
}
}
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
/* we just want to lock it for reading. if mbox hasn't been
modified don't do any syncing. */
if (!changed)
return 0;
/* have to sync to make sure offsets have stayed the same */
(void)mbox_unlock(mbox, lock_id);
lock_id = 0;
}
/* reopen input stream to make sure it has nothing buffered */
mbox_file_close_stream(mbox);
again:
if (changed) {
/* we're most likely modifying the mbox while syncing, just
lock it for writing immediately. the mbox must be locked
before index syncing is started to avoid deadlocks, so we
don't have much choice either (well, easy ones anyway). */
int lock_type = mbox->mbox_readonly ? F_RDLCK : F_WRLCK;
if ((ret = mbox_lock(mbox, lock_type, &lock_id)) <= 0) {
if (ret == 0 || lock_type == F_RDLCK)
return -1;
/* try as read-only */
if (mbox_lock(mbox, F_RDLCK, &lock_id) <= 0)
return -1;
mbox->mbox_readonly = TRUE;
}
}
sync_flags = 0;
if (!mbox->ibox.keep_recent)
sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
if ((flags & MBOX_SYNC_REWRITE) != 0)
sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
if ((flags & MBOX_SYNC_LAST_COMMIT) != 0) {
ret = mail_index_sync_begin_to(mbox->ibox.index,
&index_sync_ctx, &sync_view, &trans,
mbox->ibox.commit_log_file_seq,
mbox->ibox.commit_log_file_offset, sync_flags);
} else {
ret = mail_index_sync_begin(mbox->ibox.index, &index_sync_ctx,
&sync_view, &trans, sync_flags);
}
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(&mbox->ibox);
if (lock_id != 0)
(void)mbox_unlock(mbox, lock_id);
return ret;
}
if (!mbox->ibox.keep_recent) {
/* see if we need to drop recent flags */
sync_ctx.hdr = mail_index_get_header(sync_view);
if (sync_ctx.hdr->first_recent_uid < sync_ctx.hdr->next_uid)
changed = 1;
}
if (!changed && !mail_index_sync_have_more(index_sync_ctx)) {
/* nothing to do */
nothing_to_do:
if (lock_id != 0)
(void)mbox_unlock(mbox, lock_id);
/* index may need to do internal syncing though, so commit
instead of rollbacking. */
if (mail_index_sync_commit(&index_sync_ctx) < 0) {
mail_storage_set_index_error(&mbox->ibox);
return -1;
}
return 0;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.mbox = mbox;
sync_ctx.hdr = mail_index_get_header(sync_view);
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = trans;
sync_ctx.mail_keyword_pool =
pool_alloconly_create("mbox keywords", 512);
sync_ctx.saved_keywords_pool =
pool_alloconly_create("mbox saved keywords", 4096);
/* make sure we've read the latest keywords in index */
(void)mail_index_get_keywords(mbox->ibox.index);
i_array_init(&sync_ctx.mails, 64);
sync_ctx.flags = flags;
sync_ctx.delay_writes = delay_writes || sync_ctx.mbox->mbox_readonly;
sync_ctx.sync_changes =
index_sync_changes_init(&mbox->ibox, index_sync_ctx,
sync_view, trans,
sync_ctx.delay_writes);
if (!changed && delay_writes) {
/* if we have only flag changes, we don't need to open the
mbox file */
bool expunged;
uint32_t uid;
mbox_sync_read_index_syncs(&sync_ctx, 1, &expunged);
uid = expunged ? 1 :
index_sync_changes_get_next_uid(sync_ctx.sync_changes);
if (uid == 0) {
sync_ctx.index_sync_ctx = NULL;
mbox_sync_context_free(&sync_ctx);
goto nothing_to_do;
}
}
if (lock_id == 0) {
/* ok, we have something to do but no locks. we'll have to
restart syncing to avoid deadlocking. */
mbox_sync_context_free(&sync_ctx);
changed = 1;
goto again;
}
if (mbox_file_open_stream(mbox) < 0) {
mbox_sync_context_free(&sync_ctx);
(void)mbox_unlock(mbox, lock_id);
return -1;
}
sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream;
sync_ctx.input = sync_ctx.mbox->mbox_stream;
sync_ctx.write_fd = sync_ctx.mbox->mbox_lock_type != F_WRLCK ? -1 :
sync_ctx.mbox->mbox_fd;
ret = mbox_sync_do(&sync_ctx, flags);
if (ret < 0)
mail_index_sync_rollback(&index_sync_ctx);
else if (mail_index_sync_commit(&index_sync_ctx) < 0) {
mail_storage_set_index_error(&mbox->ibox);
ret = -1;
} else {
mbox->ibox.commit_log_file_seq = 0;
mbox->ibox.commit_log_file_offset = 0;
}
sync_ctx.t = NULL;
sync_ctx.index_sync_ctx = NULL;
if (ret == 0 && mbox->mbox_fd != -1 && mbox->ibox.keep_recent &&
!sync_ctx.mbox->mbox_readonly) {
/* try to set atime back to its original value */
struct utimbuf buf;
struct stat st;
if (fstat(mbox->mbox_fd, &st) < 0)
mbox_set_syscall_error(mbox, "fstat()");
else {
buf.modtime = st.st_mtime;
buf.actime = sync_ctx.orig_atime;
if (utime(mbox->path, &buf) < 0)
mbox_set_syscall_error(mbox, "utime()");
}
}
i_assert(lock_id != 0);
if ((mbox->storage->storage.flags &
MAIL_STORAGE_FLAG_NFS_FLUSH_STORAGE) != 0 && mbox->mbox_fd != -1) {
if (fdatasync(mbox->mbox_fd) < 0) {
mbox_set_syscall_error(mbox, "fdatasync()");
ret = -1;
}
}
if (mbox->mbox_lock_type != F_RDLCK) {
/* drop to read lock */
unsigned int read_lock_id = 0;
if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0)
ret = -1;
else {
if (mbox_unlock(mbox, lock_id) < 0)
ret = -1;
lock_id = read_lock_id;
}
}
if ((flags & MBOX_SYNC_LOCK_READING) == 0) {
if (mbox_unlock(mbox, lock_id) < 0)
ret = -1;
}
mbox_sync_context_free(&sync_ctx);
return ret;
}
int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
{
int ret;
mbox->syncing = TRUE;
ret = mbox_sync_int(mbox, flags);
mbox->syncing = FALSE;
if (mbox->ibox.box.v.sync_notify != NULL)
mbox->ibox.box.v.sync_notify(&mbox->ibox.box, 0, 0);
return ret;
}
struct mailbox_sync_context *
mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
enum mbox_sync_flags mbox_sync_flags = 0;
int ret = 0;
if (!box->opened)
index_storage_mailbox_open(&mbox->ibox);
if (index_mailbox_want_full_sync(&mbox->ibox, flags)) {
if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
!mbox->mbox_very_dirty_syncs)
mbox_sync_flags |= MBOX_SYNC_UNDIRTY;
if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0)
mbox_sync_flags |= MBOX_SYNC_REWRITE;
ret = mbox_sync(mbox, mbox_sync_flags);
}
return index_mailbox_sync_init(box, flags, ret < 0);
}