maildir-sync.c revision c282435b57b6f9696fc12d99ea70468b7bdfe24c
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/*
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Here's a description of how we handle Maildir synchronization and
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it's problems:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen We want to be as efficient as we can. The most efficient way to
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen check if changes have occured is to stat() the new/ and cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen directories and uidlist file - if their mtimes haven't changed,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen there's no changes and we don't need to do anything.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Problem 1: Multiple changes can happen within a single second -
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen nothing guarantees that once we synced it, someone else didn't just
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen then make a modification. Such modifications wouldn't get noticed
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen until a new modification occured later.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Problem 2: Syncing cur/ directory is much more costly than syncing
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new/. Moving mails from new/ to cur/ will always change mtime of
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cur/ causing us to sync it as well.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Problem 3: We may not be able to move mail from new/ to cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen because we're out of quota, or simply because we're accessing a
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen read-only mailbox.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MAILDIR_SYNC_SECS
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen -----------------
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Several checks below use MAILDIR_SYNC_SECS, which should be maximum
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen clock drift between all computers accessing the maildir (eg. via
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen NFS), rounded up to next second. Our default is 1 second, since
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen everyone should be using NTP.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Note that setting it to 0 works only if there's only one computer
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen accessing the maildir. It's practically impossible to make two
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen clocks _exactly_ synchronized.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen It might be possible to only use file server's clock by looking at
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the atime field, but I don't know how well that would actually work.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cur directory
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen -------------
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen We have dirty_cur_time variable which is set to cur/ directory's
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen synchronized the directory.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen When dirty_cur_time is non-zero, we don't synchronize the cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen directory until
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen a) cur/'s mtime changes
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen b) opening a mail fails with ENOENT
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen c) time() > dirty_cur_time + MAILDIR_SYNC_SECS
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen This allows us to modify the maildir multiple times without having
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen to sync it at every change. The sync will eventually be done to
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen make sure we didn't miss any external changes.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen The dirty_cur_time is set when:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - we change message flags
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - we expunge messages
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - we move mail from new/ to cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen It's unset when we do the final syncing, ie. when mtime is
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen older than time() - MAILDIR_SYNC_SECS.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new directory
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen -------------
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it. dirty_cur_time-like feature might save us a few syncs, but
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen that might break a client which saves a mail in one connection and
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen tries to fetch it in another one. new/ directory is almost always
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen empty, so syncing it should be very fast anyway. Actually this can
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen still happen if we sync only new/ dir while another client is also
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen moving mails from it to cur/ - it takes us a while to see them.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen That's pretty unlikely to happen however, and only way to fix it
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen would be to always synchronize cur/ after new/.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Normally we move all mails from new/ to cur/ whenever we sync it. If
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it's not possible for some reason, we mark the mail with "probably
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen exists in new/ directory" flag.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen If rename() still fails because of ENOSPC or EDQUOT, we still save
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the flag changes in index with dirty-flag on. When moving the mail
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen to cur/ directory, or when we notice it's already moved there, we
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen apply the flag changes to the filename, rename it and remove the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen dirty flag. If there's dirty flags, this should be tried every time
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen after expunge or when closing the mailbox.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uidlist
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen -------
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen This file contains UID <-> filename mappings. It's updated only when
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new mail arrives, so it may contain filenames that have already been
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen deleted. Updating is done by getting uidlist.lock file, writing the
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen whole uidlist into it and rename()ing it over the old uidlist. This
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen means there's no need to lock the file for reading.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Whenever uidlist is rewritten, it's mtime must be larger than the old
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen one's. Use utime() before rename() if needed. Note that inode checking
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen wouldn't have been sufficient as inode numbers can be reused.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen This file is usually read the first time you need to know filename for
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen given UID. After that it's not re-read unless new mails come that we
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen don't know about.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen broken clients
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen --------------
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Originally the middle identifier in Maildir filename was specified
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen only as <process id>_<delivery counter>. That however created a
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen problem with randomized PIDs which made it possible that the same
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen PID was reused within one second.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen So if within one second a mail was delivered, MUA moved it to cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen and another mail was delivered by a new process using same PID as
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the first one, we likely ended up overwriting the first mail when
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the second mail was moved over it.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Nowadays everyone should be giving a bit more specific identifier,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for example include microseconds in it which Dovecot does.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen There's a simple way to prevent this from happening in some cases:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Don't move the mail from new/ to cur/ if it's mtime is >= time() -
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MAILDIR_SYNC_SECS. The second delivery's link() call then fails
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen because the file is already in new/, and it will then use a
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen different filename. There's a few problems with this however:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - it requires extra stat() call which is unneeded extra I/O
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - another MUA might still move the mail to cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - if first file's flags are modified by either Dovecot or another
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen but that'd be ugly)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Because this is useful only for very few people and it requires
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen extra I/O, I decided not to implement this. It should be however
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen quite easy to do since we need to be able to deal with files in new/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen in any case.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen It's also possible to never accidentally overwrite a mail by using
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen link() + unlink() rather than rename(). This however isn't very
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen good idea as it introduces potential race conditions when multiple
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen clients are accessing the mailbox:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Trying to move the same mail from new/ to cur/ at the same time:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen a) Client 1 uses slightly different filename than client 2,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for example one sets read-flag on but the other doesn't.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen You have the same mail duplicated now.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen b) Client 3 sees the mail between Client 1's and 2's link() calls
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen and changes it's flag. You have the same mail duplicated now.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen And it gets worse when they're unlink()ing in cur/ directory:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen c) Client 1 changes mails's flag and client 2 changes it back
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen between 1's link() and unlink(). The mail is now expunged.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen d) If you try to deal with the duplicates by unlink()ing another
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen one of them, you might end up unlinking both of them.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen So, what should we do then if we notice a duplicate? First of all,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it might not be a duplicate at all, readdir() might have just
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen returned it twice because it was just renamed. What we should do is
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen create a completely new base name for it and rename() it to that.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen If the call fails with ENOENT, it only means that it wasn't a
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen duplicate after all.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen*/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "ioloop.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "buffer.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "hash.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "str.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "maildir-storage.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "maildir-uidlist.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <stdio.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <unistd.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <dirent.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <sys/stat.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#define MAILDIR_SYNC_SECS 1
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#define MAILDIR_FILENAME_FLAG_FOUND 128
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstruct maildir_sync_context {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct index_mailbox *ibox;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *new_dir, *cur_dir;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen int partial;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen};
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstruct maildir_index_sync_context {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct index_mailbox *ibox;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_view *view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_sync_ctx *sync_ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_transaction *trans;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_sync_rec sync_rec;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t seq;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen int have_dirty, last_dirty;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen};
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_expunge(struct index_mailbox *ibox, const char *path,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen void *context __attr_unused__)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (unlink(path) == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->dirty_cur_time = ioloop_time;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (errno == ENOENT)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_storage_set_critical(ibox->box.storage,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "unlink(%s) failed: %m", path);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_sync_flags(struct index_mailbox *ibox, const char *path,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen void *context)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct maildir_index_sync_context *ctx = context;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *newpath;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen enum mail_flags flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint8_t flags8;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen keywords_mask_t keywords;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->last_dirty = FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (void)maildir_filename_get_flags(path, &flags, keywords);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen flags8 = flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_sync_flags_apply(&ctx->sync_rec, &flags8, keywords);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen newpath = maildir_filename_set_flags(path, flags8, keywords);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (rename(path, newpath) == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->dirty_cur_time = ioloop_time;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (errno == ENOENT)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ENOSPACE(errno) || errno == EACCES) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(keywords, 0, sizeof(keywords));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_ADD,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MAIL_INDEX_MAIL_FLAG_DIRTY, keywords);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->last_dirty = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_storage_set_critical(ibox->box.storage,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "rename(%s, %s) failed: %m", path, newpath);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_sync_record(struct index_mailbox *ibox,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct maildir_index_sync_context *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_sync_rec *sync_rec = &ctx->sync_rec;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_view *view = ctx->view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mail_index_record *rec;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t seq, seq1, seq2, uid;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen switch (sync_rec->type) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen case MAIL_INDEX_SYNC_TYPE_APPEND:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* make it go through sequences to avoid looping through huge
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen holes in UID range */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_lookup_uid_range(view, sync_rec->uid1,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen sync_rec->uid2,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &seq1, &seq2) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (seq1 == 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (seq = seq1; seq <= seq2; seq++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_lookup_uid(view, seq, &uid) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (maildir_file_do(ibox, uid, maildir_expunge,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen NULL) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen case MAIL_INDEX_SYNC_TYPE_FLAGS:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_lookup_uid_range(view, sync_rec->uid1,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen sync_rec->uid2,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &seq1, &seq2) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (seq1 == 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (ctx->seq = seq1; ctx->seq <= seq2; ctx->seq++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_lookup_uid(view, ctx->seq, &uid) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (maildir_file_do(ibox, uid,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen maildir_sync_flags, ctx) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
31ddc75584c5cde53d2e78a737587f2e7fdcb0d2Timo Sirainen if (!ctx->last_dirty) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* if this flag was dirty, drop it */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_lookup(view, ctx->seq, &rec) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen keywords_mask_t keywords;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(keywords, 0, sizeof(keywords));
31ddc75584c5cde53d2e78a737587f2e7fdcb0d2Timo Sirainen mail_index_update_flags(ctx->trans,
31ddc75584c5cde53d2e78a737587f2e7fdcb0d2Timo Sirainen ctx->seq, MODIFY_REMOVE,
31ddc75584c5cde53d2e78a737587f2e7fdcb0d2Timo Sirainen MAIL_INDEX_MAIL_FLAG_DIRTY,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen keywords);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint maildir_sync_last_commit(struct index_mailbox *ibox)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct maildir_index_sync_context ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mail_index_header *hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t seq;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uoff_t offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen int ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ibox->commit_log_file_seq == 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(&ctx, 0, sizeof(ctx));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx.ibox = ibox;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = mail_index_sync_begin(ibox->index, &ctx.sync_ctx, &ctx.view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->commit_log_file_seq,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->commit_log_file_offset);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret > 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_get_header(ctx.view, &hdr) == 0 &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx.have_dirty = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx.trans = mail_index_transaction_begin(ctx.view, FALSE);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen while ((ret = mail_index_sync_next(ctx.sync_ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &ctx.sync_rec)) > 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (maildir_sync_record(ibox, &ctx) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_transaction_commit(ctx.trans, &seq, &offset) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_sync_end(ctx.sync_ctx, 0, 0) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->commit_log_file_seq = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->commit_log_file_offset = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_storage_set_index_error(ibox);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic struct maildir_sync_context *
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmaildir_sync_context_new(struct index_mailbox *ibox)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct maildir_sync_context *ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx = t_new(struct maildir_sync_context, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->ibox = ibox;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->new_dir = t_strconcat(ibox->path, "/new", NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->cur_dir = t_strconcat(ibox->path, "/cur", NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic void maildir_sync_deinit(struct maildir_sync_context *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->uidlist_sync_ctx != NULL)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_fix_duplicate(struct index_mailbox *ibox, const char *dir,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *old_fname)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *new_fname, *old_path, *new_path;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen int ret = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen t_push();
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen old_path = t_strconcat(dir, "/", old_fname, NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new_fname = maildir_generate_tmp_filename(&ioloop_timeval);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new_path = t_strconcat(ibox->path, "/new/", new_fname, NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (rename(old_path, new_path) == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_warning("Fixed duplicate in %s: %s -> %s",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->path, old_fname, new_fname);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else if (errno != ENOENT) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_storage_set_critical(ibox->box.storage,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "rename(%s, %s) failed: %m", old_path, new_path);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen t_pop();
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_storage *storage = ctx->ibox->box.storage;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *dir;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen DIR *dirp;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen string_t *src, *dest;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct dirent *dp;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen enum maildir_uidlist_rec_flag flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen int move_new, ret = 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen src = t_str_new(1024);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen dest = t_str_new(1024);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen dir = new_dir ? ctx->new_dir : ctx->cur_dir;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen dirp = opendir(dir);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (dirp == NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_storage_set_critical(storage,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "opendir(%s) failed: %m", dir);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen move_new = new_dir && !mailbox_is_readonly(&ctx->ibox->box) &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen !ctx->ibox->keep_recent;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen while ((dp = readdir(dirp)) != NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (dp->d_name[0] == '.')
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen continue;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = maildir_uidlist_sync_next_pre(ctx->uidlist_sync_ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen dp->d_name);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* new file and we couldn't lock uidlist, check this
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen later in next sync. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (new_dir)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->ibox->last_new_mtime = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen else
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->ibox->dirty_cur_time = ioloop_time;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen continue;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen flags = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (move_new) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_truncate(src, 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_truncate(dest, 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (strchr(dp->d_name, ':') == NULL)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_append(dest, ":2,");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (rename(str_c(src), str_c(dest)) == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* we moved it - it's \Recent for us */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->ibox->dirty_cur_time = ioloop_time;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MAILDIR_UIDLIST_REC_FLAG_RECENT;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else if (ENOTFOUND(errno)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* someone else moved it already */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else if (ENOSPACE(errno)) {
/* not enough disk space, leave here */
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
move_new = FALSE;
} else {
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
mail_storage_set_critical(storage,
"rename(%s, %s) failed: %m",
str_c(src), str_c(dest));
}
} else if (new_dir) {
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
}
ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
dp->d_name, flags);
if (ret <= 0) {
if (ret < 0)
break;
/* possibly duplicate - try fixing it */
if (maildir_fix_duplicate(ctx->ibox,
dir, dp->d_name) < 0) {
ret = -1;
break;
}
}
}
if (closedir(dirp) < 0) {
mail_storage_set_critical(storage,
"closedir(%s) failed: %m", dir);
}
return ret < 0 ? -1 : 0;
}
static int maildir_sync_quick_check(struct maildir_sync_context *ctx,
int *new_changed_r, int *cur_changed_r)
{
const struct mail_index_header *hdr;
struct index_mailbox *ibox = ctx->ibox;
struct stat st;
time_t new_mtime, cur_mtime;
*new_changed_r = *cur_changed_r = FALSE;
if (stat(ctx->new_dir, &st) < 0) {
mail_storage_set_critical(ibox->box.storage,
"stat(%s) failed: %m", ctx->new_dir);
return -1;
}
new_mtime = st.st_mtime;
if (stat(ctx->cur_dir, &st) < 0) {
mail_storage_set_critical(ibox->box.storage,
"stat(%s) failed: %m", ctx->cur_dir);
return -1;
}
cur_mtime = st.st_mtime;
if (ibox->last_cur_mtime == 0) {
/* first sync in this session, get cur stamp from index */
if (mail_index_get_header(ibox->view, &hdr) == 0)
ibox->last_cur_mtime = hdr->sync_stamp;
}
if (new_mtime != ibox->last_new_mtime ||
new_mtime >= ibox->last_new_sync_time - MAILDIR_SYNC_SECS) {
*new_changed_r = TRUE;
ibox->last_new_mtime = new_mtime;
ibox->last_new_sync_time = ioloop_time;
}
if (cur_mtime != ibox->last_cur_mtime ||
(ibox->dirty_cur_time != 0 &&
ioloop_time - ibox->dirty_cur_time > MAILDIR_SYNC_SECS)) {
/* cur/ changed, or delayed cur/ check */
*cur_changed_r = TRUE;
ibox->last_cur_mtime = cur_mtime;
ibox->dirty_cur_time =
cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?
cur_mtime : 0;
}
return 0;
}
static int maildir_sync_index(struct maildir_sync_context *ctx)
{
struct index_mailbox *ibox = ctx->ibox;
struct maildir_index_sync_context sync_ctx;
struct maildir_uidlist_iter_ctx *iter;
struct mail_index_transaction *trans;
struct mail_index_view *view;
const struct mail_index_header *hdr;
const struct mail_index_record *rec;
uint32_t seq, uid;
enum maildir_uidlist_rec_flag uflags;
const char *filename;
enum mail_flags flags;
keywords_mask_t keywords;
uint32_t sync_stamp;
int ret;
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.ibox = ibox;
if (mail_index_sync_begin(ibox->index, &sync_ctx.sync_ctx, &view,
(uint32_t)-1, (uoff_t)-1) <= 0) {
mail_storage_set_index_error(ibox);
return -1;
}
sync_ctx.view = view;
ret = mail_index_get_header(view, &hdr);
i_assert(ret == 0); /* view is locked, can't happen */
trans = mail_index_transaction_begin(view, FALSE);
sync_ctx.trans = trans;
seq = 0;
iter = maildir_uidlist_iter_init(ibox->uidlist);
while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
maildir_filename_get_flags(filename, &flags, keywords);
if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0 &&
(uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 &&
(uflags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0) {
/* mail is recent for next session as well */
flags |= MAIL_RECENT;
}
__again:
seq++;
if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
/* partial syncing */
continue;
}
if (seq > hdr->messages_count) {
if (uid < hdr->next_uid) {
/* message not in index, but next_uid header
is updated? shouldn't really happen.. */
mail_storage_set_critical(ibox->box.storage,
"Maildir sync: UID < next_uid "
"(%u < %u, file = %s)",
uid, hdr->next_uid, filename);
mail_index_mark_corrupted(ibox->index);
ret = -1;
break;
}
mail_index_append(trans, uid, &seq);
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
flags, keywords);
continue;
}
if (mail_index_lookup(view, seq, &rec) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
break;
}
if (rec->uid < uid) {
/* expunged */
mail_index_expunge(trans, seq);
goto __again;
}
if (rec->uid > uid) {
/* most likely a race condition: we read the
maildir, then someone else expunged messages and
committed changes to index. so, this message
shouldn't actually exist. check to be sure.
FIXME: we could avoid this stat() and just mark
this check in the uidlist and check it at next
sync.. */
struct stat st;
const char *str;
t_push();
str = t_strdup_printf("%s/%s",
(uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) ?
ctx->new_dir : ctx->cur_dir, filename);
if (stat(str, &st) == 0) {
t_pop();
mail_storage_set_critical(ibox->box.storage,
"Maildir sync: UID inserted in the "
"middle of mailbox "
"(%u > %u, file = %s)",
rec->uid, uid, filename);
mail_index_mark_corrupted(ibox->index);
ret = -1;
break;
}
t_pop();
seq--;
continue;
}
if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* we haven't been able to update maildir with this
record's flag changes. don't sync them. */
continue;
}
if (((uint8_t)flags & ~MAIL_RECENT) !=
(rec->flags & (MAIL_FLAGS_MASK^MAIL_RECENT)) ||
memcmp(keywords, rec->keywords,
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
/* FIXME: this is wrong if there's pending changes in
transaction log already. it gets fixed in next sync
however.. */
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
flags, keywords);
} else if ((flags & MAIL_RECENT) == 0 &&
(rec->flags & MAIL_RECENT) != 0) {
/* just remove recent flag */
memset(keywords, 0, sizeof(keywords));
mail_index_update_flags(trans, seq, MODIFY_REMOVE,
MAIL_RECENT, keywords);
}
}
maildir_uidlist_iter_deinit(iter);
if (!ctx->partial) {
/* expunge the rest */
for (seq++; seq <= hdr->messages_count; seq++)
mail_index_expunge(trans, seq);
}
/* now, sync the index */
while ((ret = mail_index_sync_next(sync_ctx.sync_ctx,
&sync_ctx.sync_rec)) > 0) {
if (maildir_sync_record(ibox, &sync_ctx) < 0) {
ret = -1;
break;
}
}
if (ret < 0)
mail_index_transaction_rollback(trans);
else {
uint32_t seq;
uoff_t offset;
if (mail_index_transaction_commit(trans, &seq, &offset) < 0)
mail_storage_set_index_error(ibox);
else if (seq != 0) {
ibox->commit_log_file_seq = seq;
ibox->commit_log_file_offset = offset;
}
}
sync_stamp = ibox->dirty_cur_time != 0 ? 0 : ibox->last_cur_mtime;
if (mail_index_sync_end(sync_ctx.sync_ctx, sync_stamp, 0) < 0)
ret = -1;
if (ret == 0) {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
} else {
mail_storage_set_index_error(ibox);
}
return ret;
}
static int maildir_sync_context(struct maildir_sync_context *ctx)
{
int ret, new_changed, cur_changed;
if (maildir_sync_quick_check(ctx, &new_changed, &cur_changed) < 0)
return -1;
if (!new_changed && !cur_changed)
return 0;
ctx->partial = !cur_changed;
ctx->uidlist_sync_ctx =
maildir_uidlist_sync_init(ctx->ibox->uidlist, ctx->partial);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
if (cur_changed) {
if (maildir_scan_dir(ctx, FALSE) < 0)
return -1;
}
/* finish uidlist syncing, but keep it still locked */
if (maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx)) {
if (maildir_sync_index(ctx) < 0)
return -1;
}
ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
ctx->uidlist_sync_ctx = NULL;
return ret;
}
static int maildir_sync_context_readonly(struct maildir_sync_context *ctx)
{
int ret;
ctx->uidlist_sync_ctx =
maildir_uidlist_sync_init(ctx->ibox->uidlist, FALSE);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
if (maildir_scan_dir(ctx, FALSE) < 0)
return -1;
ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
ctx->uidlist_sync_ctx = NULL;
return ret;
}
int maildir_storage_sync_readonly(struct index_mailbox *ibox)
{
struct maildir_sync_context *ctx;
int ret;
ctx = maildir_sync_context_new(ibox);
ret = maildir_sync_context_readonly(ctx);
maildir_sync_deinit(ctx);
return ret;
}
int maildir_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
struct maildir_sync_context *ctx;
int ret;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
ctx = maildir_sync_context_new(ibox);
ret = maildir_sync_context(ctx);
maildir_sync_deinit(ctx);
if (ret < 0)
return -1;
}
return index_storage_sync(box, flags);
}