index/maildir/maildir-sync.c

	maildir-sync.c revision 0db4290d60bfa00774f628276d38654c56abd68c
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi/* Copyright (C) 2004 Timo Sirainen */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi/*
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Here's a description of how we handle Maildir synchronization and
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   it's problems:
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   We want to be as efficient as we can. The most efficient way to
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   check if changes have occurred is to stat() the new/ and cur/
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   directories and uidlist file - if their mtimes haven't changed,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   there's no changes and we don't need to do anything.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Problem 1: Multiple changes can happen within a single second -
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   nothing guarantees that once we synced it, someone else didn't just
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   then make a modification. Such modifications wouldn't get noticed
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   until a new modification occurred later.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Problem 2: Syncing cur/ directory is much more costly than syncing
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   new/. Moving mails from new/ to cur/ will always change mtime of
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   cur/ causing us to sync it as well.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Problem 3: We may not be able to move mail from new/ to cur/
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   because we're out of quota, or simply because we're accessing a
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   read-only mailbox.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   MAILDIR_SYNC_SECS
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   -----------------
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Several checks below use MAILDIR_SYNC_SECS, which should be maximum
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   clock drift between all computers accessing the maildir (eg. via
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   NFS), rounded up to next second. Our default is 1 second, since
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   everyone should be using NTP.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Note that setting it to 0 works only if there's only one computer
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   accessing the maildir. It's practically impossible to make two
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   clocks _exactly_ synchronized.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   It might be possible to only use file server's clock by looking at
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   the atime field, but I don't know how well that would actually work.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   cur directory
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   -------------
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   We have dirty_cur_time variable which is set to cur/ directory's
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   synchronized the directory.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   When dirty_cur_time is non-zero, we don't synchronize the cur/
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   directory until
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      a) cur/'s mtime changes
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      b) opening a mail fails with ENOENT
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      c) time() > dirty_cur_time + MAILDIR_SYNC_SECS
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   This allows us to modify the maildir multiple times without having
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   to sync it at every change. The sync will eventually be done to
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   make sure we didn't miss any external changes.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   The dirty_cur_time is set when:
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      - we change message flags
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      - we expunge messages
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      - we move mail from new/ to cur/
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   It's unset when we do the final syncing, ie. when mtime is
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   older than time() - MAILDIR_SYNC_SECS.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   new directory
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   -------------
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   it. dirty_cur_time-like feature might save us a few syncs, but
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   that might break a client which saves a mail in one connection and
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   tries to fetch it in another one. new/ directory is almost always
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   empty, so syncing it should be very fast anyway. Actually this can
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   still happen if we sync only new/ dir while another client is also
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   moving mails from it to cur/ - it takes us a while to see them.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   That's pretty unlikely to happen however, and only way to fix it
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   would be to always synchronize cur/ after new/.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Normally we move all mails from new/ to cur/ whenever we sync it. If
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   it's not possible for some reason, we mark the mail with "probably
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   exists in new/ directory" flag.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   If rename() still fails because of ENOSPC or EDQUOT, we still save
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   the flag changes in index with dirty-flag on. When moving the mail
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   to cur/ directory, or when we notice it's already moved there, we
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   apply the flag changes to the filename, rename it and remove the
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   dirty flag. If there's dirty flags, this should be tried every time
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   after expunge or when closing the mailbox.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   uidlist
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   -------
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   This file contains UID <-> filename mappings. It's updated only when
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   new mail arrives, so it may contain filenames that have already been
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   deleted. Updating is done by getting uidlist.lock file, writing the
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   whole uidlist into it and rename()ing it over the old uidlist. This
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   means there's no need to lock the file for reading.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Whenever uidlist is rewritten, it's mtime must be larger than the old
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   one's. Use utime() before rename() if needed. Note that inode checking
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   wouldn't have been sufficient as inode numbers can be reused.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   This file is usually read the first time you need to know filename for
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   given UID. After that it's not re-read unless new mails come that we
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   don't know about.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   broken clients
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   --------------
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Originally the middle identifier in Maildir filename was specified
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   only as <process id>_<delivery counter>. That however created a
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   problem with randomized PIDs which made it possible that the same
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   PID was reused within one second.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   So if within one second a mail was delivered, MUA moved it to cur/
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   and another mail was delivered by a new process using same PID as
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   the first one, we likely ended up overwriting the first mail when
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   the second mail was moved over it.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Nowadays everyone should be giving a bit more specific identifier,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   for example include microseconds in it which Dovecot does.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   There's a simple way to prevent this from happening in some cases:
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Don't move the mail from new/ to cur/ if it's mtime is >= time() -
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   MAILDIR_SYNC_SECS. The second delivery's link() call then fails
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   because the file is already in new/, and it will then use a
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   different filename. There's a few problems with this however:
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      - it requires extra stat() call which is unneeded extra I/O
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      - another MUA might still move the mail to cur/
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      - if first file's flags are modified by either Dovecot or another
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    but that'd be ugly)
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Because this is useful only for very few people and it requires
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   extra I/O, I decided not to implement this. It should be however
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   quite easy to do since we need to be able to deal with files in new/
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   in any case.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   It's also possible to never accidentally overwrite a mail by using
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   link() + unlink() rather than rename(). This however isn't very
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   good idea as it introduces potential race conditions when multiple
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   clients are accessing the mailbox:
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   Trying to move the same mail from new/ to cur/ at the same time:
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      a) Client 1 uses slightly different filename than client 2,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi         for example one sets read-flag on but the other doesn't.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi     You have the same mail duplicated now.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      b) Client 3 sees the mail between Client 1's and 2's link() calls
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi         and changes it's flag. You have the same mail duplicated now.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   And it gets worse when they're unlink()ing in cur/ directory:
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      c) Client 1 changes mails's flag and client 2 changes it back
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi         between 1's link() and unlink(). The mail is now expunged.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi      d) If you try to deal with the duplicates by unlink()ing another
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi         one of them, you might end up unlinking both of them.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   So, what should we do then if we notice a duplicate? First of all,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   it might not be a duplicate at all, readdir() might have just
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   returned it twice because it was just renamed. What we should do is
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   create a completely new base name for it and rename() it to that.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   If the call fails with ENOENT, it only means that it wasn't a
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   duplicate after all.
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi*/
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "lib.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "ioloop.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "array.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "buffer.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "hash.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "str.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "maildir-storage.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "maildir-uidlist.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include "maildir-keywords.h"
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include <stdio.h>
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include <stddef.h>
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include <unistd.h>
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include <dirent.h>
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#include <sys/stat.h>
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#define MAILDIR_SYNC_SECS 1
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#define MAILDIR_FILENAME_FLAG_FOUND 128
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi/* When rename()ing many files from new/ to cur/, it's possible that next
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   readdir() skips some files. we don't of course wish to lose them, so we
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   go and rescan the new/ directory again from beginning until no files are
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   left. This value is just an optimization to avoid checking the directory
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   twice unneededly. usually only NFS is the problem case. 1 is the safest
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi   bet here, but I guess 5 will do just fine too. */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi#define MAILDIR_RENAME_RESCAN_COUNT 5
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomistruct maildir_sync_context {
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        struct maildir_mailbox *mbox;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    const char *new_dir, *cur_dir;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    int partial;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        struct maildir_index_sync_context *index_sync_ctx;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi};
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomistruct maildir_index_sync_context {
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        struct maildir_mailbox *mbox;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    struct mail_index_view *view;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    struct mail_index_sync_ctx *sync_ctx;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        struct maildir_keywords_sync_ctx *keywords_sync_ctx;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    struct mail_index_transaction *trans;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    array_t ARRAY_DEFINE(sync_recs, struct mail_index_sync_rec);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    uint32_t seq;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    int dirty_state;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi};
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomiint maildir_filename_get_flags(struct maildir_index_sync_context *ctx,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                   const char *fname,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                   enum mail_flags *flags_r,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                               array_t *keywords_r)
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi{
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    ARRAY_SET_TYPE(keywords_r, unsigned int);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    const char *info;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    array_clear(keywords_r);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    *flags_r = 0;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    info = strchr(fname, MAILDIR_INFO_SEP);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    if (info == NULL || info[1] != '2' || info[2] != MAILDIR_FLAGS_SEP)
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        return 0;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    for (info += 3; *info != '\0' && *info != MAILDIR_FLAGS_SEP; info++) {
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        switch (*info) {
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        case 'R': /* replied */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            *flags_r |= MAIL_ANSWERED;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            break;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        case 'S': /* seen */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            *flags_r |= MAIL_SEEN;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            break;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        case 'T': /* trashed */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            *flags_r |= MAIL_DELETED;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            break;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        case 'D': /* draft */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            *flags_r |= MAIL_DRAFT;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            break;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        case 'F': /* flagged */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            *flags_r |= MAIL_FLAGGED;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            break;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        default:
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            if (*info >= MAILDIR_KEYWORD_FIRST &&
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                *info <= MAILDIR_KEYWORD_LAST) {
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                int idx;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                idx = maildir_keywords_char_idx(
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                        ctx->keywords_sync_ctx, *info);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                if (idx < 0) {
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                    /* unknown keyword. */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                    break;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                }
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                array_append(keywords_r,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                         (unsigned int *)&idx, 1);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                break;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            }
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            /* unknown flag - ignore */
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            break;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        }
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    }
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    return 1;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi}
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomistatic void
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomimaildir_filename_append_keywords(struct maildir_keywords_sync_ctx *ctx,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                 array_t *keywords, string_t *str)
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi{
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    ARRAY_SET_TYPE(keywords, unsigned int);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    const unsigned int *indexes;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    unsigned int i, count;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    char chr;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    indexes = array_get(keywords, &count);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    for (i = 0; i < count; i++) {
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        chr = maildir_keywords_idx_char(ctx, indexes[i]);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi        if (chr != '\0')
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi            str_append_c(str, chr);
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    }
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi}
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomiconst char *maildir_filename_set_flags(struct maildir_index_sync_context *ctx,
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi                       const char *fname, enum mail_flags flags,
aa6cd0d62ab34864c476c7fb3344e8e4ba035261Aki Tuomi                       array_t *keywords)
aa6cd0d62ab34864c476c7fb3344e8e4ba035261Aki Tuomi{
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    string_t *flags_str;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    enum mail_flags flags_left;
f6de86ea29e87fba001b6231d38a4c51e8a5c543Aki Tuomi    const char *info, *oldflags;
    int nextflag;

    /* remove the old :info from file name, and get the old flags */
    info = strrchr(fname, MAILDIR_INFO_SEP);
    if (info != NULL && strrchr(fname, '/') > info)
        info = NULL;

    oldflags = "";
    if (info != NULL) {
        fname = t_strdup_until(fname, info);
        if (info[1] == '2' && info[2] == MAILDIR_FLAGS_SEP)
            oldflags = info+3;
    }

    /* insert the new flags between old flags. flags must be sorted by
       their ASCII code. unknown flags are kept. */
    flags_str = t_str_new(256);
    str_append(flags_str, fname);
    str_append(flags_str, MAILDIR_FLAGS_FULL_SEP);
    flags_left = flags;
    for (;;) {
        /* skip all known flags */
        while (*oldflags == 'D' || *oldflags == 'F' ||
               *oldflags == 'R' || *oldflags == 'S' ||
               *oldflags == 'T' ||
               (*oldflags >= MAILDIR_KEYWORD_FIRST &&
            *oldflags <= MAILDIR_KEYWORD_LAST))
            oldflags++;

        nextflag = *oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP ?
            256 : (unsigned char) *oldflags;

        if ((flags_left & MAIL_DRAFT) && nextflag > 'D') {
            str_append_c(flags_str, 'D');
            flags_left &= ~MAIL_DRAFT;
        }
        if ((flags_left & MAIL_FLAGGED) && nextflag > 'F') {
            str_append_c(flags_str, 'F');
            flags_left &= ~MAIL_FLAGGED;
        }
        if ((flags_left & MAIL_ANSWERED) && nextflag > 'R') {
            str_append_c(flags_str, 'R');
            flags_left &= ~MAIL_ANSWERED;
        }
        if ((flags_left & MAIL_SEEN) && nextflag > 'S') {
            str_append_c(flags_str, 'S');
            flags_left &= ~MAIL_SEEN;
        }
        if ((flags_left & MAIL_DELETED) && nextflag > 'T') {
            str_append_c(flags_str, 'T');
            flags_left &= ~MAIL_DELETED;
        }

        if (keywords != NULL && array_is_created(keywords) &&
            nextflag > MAILDIR_KEYWORD_FIRST) {
            maildir_filename_append_keywords(ctx->keywords_sync_ctx,
                             keywords, flags_str);
            keywords = NULL;
        }

        if (*oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP)
            break;

        str_append_c(flags_str, *oldflags);
        oldflags++;
    }

    if (*oldflags == MAILDIR_FLAGS_SEP) {
        /* another flagset, we don't know about these, just keep them */
        while (*oldflags != '\0')
            str_append_c(flags_str, *oldflags++);
    }

    return str_c(flags_str);
}

static int maildir_expunge(struct maildir_mailbox *mbox, const char *path,
               void *context __attr_unused__)
{
    if (unlink(path) == 0) {
        mbox->dirty_cur_time = ioloop_time;
        return 1;
    }
    if (errno == ENOENT)
        return 0;

    mail_storage_set_critical(STORAGE(mbox->storage),
                  "unlink(%s) failed: %m", path);
    return -1;
}

static int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path,
                  void *context)
{
        struct maildir_index_sync_context *ctx = context;
    const struct mail_index_sync_rec *recs;
    const char *newpath;
    enum mail_flags flags;
    array_t ARRAY_DEFINE(keywords, unsigned int);
    unsigned int i, count;
    uint8_t flags8;

    ctx->dirty_state = 0;

    ARRAY_CREATE(&keywords, pool_datastack_create(), unsigned int, 16);
    (void)maildir_filename_get_flags(ctx, path, &flags, &keywords);
    flags8 = flags;

    recs = array_get_modifyable(&ctx->sync_recs, &count);
    for (i = 0; i < count; i++) {
        if (recs[i].uid1 != ctx->seq)
            break;

        switch (recs[i].type) {
        case MAIL_INDEX_SYNC_TYPE_FLAGS:
            mail_index_sync_flags_apply(&recs[i], &flags8);
            break;
        case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
        case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
        case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:
            mail_index_sync_keywords_apply(&recs[i], &keywords);
            break;
        case MAIL_INDEX_SYNC_TYPE_APPEND:
        case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
            i_unreached();
            break;
        }
    }

    newpath = maildir_filename_set_flags(ctx, path, flags8, &keywords);
    if (rename(path, newpath) == 0) {
        if ((flags8 & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
            ctx->dirty_state = -1;
        mbox->dirty_cur_time = ioloop_time;
        return 1;
    }
    if (errno == ENOENT)
        return 0;

    if (ENOSPACE(errno) || errno == EACCES) {
        mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_ADD,
                    MAIL_INDEX_MAIL_FLAG_DIRTY);
        ctx->dirty_state = 1;
        return 1;
    }

    mail_storage_set_critical(STORAGE(mbox->storage),
                  "rename(%s, %s) failed: %m", path, newpath);
    return -1;
}

static int
maildir_sync_record_commit_until(struct maildir_index_sync_context *ctx,
                 uint32_t last_seq)
{
    struct mail_index_sync_rec *recs;
    unsigned int i, count;
    uint32_t seq, uid;
    int expunged, flag_changed;

    recs = array_get_modifyable(&ctx->sync_recs, &count);
    for (seq = recs[0].uid1; seq <= last_seq; seq++) {
        expunged = flag_changed = FALSE;
        for (i = 0; i < count; i++) {
            if (recs[i].uid1 > seq)
                break;

            i_assert(recs[i].uid1 == seq);
            switch (recs[i].type) {
            case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
                expunged = TRUE;
                break;
            case MAIL_INDEX_SYNC_TYPE_FLAGS:
            case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
            case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
            case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:
                flag_changed = TRUE;
                break;
            case MAIL_INDEX_SYNC_TYPE_APPEND:
                i_unreached();
                break;
            }
        }

        if (mail_index_lookup_uid(ctx->view, seq, &uid) < 0)
            return -1;

        ctx->seq = seq;
        if (expunged) {
            if (maildir_file_do(ctx->mbox, uid,
                        maildir_expunge, ctx) < 0)
                return -1;
        } else if (flag_changed) {
            if (maildir_file_do(ctx->mbox, uid,
                        maildir_sync_flags, ctx) < 0)
                return -1;
        }

        for (i = count; i > 0; i--) {
            if (++recs[i-1].uid1 > recs[i-1].uid2) {
                array_delete(&ctx->sync_recs, i-1, 1);
                recs = array_get_modifyable(&ctx->sync_recs,
                                &count);
                if (count == 0) {
                    /* all sync_recs committed */
                    return 0;
                }
            }
        }
    }
    return 0;
}

static int maildir_sync_record(struct maildir_index_sync_context *ctx,
                   const struct mail_index_sync_rec *sync_rec)
{
    struct mail_index_view *view = ctx->view;
        struct mail_index_sync_rec sync_copy;

    if (sync_rec == NULL) {
        /* deinit */
        while (array_count(&ctx->sync_recs) > 0) {
            if (maildir_sync_record_commit_until(ctx,
                                 (uint32_t)-1) < 0)
                return -1;
        }
        return 0;
    }

    if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND)
        return 0; /* ignore */

    /* convert to sequences to avoid looping through huge holes in
       UID range */
        sync_copy = *sync_rec;
    if (mail_index_lookup_uid_range(view, sync_rec->uid1,
                    sync_rec->uid2,
                    &sync_copy.uid1,
                    &sync_copy.uid2) < 0)
        return -1;

    if (sync_copy.uid1 == 0) {
        /* UIDs were expunged */
        return 0;
    }

    while (array_count(&ctx->sync_recs) > 0) {
        const struct mail_index_sync_rec *rec =
            array_idx(&ctx->sync_recs, 0);

        i_assert(rec->uid1 <= sync_copy.uid1);
        if (rec->uid1 == sync_copy.uid1)
            break;

        if (maildir_sync_record_commit_until(ctx, sync_copy.uid1-1) < 0)
            return -1;
    }

    array_append(&ctx->sync_recs, &sync_copy, 1);
    return 0;
}

static int maildir_sync_index_records(struct maildir_index_sync_context *ctx)
{
    struct mail_index_sync_rec sync_rec;
    int ret;

    ret = mail_index_sync_next(ctx->sync_ctx, &sync_rec);
    if (ret <= 0)
        return ret;

    ARRAY_CREATE(&ctx->sync_recs, pool_datastack_create(),
             struct mail_index_sync_rec, 32);
    do {
        if (maildir_sync_record(ctx, &sync_rec) < 0)
            return -1;

        ret = mail_index_sync_next(ctx->sync_ctx, &sync_rec);
    } while (ret > 0);

    if (maildir_sync_record(ctx, NULL) < 0)
        return -1;
    return ret;
}

static struct maildir_sync_context *
maildir_sync_context_new(struct maildir_mailbox *mbox)
{
        struct maildir_sync_context *ctx;

    ctx = t_new(struct maildir_sync_context, 1);
    ctx->mbox = mbox;
    ctx->new_dir = t_strconcat(mbox->path, "/new", NULL);
    ctx->cur_dir = t_strconcat(mbox->path, "/cur", NULL);
    return ctx;
}

static void maildir_sync_deinit(struct maildir_sync_context *ctx)
{
    if (ctx->uidlist_sync_ctx != NULL)
        (void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
    if (ctx->index_sync_ctx != NULL)
        maildir_sync_index_abort(ctx->index_sync_ctx);
}

static int maildir_fix_duplicate(struct maildir_mailbox *mbox, const char *dir,
                 const char *old_fname)
{
    const char *new_fname, *old_path, *new_path;
    int ret = 0;

    t_push();

    old_path = t_strconcat(dir, "/", old_fname, NULL);
    new_fname = maildir_generate_tmp_filename(&ioloop_timeval);
    new_path = t_strconcat(mbox->path, "/new/", new_fname, NULL);

    if (rename(old_path, new_path) == 0) {
        i_warning("Fixed duplicate in %s: %s -> %s",
              mbox->path, old_fname, new_fname);
    } else if (errno != ENOENT) {
        mail_storage_set_critical(STORAGE(mbox->storage),
            "rename(%s, %s) failed: %m", old_path, new_path);
        ret = -1;
    }
    t_pop();

    return ret;
}

static int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
{
    struct mail_storage *storage = STORAGE(ctx->mbox->storage);
    const char *dir;
    DIR *dirp;
    string_t *src, *dest;
    struct dirent *dp;
    enum maildir_uidlist_rec_flag flags;
    unsigned int moves = 0;
    int move_new, ret = 1;

    dir = new_dir ? ctx->new_dir : ctx->cur_dir;
    dirp = opendir(dir);
    if (dirp == NULL) {
        mail_storage_set_critical(storage,
                      "opendir(%s) failed: %m", dir);
        return -1;
    }

    t_push();
    src = t_str_new(1024);
    dest = t_str_new(1024);

    move_new = new_dir && !mailbox_is_readonly(&ctx->mbox->ibox.box) &&
        !ctx->mbox->ibox.keep_recent;
    while ((dp = readdir(dirp)) != NULL) {
        if (dp->d_name[0] == '.')
            continue;

        ret = maildir_uidlist_sync_next_pre(ctx->uidlist_sync_ctx,
                            dp->d_name);
        if (ret == 0) {
            /* new file and we couldn't lock uidlist, check this
               later in next sync. */
            if (new_dir)
                ctx->mbox->last_new_mtime = 0;
            else
                ctx->mbox->dirty_cur_time = ioloop_time;
            continue;
        }
        if (ret < 0)
            break;

        flags = 0;
        if (move_new) {
            str_truncate(src, 0);
            str_truncate(dest, 0);
            str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
            str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
            if (strchr(dp->d_name, MAILDIR_INFO_SEP) == NULL) {
                str_append(dest, MAILDIR_FLAGS_FULL_SEP);
            }
            if (rename(str_c(src), str_c(dest)) == 0) {
                /* we moved it - it's \Recent for us */
                moves++;
                ctx->mbox->dirty_cur_time = ioloop_time;
                flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
                    MAILDIR_UIDLIST_REC_FLAG_RECENT;
            } else if (ENOTFOUND(errno)) {
                /* someone else moved it already */
                moves++;
                flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED;
            } else if (ENOSPACE(errno)) {
                /* not enough disk space, leave here */
                flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
                    MAILDIR_UIDLIST_REC_FLAG_RECENT;
                move_new = FALSE;
            } else {
                flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
                    MAILDIR_UIDLIST_REC_FLAG_RECENT;
                mail_storage_set_critical(storage,
                    "rename(%s, %s) failed: %m",
                    str_c(src), str_c(dest));
            }
        } else if (new_dir) {
            flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
                MAILDIR_UIDLIST_REC_FLAG_RECENT;
        }

        ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
                        dp->d_name, flags);
        if (ret <= 0) {
            if (ret < 0)
                break;

            /* possibly duplicate - try fixing it */
            if (maildir_fix_duplicate(ctx->mbox,
                          dir, dp->d_name) < 0) {
                ret = -1;
                break;
            }
        }
    }

    if (closedir(dirp) < 0) {
        mail_storage_set_critical(storage,
                      "closedir(%s) failed: %m", dir);
    }

    t_pop();
    return ret < 0 ? -1 : (moves <= MAILDIR_RENAME_RESCAN_COUNT ? 0 : 1);
}

static void
maildir_sync_update_from_header(struct maildir_mailbox *mbox)
{
    uint64_t value;

    /* FIXME: ugly, replace with extension header */
    value = mail_index_get_header(mbox->ibox.view)->sync_size;
    mbox->last_new_mtime = value & 0xffffffff;
    mbox->last_new_sync_time = value >> 32;

    mbox->last_cur_mtime =
        mail_index_get_header(mbox->ibox.view)->sync_stamp;
}

static int
maildir_sync_quick_check(struct maildir_mailbox *mbox,
             const char *new_dir, const char *cur_dir,
             int *new_changed_r, int *cur_changed_r)
{
    struct stat st;
    time_t new_mtime, cur_mtime;

    *new_changed_r = *cur_changed_r = FALSE;

    if (stat(new_dir, &st) < 0) {
        mail_storage_set_critical(STORAGE(mbox->storage),
                      "stat(%s) failed: %m", new_dir);
        return -1;
    }
    new_mtime = st.st_mtime;

    if (stat(cur_dir, &st) < 0) {
        mail_storage_set_critical(STORAGE(mbox->storage),
                      "stat(%s) failed: %m", cur_dir);
        return -1;
    }
    cur_mtime = st.st_mtime;

    /* cur stamp is kept in index, we don't have to sync if
       someone else has done it and updated the index.

       FIXME: For now we're using sync_size field as the new/ dir's stamp.
       Pretty ugly.. */
        maildir_sync_update_from_header(mbox);
    if ((mbox->dirty_cur_time == 0 && cur_mtime != mbox->last_cur_mtime) ||
        (new_mtime != mbox->last_new_mtime)) {
        /* check if the index has been updated.. */
        if (mail_index_refresh(mbox->ibox.index) < 0) {
            mail_storage_set_index_error(&mbox->ibox);
            return -1;
        }

        maildir_sync_update_from_header(mbox);
    }

    if (new_mtime != mbox->last_new_mtime ||
        new_mtime >= mbox->last_new_sync_time - MAILDIR_SYNC_SECS) {
        *new_changed_r = TRUE;
        mbox->last_new_mtime = new_mtime;
        mbox->last_new_sync_time = ioloop_time;
    }

    if (cur_mtime != mbox->last_cur_mtime ||
        (mbox->dirty_cur_time != 0 &&
         ioloop_time - mbox->dirty_cur_time > MAILDIR_SYNC_SECS)) {
        /* cur/ changed, or delayed cur/ check */
        *cur_changed_r = TRUE;
        mbox->last_cur_mtime = cur_mtime;

        mbox->dirty_cur_time =
            cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?
            cur_mtime : 0;
    }

    return 0;
}

struct maildir_index_sync_context *
maildir_sync_index_begin(struct maildir_mailbox *mbox)
{
    struct maildir_index_sync_context *sync_ctx;

    sync_ctx = i_new(struct maildir_index_sync_context, 1);
    sync_ctx->mbox = mbox;

    if (mail_index_sync_begin(mbox->ibox.index, &sync_ctx->sync_ctx,
                  &sync_ctx->view, (uint32_t)-1, (uoff_t)-1,
                  FALSE, FALSE) <= 0) {
        mail_storage_set_index_error(&mbox->ibox);
        i_free(sync_ctx);
        return NULL;
    }

    sync_ctx->keywords_sync_ctx =
        maildir_keywords_sync_init(mbox->keywords, mbox->ibox.index);
    return sync_ctx;
}

void maildir_sync_index_abort(struct maildir_index_sync_context *sync_ctx)
{
    mail_index_sync_rollback(sync_ctx->sync_ctx);
    maildir_keywords_sync_deinit(sync_ctx->keywords_sync_ctx);
    i_free(sync_ctx);
}

int maildir_sync_index_finish(struct maildir_index_sync_context *sync_ctx,
                  int partial)
{
    struct maildir_mailbox *mbox = sync_ctx->mbox;
    struct mail_index_view *view = sync_ctx->view;
    struct maildir_uidlist_iter_ctx *iter;
    struct mail_index_transaction *trans;
    const struct mail_index_header *hdr;
    const struct mail_index_record *rec;
    uint32_t seq, uid;
        enum maildir_uidlist_rec_flag uflags;
    const char *filename;
    enum mail_flags flags;
    array_t ARRAY_DEFINE(keywords, unsigned int);
    array_t ARRAY_DEFINE(idx_keywords, unsigned int);
    uint32_t uid_validity, next_uid;
    uint64_t value;
    time_t old_new_sync_time;
    int ret = 0, full_rescan = FALSE;

    i_assert(maildir_uidlist_is_locked(sync_ctx->mbox->uidlist));

    trans = mail_index_transaction_begin(view, FALSE, TRUE);
    sync_ctx->trans = trans;

    hdr = mail_index_get_header(view);
    uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
    if (uid_validity != hdr->uid_validity &&
        uid_validity != 0 && hdr->uid_validity != 0) {
        /* uidvalidity changed and mailbox isn't being initialized,
           reset mailbox so we can add all messages as new */
        mail_storage_set_critical(STORAGE(mbox->storage),
            "Maildir %s sync: UIDVALIDITY changed (%u -> %u)",
            mbox->path, hdr->uid_validity, uid_validity);

        mail_index_mark_corrupted(mbox->ibox.index);
        return -1;
    }

    seq = 0;
    ARRAY_CREATE(&keywords, pool_datastack_create(),
             unsigned int, MAILDIR_MAX_KEYWORDS);
    ARRAY_CREATE(&idx_keywords, pool_datastack_create(),
             unsigned int, MAILDIR_MAX_KEYWORDS);
    iter = maildir_uidlist_iter_init(mbox->uidlist);
    while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
        maildir_filename_get_flags(sync_ctx, filename,
                       &flags, &keywords);

        if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0 &&
            (uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 &&
            (uflags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0) {
            /* mail is recent for next session as well */
            flags |= MAIL_RECENT;
        }

    __again:
        seq++;

        if (seq > hdr->messages_count) {
            if (uid < hdr->next_uid) {
                /* most likely a race condition: we read the
                   maildir, then someone else expunged messages
                   and committed changes to index. so, this
                   message shouldn't actually exist. mark it
                   racy and check in next sync.

                   the difference between this and the later
                   check is that this one happens when messages
                   are expunged from the end */
                if ((uflags &
                    MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
                    /* partial syncing */
                    continue;
                }
                if ((uflags &
                     MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {
                    mail_storage_set_critical(
                        STORAGE(mbox->storage),
                        "Maildir %s sync: "
                        "UID < next_uid "
                        "(%u < %u, file = %s)",
                        mbox->path, uid, hdr->next_uid,
                        filename);
                    mail_index_mark_corrupted(
                        mbox->ibox.index);
                    ret = -1;
                    break;
                }
                mbox->dirty_cur_time = ioloop_time;
                maildir_uidlist_add_flags(mbox->uidlist,
                    filename,
                    MAILDIR_UIDLIST_REC_FLAG_RACING);

                seq--;
                continue;
            }

            mail_index_append(trans, uid, &seq);
            mail_index_update_flags(trans, seq, MODIFY_REPLACE,
                        flags);

            if (array_count(&keywords) > 0) {
                struct mail_keywords *kw;

                kw = mail_index_keywords_create_from_indexes(
                    trans, &keywords);
                mail_index_update_keywords(trans, seq,
                               MODIFY_REPLACE, kw);
                mail_index_keywords_free(kw);
            }
            continue;
        }

        if (mail_index_lookup(view, seq, &rec) < 0) {
            ret = -1;
            break;
        }

        if (rec->uid < uid) {
            /* expunged */
            mail_index_expunge(trans, seq);
            goto __again;
        }

        if (rec->uid > uid) {
            /* most likely a race condition: we read the
               maildir, then someone else expunged messages and
               committed changes to index. so, this message
               shouldn't actually exist. mark it racy and check
               in next sync. */
            if ((uflags &
                MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
                /* partial syncing */
                seq--;
                continue;
            }
            if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {
                mail_storage_set_critical(
                    STORAGE(mbox->storage),
                    "Maildir %s sync: "
                    "UID inserted in the middle of mailbox "
                    "(%u > %u, file = %s)",
                    mbox->path, rec->uid, uid, filename);
                mail_index_mark_corrupted(mbox->ibox.index);
                ret = -1;
                break;
            }

            mbox->dirty_cur_time = ioloop_time;
            maildir_uidlist_add_flags(mbox->uidlist, filename,
                MAILDIR_UIDLIST_REC_FLAG_RACING);

            seq--;
            continue;
        }

        if ((rec->flags & MAIL_RECENT) != 0) {
            index_mailbox_set_recent(&mbox->ibox, seq);
            if (mbox->ibox.keep_recent) {
                flags |= MAIL_RECENT;
            } else {
                mail_index_update_flags(trans, seq,
                            MODIFY_REMOVE,
                            MAIL_RECENT);
            }
        }

        if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
            /* partial syncing */
            if ((flags & MAIL_RECENT) != 0) {
                /* we last saw this mail in new/, but it's
                   not there anymore. possibly expunged,
                   make sure. */
                full_rescan = TRUE;
            }
            continue;
        }

        if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
            /* we haven't been able to update maildir with this
               record's flag changes. don't sync them. */
            continue;
        }

        if (((uint8_t)flags & ~MAIL_RECENT) !=
            (rec->flags & (MAIL_FLAGS_MASK^MAIL_RECENT))) {
            /* FIXME: this is wrong if there's pending changes in
               transaction log already. it gets fixed in next sync
               however.. */
            mail_index_update_flags(trans, seq, MODIFY_REPLACE,
                        flags);
        } else if ((flags & MAIL_RECENT) == 0 &&
               (rec->flags & MAIL_RECENT) != 0) {
            /* just remove recent flag */
            mail_index_update_flags(trans, seq, MODIFY_REMOVE,
                        MAIL_RECENT);
        }

        /* update keywords if they have changed */
        array_clear(&idx_keywords);
        if (mail_index_lookup_keywords(view, seq, &idx_keywords) < 0) {
            ret = -1;
            break;
        }
        if (!array_cmp(&keywords, &idx_keywords)) {
            struct mail_keywords *kw;

            kw = mail_index_keywords_create_from_indexes(
                trans, &keywords);
            mail_index_update_keywords(trans, seq,
                           MODIFY_REPLACE, kw);
            mail_index_keywords_free(kw);
        }
    }
    maildir_uidlist_iter_deinit(iter);
    array_free(&keywords);

    if (!partial) {
        /* expunge the rest */
        for (seq++; seq <= hdr->messages_count; seq++)
            mail_index_expunge(trans, seq);
    }

    /* now, sync the index */
        mbox->syncing_commit = TRUE;
    if (maildir_sync_index_records(sync_ctx) < 0)
        ret = -1;
    mbox->syncing_commit = FALSE;

    if (mbox->dirty_cur_time == 0 &&
        mbox->last_cur_mtime != (time_t)hdr->sync_stamp) {
        uint32_t sync_stamp = mbox->last_cur_mtime;

        mail_index_update_header(trans,
            offsetof(struct mail_index_header, sync_stamp),
            &sync_stamp, sizeof(sync_stamp), TRUE);
    }

    /* FIXME: use a header extension instead of sync_size.. */
    value = mbox->last_new_mtime;
    old_new_sync_time = hdr->sync_size >> 32;
    if (mbox->last_new_mtime >= old_new_sync_time - MAILDIR_SYNC_SECS) {
        value |= (uint64_t)mbox->last_new_sync_time << 32;
    } else {
        value |= (uint64_t)old_new_sync_time << 32;
    }
    if (value != hdr->sync_size) {
        uint64_t sync_stamp = mbox->last_new_mtime |
            ((uint64_t)mbox->last_new_sync_time << 32);

        mail_index_update_header(trans,
            offsetof(struct mail_index_header, sync_size),
            &sync_stamp, sizeof(sync_stamp), TRUE);
    }

    if (hdr->uid_validity == 0) {
        /* get the initial uidvalidity */
        if (maildir_uidlist_update(mbox->uidlist) < 0)
            ret = -1;
        uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
        if (uid_validity == 0) {
            uid_validity = ioloop_time;
            maildir_uidlist_set_uid_validity(mbox->uidlist,
                             uid_validity);
        }
    } else if (uid_validity == 0) {
        maildir_uidlist_set_uid_validity(mbox->uidlist,
                         hdr->uid_validity);
    }

    if (uid_validity != hdr->uid_validity && uid_validity != 0) {
        mail_index_update_header(trans,
            offsetof(struct mail_index_header, uid_validity),
            &uid_validity, sizeof(uid_validity), TRUE);
    }

    next_uid = maildir_uidlist_get_next_uid(mbox->uidlist);
    if (next_uid != 0 && hdr->next_uid != next_uid) {
        mail_index_update_header(trans,
            offsetof(struct mail_index_header, next_uid),
            &next_uid, sizeof(next_uid), FALSE);
    }

    if (ret < 0) {
        mail_index_transaction_rollback(trans);
        mail_index_sync_rollback(sync_ctx->sync_ctx);
    } else {
        uint32_t seq;
        uoff_t offset;

        if (mail_index_transaction_commit(trans, &seq, &offset) < 0)
            ret = -1;
        else if (seq != 0) {
            mbox->ibox.commit_log_file_seq = seq;
            mbox->ibox.commit_log_file_offset = offset;
        }
        if (mail_index_sync_commit(sync_ctx->sync_ctx) < 0)
            ret = -1;
    }
    maildir_keywords_sync_deinit(sync_ctx->keywords_sync_ctx);
        sync_ctx->keywords_sync_ctx = NULL;

    if (ret == 0) {
        mbox->ibox.commit_log_file_seq = 0;
        mbox->ibox.commit_log_file_offset = 0;
    } else {
        mail_storage_set_index_error(&mbox->ibox);
    }

    i_free(sync_ctx);
    return ret < 0 ? -1 : (full_rescan ? 0 : 1);
}

static int maildir_sync_context(struct maildir_sync_context *ctx, int forced,
                int sync_last_commit)
{
    int ret, new_changed, cur_changed;
    int full_rescan = FALSE;

    if (sync_last_commit) {
        new_changed = cur_changed = FALSE;
    } else if (!forced) {
        if (maildir_sync_quick_check(ctx->mbox,
                         ctx->new_dir, ctx->cur_dir,
                         &new_changed, &cur_changed) < 0)
            return -1;

        if (!new_changed && !cur_changed)
            return 0;
    } else {
        new_changed = cur_changed = TRUE;
    }

    /*
       Locking, locking, locking.. Wasn't maildir supposed to be lockless?

       We can get here either as beginning a real maildir sync, or when
       committing changes to maildir but a file was lost (maybe renamed).

       So, we're going to need two locks. One for index and one for
       uidlist. To avoid deadlocking do the index lock first.

       uidlist is needed only for figuring out UIDs for newly seen files,
       so theoretically we wouldn't need to lock it unless there are new
       files. It has a few problems though, assuming the index lock didn't
       already protect it (eg. in-memory indexes):

       1. Just because you see a new file which doesn't exist in uidlist
       file, doesn't mean that the file really exists anymore, or that
       your readdir() lists all new files. Meaning that this is possible:

         A: opendir(), readdir() -> new file ...
         -- new files are written to the maildir --
         B: opendir(), readdir() -> new file, lock uidlist,
        readdir() -> another new file, rewrite uidlist, unlock
         A: ... lock uidlist, readdir() -> nothing left, rewrite uidlist,
        unlock

       The second time running A didn't see the two new files. To handle
       this correctly, it must not remove the new unseen files from
       uidlist. This is possible to do, but adds extra complexity.

       2. If another process is rename()ing files while we are
       readdir()ing, it's possible that readdir() never lists some files,
       causing Dovecot to assume they were expunged. In next sync they
       would show up again, but client could have already been notified of
       that and they would show up under new UIDs, so the damage is
       already done.

       Both of the problems can be avoided if we simply lock the uidlist
       before syncing and keep it until sync is finished. Typically this
       would happen in any case, as there is the index lock..

       The second case is still a problem with external changes though,
       because maildir doesn't require any kind of locking. Luckily this
       problem rarely happens except under high amount of modifications.
    */

    if (!ctx->mbox->syncing_commit) {
        ctx->index_sync_ctx = maildir_sync_index_begin(ctx->mbox);
        if (ctx->index_sync_ctx == NULL)
            return -1;
    }

    ctx->partial = !cur_changed;
    ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, ctx->partial,
                    &ctx->uidlist_sync_ctx);
    if (ret <= 0) {
        /* failure / timeout. if forced is TRUE, we could still go
           forward and check only for renamed files, but is it worth
           the trouble? .. */
        return ret;
    }

    if (new_changed || cur_changed) {
        /* if we're going to check cur/ dir our current logic requires
           that new/ dir is checked as well. it's a good idea anyway. */
        while ((ret = maildir_scan_dir(ctx, TRUE)) > 0) {
            /* rename()d at least some files, which might have
               caused some other files to be missed. check again
               (see MAILDIR_RENAME_RESCAN_COUNT). */
        }
        if (ret < 0)
            return -1;

        if (cur_changed) {
            if (maildir_scan_dir(ctx, FALSE) < 0)
                return -1;
        }

        /* finish uidlist syncing, but keep it still locked */
        maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
    }

    if (!ctx->mbox->syncing_commit) {
        /* NOTE: index syncing here might cause a re-sync due to
           files getting lost, so this function might be called
           re-entrantly. FIXME: and that breaks in
           maildir_uidlist_sync_deinit() */
        ret = maildir_sync_index_finish(ctx->index_sync_ctx,
                        ctx->partial);
        if (ret < 0) {
            ctx->index_sync_ctx = NULL;
            return -1;
        }
        if (ret == 0)
            full_rescan = TRUE;

        i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist));
        ctx->index_sync_ctx = NULL;
    }

    ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
    ctx->uidlist_sync_ctx = NULL;

    return ret < 0 ? -1 : (full_rescan ? 0 : 1);
}

int maildir_storage_sync_force(struct maildir_mailbox *mbox)
{
        struct maildir_sync_context *ctx;
    int ret;

    ctx = maildir_sync_context_new(mbox);
    ret = maildir_sync_context(ctx, TRUE, FALSE);
    maildir_sync_deinit(ctx);
    return ret < 0 ? -1 : 0;
}

int maildir_sync_last_commit(struct maildir_mailbox *mbox)
{
        struct maildir_sync_context *ctx;
    int ret;

    if (mbox->ibox.commit_log_file_seq == 0)
        return 0;

    ctx = maildir_sync_context_new(mbox);
    ret = maildir_sync_context(ctx, FALSE, TRUE);
    maildir_sync_deinit(ctx);
    return ret < 0 ? -1 : 0;
}

struct mailbox_sync_context *
maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
    struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
    struct maildir_sync_context *ctx;
    int ret = 0;

    if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
        mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
        ioloop_time) {
        mbox->ibox.sync_last_check = ioloop_time;

        ctx = maildir_sync_context_new(mbox);
        ret = maildir_sync_context(ctx, FALSE, FALSE);
        maildir_sync_deinit(ctx);

        i_assert(!maildir_uidlist_is_locked(mbox->uidlist));

        if (ret == 0) {
            /* lost some files from new/, see if thery're in cur/ */
            ret = maildir_storage_sync_force(mbox);
        }
    }

    return index_mailbox_sync_init(box, flags, ret < 0);
}

int maildir_sync_is_synced(struct maildir_mailbox *mbox)
{
    const char *new_dir, *cur_dir;
    int ret, new_changed, cur_changed;

    t_push();
    new_dir = t_strconcat(mbox->path, "/new", NULL);
    cur_dir = t_strconcat(mbox->path, "/cur", NULL);

    ret = maildir_sync_quick_check(mbox, new_dir, cur_dir,
                       &new_changed, &cur_changed);
    t_pop();
    return ret < 0 ? -1 : (!new_changed && !cur_changed);
}