index/maildir/maildir-sync.c

	maildir-sync.c revision 8a3d609fdd84f5938c82e8e7eeb84a24ab41b317
45312f52ff3a3d4c137447be4c7556500c2f8bf2Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen/*
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   Here's a description of how we handle Maildir synchronization and
08d6658a4e2ec8104cd1307f6baa75fdb07a24f8Mark Washenberger   it's problems:
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   We want to be as efficient as we can. The most efficient way to
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   check if changes have occured is to stat() the new/ and cur/
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   directories and uidlist file - if their mtimes haven't changed,
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   there's no changes and we don't need to do anything.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   Problem 1: Multiple changes can happen within a single second -
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   nothing guarantees that once we synced it, someone else didn't just
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   then make a modification. Such modifications wouldn't get noticed
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   until a new modification occured later.
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   Problem 2: Syncing cur/ directory is much more costly than syncing
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   new/. Moving mails from new/ to cur/ will always change mtime of
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   cur/ causing us to sync it as well.
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   Problem 3: We may not be able to move mail from new/ to cur/
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   because we're out of quota, or simply because we're accessing a
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   read-only mailbox.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen   MAILDIR_SYNC_SECS
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen   -----------------
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen   Several checks below use MAILDIR_SYNC_SECS, which should be maximum
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen   clock drift between all computers accessing the maildir (eg. via
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   NFS), rounded up to next second. Our default is 1 second, since
356303df200c991580bd24041996a070ad08c05eTimo Sirainen   everyone should be using NTP.
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen   Note that setting it to 0 works only if there's only one computer
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen   accessing the maildir. It's practically impossible to make two
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen   clocks _exactly_ synchronized.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   It might be possible to only use file server's clock by looking at
d9b73f3eb3d5f608dfd438cee89ce2b27547173fTimo Sirainen   the atime field, but I don't know how well that would actually work.
00bde9ae9eab9e720462bf6ec9a4dd85e88c3bbfTimo Sirainen
00bde9ae9eab9e720462bf6ec9a4dd85e88c3bbfTimo Sirainen   cur directory
00bde9ae9eab9e720462bf6ec9a4dd85e88c3bbfTimo Sirainen   -------------
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   We have dirty_cur_time variable which is set to cur/ directory's
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen   mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   synchronized the directory.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   When dirty_cur_time is non-zero, we don't synchronize the cur/
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   directory until
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen      a) cur/'s mtime changes
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen      b) opening a mail fails with ENOENT
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen      c) time() > dirty_cur_time + MAILDIR_SYNC_SECS
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   This allows us to modify the maildir multiple times without having
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   to sync it at every change. The sync will eventually be done to
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   make sure we didn't miss any external changes.
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   The dirty_cur_time is set when:
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen      - we change message flags
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen      - we expunge messages
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen      - we move mail from new/ to cur/
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen      - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   It's unset when we do the final syncing, ie. when mtime is
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   older than time() - MAILDIR_SYNC_SECS.
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   new directory
3b94ff5951db4d4eddb7a80ed4e3f61207202635Timo Sirainen   -------------
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   it. dirty_cur_time-like feature might save us a few syncs, but
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   that might break a client which saves a mail in one connection and
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   tries to fetch it in another one. new/ directory is almost always
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   empty, so syncing it should be very fast anyway. Actually this can
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   still happen if we sync only new/ dir while another client is also
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   moving mails from it to cur/ - it takes us a while to see them.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   That's pretty unlikely to happen however, and only way to fix it
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   would be to always synchronize cur/ after new/.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen   Normally we move all mails from new/ to cur/ whenever we sync it. If
22535a9e685e29214082878e37a267157044618eTimo Sirainen   it's not possible for some reason, we mark the mail with "probably
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen   exists in new/ directory" flag.
22535a9e685e29214082878e37a267157044618eTimo Sirainen
22535a9e685e29214082878e37a267157044618eTimo Sirainen   If rename() still fails because of ENOSPC or EDQUOT, we still save
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   the flag changes in index with dirty-flag on. When moving the mail
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   to cur/ directory, or when we notice it's already moved there, we
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   apply the flag changes to the filename, rename it and remove the
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   dirty flag. If there's dirty flags, this should be tried every time
356303df200c991580bd24041996a070ad08c05eTimo Sirainen   after expunge or when closing the mailbox.
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen   uidlist
22535a9e685e29214082878e37a267157044618eTimo Sirainen   -------
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   This file contains UID <-> filename mappings. It's updated only when
e15f1d736c225c7ce6f3d08a37c1b2ae66b57c50Timo Sirainen   new mail arrives, so it may contain filenames that have already been
e15f1d736c225c7ce6f3d08a37c1b2ae66b57c50Timo Sirainen   deleted. Updating is done by getting uidlist.lock file, writing the
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   whole uidlist into it and rename()ing it over the old uidlist. This
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   means there's no need to lock the file for reading.
e15f1d736c225c7ce6f3d08a37c1b2ae66b57c50Timo Sirainen
e15f1d736c225c7ce6f3d08a37c1b2ae66b57c50Timo Sirainen   Whenever uidlist is rewritten, it's mtime must be larger than the old
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   one's. Use utime() before rename() if needed. Note that inode checking
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   wouldn't have been sufficient as inode numbers can be reused.
356303df200c991580bd24041996a070ad08c05eTimo Sirainen
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen   This file is usually read the first time you need to know filename for
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   given UID. After that it's not re-read unless new mails come that we
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   don't know about.
356303df200c991580bd24041996a070ad08c05eTimo Sirainen
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen   broken clients
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen   --------------
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen
22535a9e685e29214082878e37a267157044618eTimo Sirainen   Originally the middle identifier in Maildir filename was specified
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   only as <process id>_<delivery counter>. That however created a
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   problem with randomized PIDs which made it possible that the same
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   PID was reused within one second.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
22535a9e685e29214082878e37a267157044618eTimo Sirainen   So if within one second a mail was delivered, MUA moved it to cur/
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen   and another mail was delivered by a new process using same PID as
22535a9e685e29214082878e37a267157044618eTimo Sirainen   the first one, we likely ended up overwriting the first mail when
22535a9e685e29214082878e37a267157044618eTimo Sirainen   the second mail was moved over it.
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   Nowadays everyone should be giving a bit more specific identifier,
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen   for example include microseconds in it which Dovecot does.
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen   There's a simple way to prevent this from happening in some cases:
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen   Don't move the mail from new/ to cur/ if it's mtime is >= time() -
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   MAILDIR_SYNC_SECS. The second delivery's link() call then fails
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   because the file is already in new/, and it will then use a
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   different filename. There's a few problems with this however:
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen      - it requires extra stat() call which is unneeded extra I/O
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen      - another MUA might still move the mail to cur/
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen      - if first file's flags are modified by either Dovecot or another
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen        MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen    but that'd be ugly)
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen   Because this is useful only for very few people and it requires
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen   extra I/O, I decided not to implement this. It should be however
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen   quite easy to do since we need to be able to deal with files in new/
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen   in any case.
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   It's also possible to never accidentally overwrite a mail by using
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   link() + unlink() rather than rename(). This however isn't very
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   good idea as it introduces potential race conditions when multiple
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   clients are accessing the mailbox:
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   Trying to move the same mail from new/ to cur/ at the same time:
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen      a) Client 1 uses slightly different filename than client 2,
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen         for example one sets read-flag on but the other doesn't.
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen     You have the same mail duplicated now.
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen      b) Client 3 sees the mail between Client 1's and 2's link() calls
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen         and changes it's flag. You have the same mail duplicated now.
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen   And it gets worse when they're unlink()ing in cur/ directory:
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen      c) Client 1 changes mails's flag and client 2 changes it back
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen         between 1's link() and unlink(). The mail is now expunged.
dd93aba1901a457346990f49c54a738947dc7128Timo Sirainen
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen      d) If you try to deal with the duplicates by unlink()ing another
806cb455553b71934314da98f1b4a694a3aa152eTimo Sirainen         one of them, you might end up unlinking both of them.
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen   So, what should we do then if we notice a duplicate? First of all,
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen   it might not be a duplicate at all, readdir() might have just
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen   returned it twice because it was just renamed. What we should do is
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen   create a completely new base name for it and rename() it to that.
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen   If the call fails with ENOENT, it only means that it wasn't a
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen   duplicate after all.
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen*/
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen#include "lib.h"
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen#include "ioloop.h"
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen#include "buffer.h"
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen#include "hash.h"
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen#include "str.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "maildir-storage.h"
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen#include "maildir-uidlist.h"
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen#include <stdio.h>
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen#include <unistd.h>
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include <dirent.h>
#include <sys/stat.h>

#define MAILDIR_SYNC_SECS 1

#define MAILDIR_FILENAME_FLAG_FOUND 128

struct maildir_sync_context {
        struct index_mailbox *ibox;
    const char *new_dir, *cur_dir;
    int partial;

        struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
};

struct maildir_index_sync_context {
        struct index_mailbox *ibox;
        struct mail_index_view *view;
    struct mail_index_sync_ctx *sync_ctx;

    struct mail_index_sync_rec sync_rec;
    uint32_t seq;
};

static int maildir_expunge(struct index_mailbox *ibox, const char *path,
               void *context __attr_unused__)
{
    if (unlink(path) == 0) {
        ibox->dirty_cur_time = ioloop_time;
        return 1;
    }
    if (errno == ENOENT)
        return 0;

    mail_storage_set_critical(ibox->box.storage,
                  "unlink(%s) failed: %m", path);
    return -1;
}

static int maildir_sync_flags(struct index_mailbox *ibox, const char *path,
                  void *context)
{
        struct maildir_index_sync_context *ctx = context;
    const char *newpath;
    enum mail_flags flags;
    uint8_t flags8;
        keywords_mask_t keywords;

    (void)maildir_filename_get_flags(path, &flags, keywords);

    flags8 = flags;
    mail_index_sync_flags_apply(&ctx->sync_rec, &flags8, keywords);

    newpath = maildir_filename_set_flags(path, flags8, keywords);
    if (rename(path, newpath) == 0) {
        ibox->dirty_cur_time = ioloop_time;
        return 1;
    }
    if (errno == ENOENT)
        return 0;

    if (ENOSPACE(errno)) {
        if (mail_index_sync_set_dirty(ctx->sync_ctx, ctx->seq) < 0)
            return -1;
        return 1;
    }

    mail_storage_set_critical(ibox->box.storage,
                  "rename(%s, %s) failed: %m", path, newpath);
    return -1;
}

static int maildir_sync_record(struct index_mailbox *ibox,
                               struct maildir_index_sync_context *ctx)
{
    struct mail_index_sync_rec *sync_rec = &ctx->sync_rec;
    struct mail_index_view *view = ctx->view;
    uint32_t seq, uid;

    switch (sync_rec->type) {
    case MAIL_INDEX_SYNC_TYPE_APPEND:
        break;
    case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
        for (seq = sync_rec->seq1; seq <= sync_rec->seq2; seq++) {
            if (mail_index_lookup_uid(view, seq, &uid) < 0)
                return -1;
            if (maildir_file_do(ibox, uid, maildir_expunge,
                        NULL) < 0)
                return -1;
        }
        break;
    case MAIL_INDEX_SYNC_TYPE_FLAGS:
                ctx->seq = sync_rec->seq1;
        for (; ctx->seq <= sync_rec->seq2; ctx->seq++) {
            if (mail_index_lookup_uid(view, ctx->seq, &uid) < 0)
                return -1;
            if (maildir_file_do(ibox, uid, maildir_sync_flags,
                        ctx) < 0)
                return -1;
        }
        break;
    }

    return 0;
}

int maildir_sync_last_commit(struct index_mailbox *ibox)
{
    struct maildir_index_sync_context ctx;
    int ret;

    if (ibox->commit_log_file_seq == 0)
        return 0;

    memset(&ctx, 0, sizeof(ctx));
    ctx.ibox = ibox;

    ret = mail_index_sync_begin(ibox->index, &ctx.sync_ctx, &ctx.view,
                    ibox->commit_log_file_seq,
                    ibox->commit_log_file_offset);
    if (ret > 0) {
        while ((ret = mail_index_sync_next(ctx.sync_ctx,
                           &ctx.sync_rec)) > 0) {
            if (maildir_sync_record(ibox, &ctx) < 0) {
                ret = -1;
                break;
            }
        }
        if (mail_index_sync_end(ctx.sync_ctx, 0, 0) < 0)
            ret = -1;
    }

    if (ret == 0) {
        ibox->commit_log_file_seq = 0;
        ibox->commit_log_file_offset = 0;
    } else {
        mail_storage_set_index_error(ibox);
    }
    return ret;
}

static struct maildir_sync_context *
maildir_sync_context_new(struct index_mailbox *ibox)
{
        struct maildir_sync_context *ctx;

    ctx = t_new(struct maildir_sync_context, 1);
    ctx->ibox = ibox;
    ctx->new_dir = t_strconcat(ibox->path, "/new", NULL);
    ctx->cur_dir = t_strconcat(ibox->path, "/cur", NULL);
    return ctx;
}

static void maildir_sync_deinit(struct maildir_sync_context *ctx)
{
    if (ctx->uidlist_sync_ctx != NULL)
        (void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
}

static int maildir_fix_duplicate(struct index_mailbox *ibox, const char *dir,
                 const char *old_fname)
{
    const char *new_fname, *old_path, *new_path;
    int ret = 0;

    t_push();

    old_path = t_strconcat(dir, "/", old_fname, NULL);
    new_fname = maildir_generate_tmp_filename(&ioloop_timeval);
    new_path = t_strconcat(ibox->path, "/new/", new_fname, NULL);

    if (rename(old_path, new_path) == 0) {
        i_warning("Fixed duplicate in %s: %s -> %s",
              ibox->path, old_fname, new_fname);
    } else if (errno != ENOENT) {
        mail_storage_set_critical(ibox->box.storage,
            "rename(%s, %s) failed: %m", old_path, new_path);
        ret = -1;
    }
    t_pop();

    return ret;
}

static int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
{
    struct mail_storage *storage = ctx->ibox->box.storage;
    const char *dir;
    DIR *dirp;
    string_t *src, *dest;
    struct dirent *dp;
        enum maildir_uidlist_rec_flag flags;
    int move_new, ret = 1;

    src = t_str_new(1024);
    dest = t_str_new(1024);

    dir = new_dir ? ctx->new_dir : ctx->cur_dir;
    dirp = opendir(dir);
    if (dirp == NULL) {
        mail_storage_set_critical(storage,
                      "opendir(%s) failed: %m", dir);
        return -1;
    }

    move_new = new_dir && !mailbox_is_readonly(&ctx->ibox->box);
    while ((dp = readdir(dirp)) != NULL) {
        if (dp->d_name[0] == '.')
            continue;

        ret = maildir_uidlist_sync_next_pre(ctx->uidlist_sync_ctx,
                            dp->d_name);
        if (ret == 0) {
            if (new_dir)
                ctx->ibox->last_new_mtime = 0;
            else
                ctx->ibox->dirty_cur_time = ioloop_time;
            continue;
        }
        if (ret < 0)
            break;

        flags = 0;
        if (move_new) {
            str_truncate(src, 0);
            str_truncate(dest, 0);
            str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
            str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
            if (strchr(dp->d_name, ':') == NULL)
                str_append(dest, ":2,");
            if (rename(str_c(src), str_c(dest)) == 0) {
                /* we moved it - it's \Recent for us */
                                ctx->ibox->dirty_cur_time = ioloop_time;
                flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
                    MAILDIR_UIDLIST_REC_FLAG_RECENT;
            } else if (ENOTFOUND(errno)) {
                /* someone else moved it already */
                flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED;
            } else if (ENOSPACE(errno)) {
                /* not enough disk space, leave here */
                flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
                    MAILDIR_UIDLIST_REC_FLAG_RECENT;
                move_new = FALSE;
            } else {
                flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
                    MAILDIR_UIDLIST_REC_FLAG_RECENT;
                mail_storage_set_critical(storage,
                    "rename(%s, %s) failed: %m",
                    str_c(src), str_c(dest));
            }
        } else if (new_dir) {
            flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
                MAILDIR_UIDLIST_REC_FLAG_RECENT;
        }

        ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
                        dp->d_name, flags);
        if (ret <= 0) {
            if (ret < 0)
                break;

            /* possibly duplicate - try fixing it */
            if (maildir_fix_duplicate(ctx->ibox,
                          dir, dp->d_name) < 0) {
                ret = -1;
                break;
            }
        }
    }

    if (closedir(dirp) < 0) {
        mail_storage_set_critical(storage,
                      "closedir(%s) failed: %m", dir);
    }
    return ret < 0 ? -1 : 0;
}

static int maildir_sync_quick_check(struct maildir_sync_context *ctx,
                    int *new_changed_r, int *cur_changed_r)
{
    const struct mail_index_header *hdr;
    struct index_mailbox *ibox = ctx->ibox;
    struct stat st;
    time_t new_mtime, cur_mtime;

    *new_changed_r = *cur_changed_r = FALSE;

    if (stat(ctx->new_dir, &st) < 0) {
        mail_storage_set_critical(ibox->box.storage,
                      "stat(%s) failed: %m", ctx->new_dir);
        return -1;
    }
    new_mtime = st.st_mtime;

    if (stat(ctx->cur_dir, &st) < 0) {
        mail_storage_set_critical(ibox->box.storage,
                      "stat(%s) failed: %m", ctx->cur_dir);
        return -1;
    }
    cur_mtime = st.st_mtime;

    if (ibox->last_cur_mtime == 0) {
        /* first sync in this session, get cur stamp from index */
        if (mail_index_get_header(ibox->view, &hdr) == 0)
            ibox->last_cur_mtime = hdr->sync_stamp;
    }

    if (new_mtime != ibox->last_new_mtime ||
        new_mtime >= ibox->last_new_sync_time - MAILDIR_SYNC_SECS) {
        *new_changed_r = TRUE;
        ibox->last_new_mtime = new_mtime;
        ibox->last_new_sync_time = ioloop_time;
    }

    if (cur_mtime != ibox->last_cur_mtime ||
        (ibox->dirty_cur_time != 0 &&
         ioloop_time - ibox->dirty_cur_time > MAILDIR_SYNC_SECS)) {
        /* cur/ changed, or delayed cur/ check */
        *cur_changed_r = TRUE;
        ibox->last_cur_mtime = cur_mtime;

        ibox->dirty_cur_time =
            cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?
            cur_mtime : 0;
    }

    return 0;
}

static int maildir_sync_index(struct maildir_sync_context *ctx)
{
    struct index_mailbox *ibox = ctx->ibox;
    struct maildir_index_sync_context sync_ctx;
    struct maildir_uidlist_iter_ctx *iter;
    struct mail_index_transaction *trans;
    struct mail_index_view *view;
    const struct mail_index_header *hdr;
    const struct mail_index_record *rec;
    uint32_t seq, uid;
        enum maildir_uidlist_rec_flag uflags;
    const char *filename;
    enum mail_flags flags;
    keywords_mask_t keywords;
    uint32_t sync_stamp;
    int ret;

    memset(&sync_ctx, 0, sizeof(sync_ctx));
    sync_ctx.ibox = ibox;

    if (mail_index_sync_begin(ibox->index, &sync_ctx.sync_ctx, &view,
                  (uint32_t)-1, (uoff_t)-1) <= 0) {
        mail_storage_set_index_error(ibox);
        return -1;
    }
    sync_ctx.view = view;

    ret = mail_index_get_header(view, &hdr);
    i_assert(ret == 0); /* view is locked, can't happen */

    trans = mail_index_transaction_begin(view, FALSE);

    seq = 0;
    iter = maildir_uidlist_iter_init(ibox->uidlist);
    while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
        maildir_filename_get_flags(filename, &flags, keywords);

    __again:
        seq++;
        if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
            /* partial syncing */
            continue;
        }

        if (seq > hdr->messages_count) {
            if (uid < hdr->next_uid) {
                /* message not in index, but next_uid header
                   is updated? shouldn't really happen.. */
                mail_storage_set_critical(ibox->box.storage,
                    "Maildir sync: UID < next_uid "
                    "(%u < %u, file = %s)",
                    uid, hdr->next_uid, filename);
                mail_index_mark_corrupted(ibox->index);
                ret = -1;
                break;
            }

            mail_index_append(trans, uid, &seq);
            mail_index_update_flags(trans, seq, MODIFY_REPLACE,
                        flags, keywords);
            continue;
        }

        if (mail_index_lookup(view, seq, &rec) < 0) {
            mail_storage_set_index_error(ibox);
            ret = -1;
            break;
        }

        if (rec->uid < uid) {
            /* expunged */
            mail_index_expunge(trans, seq);
            goto __again;
        }

        if (rec->uid > uid) {
            /* new UID in the middle of the mailbox -
               shouldn't happen */
            mail_storage_set_critical(ibox->box.storage,
                "Maildir sync: UID inserted in the middle "
                "of mailbox (%u > %u, file = %s)",
                rec->uid, uid, filename);
            mail_index_mark_corrupted(ibox->index);
            ret = -1;
            break;
        }

        if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
            /* we haven't been able to update maildir with this
               record's flag changes. don't sync them. */
            continue;
        }

        maildir_filename_get_flags(filename, &flags, keywords);
        if ((uint8_t)flags != (rec->flags & MAIL_FLAGS_MASK) ||
            memcmp(keywords, rec->keywords,
               INDEX_KEYWORDS_BYTE_COUNT) != 0) {
            /* FIXME: this is wrong if there's syncs later.
               it gets fixed in next sync however.. */
            mail_index_update_flags(trans, seq, MODIFY_REPLACE,
                        flags, keywords);
        }
    }
    maildir_uidlist_iter_deinit(iter);

    if (!ctx->partial) {
        /* expunge the rest */
        for (seq++; seq <= hdr->messages_count; seq++)
            mail_index_expunge(trans, seq);
    }

    if (ret < 0)
        mail_index_transaction_rollback(trans);
    else {
        uint32_t seq;
        uoff_t offset;

        if (mail_index_transaction_commit(trans, &seq, &offset) < 0)
            mail_storage_set_index_error(ibox);
        else if (seq != 0) {
            ibox->commit_log_file_seq = seq;
            ibox->commit_log_file_offset = offset;
        }
    }

    /* now, sync the index */
    while ((ret = mail_index_sync_next(sync_ctx.sync_ctx,
                       &sync_ctx.sync_rec)) > 0) {
        if (maildir_sync_record(ibox, &sync_ctx) < 0) {
            ret = -1;
            break;
        }
    }

    sync_stamp = ibox->dirty_cur_time != 0 ? 0 : ibox->last_cur_mtime;
    if (mail_index_sync_end(sync_ctx.sync_ctx, sync_stamp, 0) < 0)
        ret = -1;

    if (ret == 0) {
        ibox->commit_log_file_seq = 0;
        ibox->commit_log_file_offset = 0;
    } else {
        mail_storage_set_index_error(ibox);
    }

    return ret;
}

static int maildir_sync_context(struct maildir_sync_context *ctx)
{
    int ret, new_changed, cur_changed;

    if (maildir_sync_quick_check(ctx, &new_changed, &cur_changed) < 0)
        return -1;

    if (!new_changed && !cur_changed)
        return 0;

    ctx->partial = !cur_changed;
    ctx->uidlist_sync_ctx =
        maildir_uidlist_sync_init(ctx->ibox->uidlist, ctx->partial);

    if (maildir_scan_dir(ctx, TRUE) < 0)
        return -1;
    if (cur_changed) {
        if (maildir_scan_dir(ctx, FALSE) < 0)
            return -1;
    }

    /* finish uidlist syncing, but keep it still locked */
    if (maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx)) {
        if (maildir_sync_index(ctx) < 0)
            return -1;
    }

    ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
        ctx->uidlist_sync_ctx = NULL;

    return ret;
}

static int maildir_sync_context_readonly(struct maildir_sync_context *ctx)
{
    int ret;

    ctx->uidlist_sync_ctx =
        maildir_uidlist_sync_init(ctx->ibox->uidlist, FALSE);

    if (maildir_scan_dir(ctx, TRUE) < 0)
        return -1;
    if (maildir_scan_dir(ctx, FALSE) < 0)
        return -1;

    ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
        ctx->uidlist_sync_ctx = NULL;

    return ret;
}

int maildir_storage_sync_readonly(struct index_mailbox *ibox)
{
        struct maildir_sync_context *ctx;
    int ret;

    ctx = maildir_sync_context_new(ibox);
    ret = maildir_sync_context_readonly(ctx);
    maildir_sync_deinit(ctx);
    return ret;
}

int maildir_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
    struct index_mailbox *ibox = (struct index_mailbox *)box;
    struct maildir_sync_context *ctx;
    int ret;

    if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
        ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
        ibox->sync_last_check = ioloop_time;

        ctx = maildir_sync_context_new(ibox);
        ret = maildir_sync_context(ctx);
        maildir_sync_deinit(ctx);

        if (ret < 0)
            return -1;
    }

    return index_storage_sync(box, flags);
}