maildir-sync.c revision ea9fd7f876643e985946a2563140359064819b8e
08b30498acefc69e223baf7eda6429be98cc3a10Timo Sirainen/* Copyright (c) 2004-2009 Dovecot authors, see the included COPYING file */
08b30498acefc69e223baf7eda6429be98cc3a10Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/*
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen Here's a description of how we handle Maildir synchronization and
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it's problems:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen We want to be as efficient as we can. The most efficient way to
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen check if changes have occurred is to stat() the new/ and cur/
08b30498acefc69e223baf7eda6429be98cc3a10Timo Sirainen directories and uidlist file - if their mtimes haven't changed,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen there's no changes and we don't need to do anything.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Problem 1: Multiple changes can happen within a single second -
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen nothing guarantees that once we synced it, someone else didn't just
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen then make a modification. Such modifications wouldn't get noticed
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen until a new modification occurred later.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen Problem 2: Syncing cur/ directory is much more costly than syncing
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen new/. Moving mails from new/ to cur/ will always change mtime of
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cur/ causing us to sync it as well.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Problem 3: We may not be able to move mail from new/ to cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen because we're out of quota, or simply because we're accessing a
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen read-only mailbox.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen MAILDIR_SYNC_SECS
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen -----------------
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen Several checks below use MAILDIR_SYNC_SECS, which should be maximum
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen clock drift between all computers accessing the maildir (eg. via
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen NFS), rounded up to next second. Our default is 1 second, since
2d3aac5be07b96f72cf0551fac35ac74a4f07770Timo Sirainen everyone should be using NTP.
2d3aac5be07b96f72cf0551fac35ac74a4f07770Timo Sirainen
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen Note that setting it to 0 works only if there's only one computer
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen accessing the maildir. It's practically impossible to make two
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen clocks _exactly_ synchronized.
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen It might be possible to only use file server's clock by looking at
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen the atime field, but I don't know how well that would actually work.
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen
a205d315b0978985ba77d871f44e4a98273612e6Timo Sirainen cur directory
a205d315b0978985ba77d871f44e4a98273612e6Timo Sirainen -------------
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen We have dirty_cur_time variable which is set to cur/ directory's
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen synchronized the directory.
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen When dirty_cur_time is non-zero, we don't synchronize the cur/
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen directory until
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen a) cur/'s mtime changes
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen b) opening a mail fails with ENOENT
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen c) time() > dirty_cur_time + MAILDIR_SYNC_SECS
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen This allows us to modify the maildir multiple times without having
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen to sync it at every change. The sync will eventually be done to
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen make sure we didn't miss any external changes.
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen The dirty_cur_time is set when:
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen - we change message flags
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen - we expunge messages
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen - we move mail from new/ to cur/
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen It's unset when we do the final syncing, ie. when mtime is
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen older than time() - MAILDIR_SYNC_SECS.
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen new directory
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen -------------
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen it. dirty_cur_time-like feature might save us a few syncs, but
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen that might break a client which saves a mail in one connection and
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen tries to fetch it in another one. new/ directory is almost always
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen empty, so syncing it should be very fast anyway. Actually this can
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen still happen if we sync only new/ dir while another client is also
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen moving mails from it to cur/ - it takes us a while to see them.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen That's pretty unlikely to happen however, and only way to fix it
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen would be to always synchronize cur/ after new/.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen Normally we move all mails from new/ to cur/ whenever we sync it. If
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen it's not possible for some reason, we mark the mail with "probably
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen exists in new/ directory" flag.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen If rename() still fails because of ENOSPC or EDQUOT, we still save
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen the flag changes in index with dirty-flag on. When moving the mail
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen to cur/ directory, or when we notice it's already moved there, we
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen apply the flag changes to the filename, rename it and remove the
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen dirty flag. If there's dirty flags, this should be tried every time
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen after expunge or when closing the mailbox.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen uidlist
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen -------
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen This file contains UID <-> filename mappings. It's updated only when
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen new mail arrives, so it may contain filenames that have already been
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen deleted. Updating is done by getting uidlist.lock file, writing the
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen whole uidlist into it and rename()ing it over the old uidlist. This
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen means there's no need to lock the file for reading.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen Whenever uidlist is rewritten, it's mtime must be larger than the old
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen one's. Use utime() before rename() if needed. Note that inode checking
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen wouldn't have been sufficient as inode numbers can be reused.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen This file is usually read the first time you need to know filename for
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen given UID. After that it's not re-read unless new mails come that we
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen don't know about.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen broken clients
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen --------------
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen Originally the middle identifier in Maildir filename was specified
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen only as <process id>_<delivery counter>. That however created a
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen problem with randomized PIDs which made it possible that the same
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen PID was reused within one second.
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen So if within one second a mail was delivered, MUA moved it to cur/
f0569d9fbb25c8437760be69f194595a841ad711Timo Sirainen and another mail was delivered by a new process using same PID as
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen the first one, we likely ended up overwriting the first mail when
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen the second mail was moved over it.
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen Nowadays everyone should be giving a bit more specific identifier,
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen for example include microseconds in it which Dovecot does.
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen There's a simple way to prevent this from happening in some cases:
dda2c506c8fc8ac2f88272de4523ded42baa0aa0Timo Sirainen Don't move the mail from new/ to cur/ if it's mtime is >= time() -
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen MAILDIR_SYNC_SECS. The second delivery's link() call then fails
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen because the file is already in new/, and it will then use a
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen different filename. There's a few problems with this however:
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen - it requires extra stat() call which is unneeded extra I/O
d8b77aef97e89f1ccc5cbdaef77be9052279e35fTimo Sirainen - another MUA might still move the mail to cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - if first file's flags are modified by either Dovecot or another
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen but that'd be ugly)
907723f35f4d3dfc774ca42d00a8a7b8ef90dd5dTimo Sirainen
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen Because this is useful only for very few people and it requires
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen extra I/O, I decided not to implement this. It should be however
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen quite easy to do since we need to be able to deal with files in new/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen in any case.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen It's also possible to never accidentally overwrite a mail by using
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen link() + unlink() rather than rename(). This however isn't very
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen good idea as it introduces potential race conditions when multiple
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen clients are accessing the mailbox:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Trying to move the same mail from new/ to cur/ at the same time:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen a) Client 1 uses slightly different filename than client 2,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for example one sets read-flag on but the other doesn't.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen You have the same mail duplicated now.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen b) Client 3 sees the mail between Client 1's and 2's link() calls
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen and changes it's flag. You have the same mail duplicated now.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen And it gets worse when they're unlink()ing in cur/ directory:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen c) Client 1 changes mails's flag and client 2 changes it back
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen between 1's link() and unlink(). The mail is now expunged.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d371507847d62ba311b4bcc23d18f45c3d0f1a38Timo Sirainen d) If you try to deal with the duplicates by unlink()ing another
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen one of them, you might end up unlinking both of them.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen So, what should we do then if we notice a duplicate? First of all,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it might not be a duplicate at all, readdir() might have just
08b30498acefc69e223baf7eda6429be98cc3a10Timo Sirainen returned it twice because it was just renamed. What we should do is
08b30498acefc69e223baf7eda6429be98cc3a10Timo Sirainen create a completely new base name for it and rename() it to that.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen If the call fails with ENOENT, it only means that it wasn't a
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen duplicate after all.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen*/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "ioloop.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "array.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "buffer.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "hash.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "str.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "eacces-error.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "nfs-workarounds.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "maildir-storage.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "maildir-uidlist.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "maildir-filename.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "maildir-sync.h"
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen#include <stdio.h>
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen#include <stddef.h>
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen#include <stdlib.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <unistd.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <dirent.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <sys/stat.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen#define MAILDIR_FILENAME_FLAG_FOUND 128
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen/* When rename()ing many files from new/ to cur/, it's possible that next
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen readdir() skips some files. we don't of course wish to lose them, so we
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen go and rescan the new/ directory again from beginning until no files are
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen left. This value is just an optimization to avoid checking the directory
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen twice unneededly. usually only NFS is the problem case. 1 is the safest
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen bet here, but I guess 5 will do just fine too. */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen#define MAILDIR_RENAME_RESCAN_COUNT 5
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen/* This is mostly to avoid infinite looping when rename() destination already
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen exists as the hard link of the file itself. */
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen#define MAILDIR_SCAN_DIR_MAX_COUNT 5
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen#define DUPE_LINKS_DELETE_SECS 30
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainenstruct maildir_sync_context {
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen struct maildir_mailbox *mbox;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen const char *new_dir, *cur_dir;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen enum mailbox_sync_flags flags;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen time_t last_touch, last_notify;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen struct maildir_index_sync_context *index_sync_ctx;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen unsigned int partial:1;
a60c1c1fca85402e6fccbf3ae0784b7179ae186cTimo Sirainen unsigned int locked:1;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen};
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainenvoid maildir_sync_notify(struct maildir_sync_context *ctx)
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen{
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen time_t now;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (ctx == NULL) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen /* we got here from maildir-save.c. it has no
325f4573edfa5b751832ac01023f3e81be992bf0Timo Sirainen maildir_sync_context, */
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen return;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen }
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen now = time(NULL);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (now - ctx->last_touch > MAILDIR_LOCK_TOUCH_SECS && ctx->locked) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen (void)maildir_uidlist_lock_touch(ctx->mbox->uidlist);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen ctx->last_touch = now;
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen }
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (now - ctx->last_notify > MAIL_STORAGE_STAYALIVE_SECS) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen struct mailbox *box = &ctx->mbox->ibox.box;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen if (box->storage->callbacks.notify_ok != NULL) {
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen box->storage->callbacks.
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen notify_ok(box, "Hang in there..",
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen box->storage->callback_context);
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen }
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen ctx->last_notify = now;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen }
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen}
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainenstatic struct maildir_sync_context *
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainenmaildir_sync_context_new(struct maildir_mailbox *mbox,
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen enum mailbox_sync_flags flags)
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen{
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen struct maildir_sync_context *ctx;
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen ctx = t_new(struct maildir_sync_context, 1);
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen ctx->mbox = mbox;
a60c1c1fca85402e6fccbf3ae0784b7179ae186cTimo Sirainen ctx->new_dir = t_strconcat(mbox->ibox.box.path, "/new", NULL);
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen ctx->cur_dir = t_strconcat(mbox->ibox.box.path, "/cur", NULL);
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen ctx->last_touch = ioloop_time;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->last_notify = ioloop_time;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->flags = flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return ctx;
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainenstatic void maildir_sync_deinit(struct maildir_sync_context *ctx)
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->uidlist_sync_ctx != NULL)
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen (void)maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, FALSE);
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen if (ctx->index_sync_ctx != NULL)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen maildir_sync_index_rollback(&ctx->index_sync_ctx);
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen}
7a6b45405fb1544ac476e6eb1402a70cc1ddcdcfTimo Sirainen
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainenstatic int maildir_fix_duplicate(struct maildir_sync_context *ctx,
dc049c5e83d947aaf1b97c26ae819cc9577e0475Timo Sirainen const char *dir, const char *fname2)
dc049c5e83d947aaf1b97c26ae819cc9577e0475Timo Sirainen{
39775ad03c459efe64cce924658da5094ba417e1Timo Sirainen const char *fname1, *path1, *path2;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen const char *new_fname, *new_path;
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen struct stat st1, st2;
88553367d677170a4b703b9d52aac9eabf91c656Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen fname1 = maildir_uidlist_sync_get_full_filename(ctx->uidlist_sync_ctx,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen fname2);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(fname1 != NULL);
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen path1 = t_strconcat(dir, "/", fname1, NULL);
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen path2 = t_strconcat(dir, "/", fname2, NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (stat(path1, &st1) < 0 || stat(path2, &st2) < 0) {
08fa343b3aace9343da3195686c65c5326eda207Timo Sirainen /* most likely the files just don't exist anymore.
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen don't really care about other errors much. */
08fa343b3aace9343da3195686c65c5326eda207Timo Sirainen return 0;
dc049c5e83d947aaf1b97c26ae819cc9577e0475Timo Sirainen }
dc049c5e83d947aaf1b97c26ae819cc9577e0475Timo Sirainen if (st1.st_ino == st2.st_ino &&
dc049c5e83d947aaf1b97c26ae819cc9577e0475Timo Sirainen CMP_DEV_T(st1.st_dev, st2.st_dev)) {
dc049c5e83d947aaf1b97c26ae819cc9577e0475Timo Sirainen /* Files are the same. this means either a race condition
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen between stat() calls, or that the files were link()ed. */
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen if (st1.st_nlink > 1 && st2.st_nlink == st1.st_nlink &&
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen st1.st_ctime == st2.st_ctime &&
33ae95df45c9b5ec51332a6b39eb5322038686b9Timo Sirainen st1.st_ctime < ioloop_time - DUPE_LINKS_DELETE_SECS) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* The file has hard links and it hasn't had any
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen changes (such as renames) for a while, so this
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen isn't a race condition.
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen rename()ing one file on top of the other would fix
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen this safely, except POSIX decided that rename()
9aa52288a4b53186d81b0ec9afa7d9e0a8ee8753Timo Sirainen doesn't work that way. So we'll have unlink() one
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen and hope that another process didn't just decide to
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen unlink() the other (uidlist lock prevents this from
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen happening) */
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen if (unlink(path2) == 0)
d0d7fcf3ce44f26fdf34c1542a25cec644c5c4c7Timo Sirainen i_warning("Unlinked a duplicate: %s", path2);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen else {
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen mail_storage_set_critical(
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen &ctx->mbox->storage->storage,
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen "unlink(%s) failed: %m", path2);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return 0;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen }
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen new_fname = maildir_filename_generate();
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen new_path = t_strconcat(ctx->mbox->ibox.box.path, "/new/",
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen new_fname, NULL);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen if (rename(path2, new_path) == 0)
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen i_warning("Fixed a duplicate: %s -> %s", path2, new_fname);
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen else if (errno != ENOENT) {
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen mail_storage_set_critical(&ctx->mbox->storage->storage,
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen "Couldn't fix a duplicate: rename(%s, %s) failed: %m",
abe8230dd1dd37d7ccf0163100e934bb5e658c20Timo Sirainen path2, new_path);
6843896c40bee4f9b6680ca7ced598c446e9f999Timo Sirainen return -1;
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen }
313fe89df4d91cd0cd7f3558dc6d7fd21ad39eeeTimo Sirainen return 0;
}
static int
maildir_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st_r)
{
struct mailbox *box = &mbox->ibox.box;
int i;
for (i = 0;; i++) {
if (nfs_safe_stat(path, st_r) == 0)
return 0;
if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT)
break;
if (!maildir_set_deleted(box))
return -1;
/* try again */
}
mail_storage_set_critical(box->storage, "stat(%s) failed: %m", path);
return -1;
}
static int
maildir_scan_dir(struct maildir_sync_context *ctx, bool new_dir, bool final)
{
struct mail_storage *storage = &ctx->mbox->storage->storage;
const char *path;
DIR *dirp;
string_t *src, *dest;
struct dirent *dp;
struct stat st;
enum maildir_uidlist_rec_flag flags;
unsigned int i = 0, move_count = 0;
time_t now;
int ret = 1;
bool move_new, check_touch, dir_changed = FALSE;
path = new_dir ? ctx->new_dir : ctx->cur_dir;
for (i = 0;; i++) {
dirp = opendir(path);
if (dirp != NULL)
break;
if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) {
if (errno == EACCES) {
mail_storage_set_critical(storage, "%s",
eacces_error_get("opendir", path));
} else {
mail_storage_set_critical(storage,
"opendir(%s) failed: %m", path);
}
return -1;
}
if (!maildir_set_deleted(&ctx->mbox->ibox.box))
return -1;
/* try again */
}
#ifdef HAVE_DIRFD
if (fstat(dirfd(dirp), &st) < 0) {
mail_storage_set_critical(storage,
"fstat(%s) failed: %m", path);
(void)closedir(dirp);
return -1;
}
#else
if (maildir_stat(ctx->mbox, path, &st) < 0) {
(void)closedir(dirp);
return -1;
}
#endif
now = time(NULL);
if (new_dir) {
ctx->mbox->maildir_hdr.new_check_time = now;
ctx->mbox->maildir_hdr.new_mtime = st.st_mtime;
ctx->mbox->maildir_hdr.new_mtime_nsecs = ST_MTIME_NSEC(st);
} else {
ctx->mbox->maildir_hdr.cur_check_time = now;
ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime;
ctx->mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st);
}
src = t_str_new(1024);
dest = t_str_new(1024);
move_new = new_dir && !mailbox_is_readonly(&ctx->mbox->ibox.box) &&
!ctx->mbox->ibox.keep_recent && ctx->locked;
errno = 0;
for (; (dp = readdir(dirp)) != NULL; errno = 0) {
if (dp->d_name[0] == '.')
continue;
check_touch = FALSE;
flags = 0;
if (move_new) {
str_truncate(src, 0);
str_truncate(dest, 0);
str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
if (strchr(dp->d_name, MAILDIR_INFO_SEP) == NULL) {
str_append(dest, MAILDIR_FLAGS_FULL_SEP);
}
if (rename(str_c(src), str_c(dest)) == 0) {
/* we moved it - it's \Recent for us */
dir_changed = TRUE;
move_count++;
flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
} else if (ENOTFOUND(errno)) {
/* someone else moved it already */
dir_changed = TRUE;
move_count++;
flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
} else if (ENOSPACE(errno) || errno == EACCES) {
/* not enough disk space / read-only maildir,
leave here */
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
move_new = FALSE;
} else {
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
mail_storage_set_critical(storage,
"rename(%s, %s) failed: %m",
str_c(src), str_c(dest));
}
if ((move_count % MAILDIR_SLOW_MOVE_COUNT) == 0)
maildir_sync_notify(ctx);
} else if (new_dir) {
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
}
i++;
if ((i % MAILDIR_SLOW_CHECK_COUNT) == 0)
maildir_sync_notify(ctx);
ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
dp->d_name, flags);
if (ret <= 0) {
if (ret < 0)
break;
/* possibly duplicate - try fixing it */
T_BEGIN {
ret = maildir_fix_duplicate(ctx, path,
dp->d_name);
} T_END;
if (ret < 0)
break;
}
}
#ifdef __APPLE__
if (errno == EINVAL && move_count > 0 && !final) {
/* OS X HFS+: readdir() fails sometimes when rename()
have been done. */
move_count = MAILDIR_RENAME_RESCAN_COUNT + 1;
} else
#endif
if (errno != 0) {
mail_storage_set_critical(storage,
"readdir(%s) failed: %m", path);
ret = -1;
}
if (closedir(dirp) < 0) {
mail_storage_set_critical(storage,
"closedir(%s) failed: %m", path);
ret = -1;
}
if (dir_changed) {
/* save the exact new times. the new mtimes should be >=
"now", but just in case something weird happens and mtime
doesn't update, use "now". */
if (stat(ctx->new_dir, &st) == 0) {
ctx->mbox->maildir_hdr.new_check_time =
I_MAX(st.st_mtime, now);
ctx->mbox->maildir_hdr.new_mtime = st.st_mtime;
ctx->mbox->maildir_hdr.new_mtime_nsecs =
ST_MTIME_NSEC(st);
}
if (stat(ctx->cur_dir, &st) == 0) {
ctx->mbox->maildir_hdr.new_check_time =
I_MAX(st.st_mtime, now);
ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime;
ctx->mbox->maildir_hdr.cur_mtime_nsecs =
ST_MTIME_NSEC(st);
}
}
return ret < 0 ? -1 :
(move_count <= MAILDIR_RENAME_RESCAN_COUNT || final ? 0 : 1);
}
int maildir_sync_header_refresh(struct maildir_mailbox *mbox)
{
const void *data;
size_t data_size;
if (mail_index_refresh(mbox->ibox.index) < 0) {
mail_storage_set_index_error(&mbox->ibox);
return -1;
}
mail_index_get_header_ext(mbox->ibox.view, mbox->maildir_ext_id,
&data, &data_size);
if (data_size == 0) {
/* doesn't exist */
return 0;
}
memcpy(&mbox->maildir_hdr, data,
I_MIN(sizeof(mbox->maildir_hdr), data_size));
return 0;
}
static int maildir_sync_quick_check(struct maildir_mailbox *mbox, bool undirty,
const char *new_dir, const char *cur_dir,
bool *new_changed_r, bool *cur_changed_r)
{
#define DIR_DELAYED_REFRESH(hdr, name) \
((hdr)->name ## _check_time <= \
(hdr)->name ## _mtime + MAILDIR_SYNC_SECS && \
(undirty || \
(time_t)(hdr)->name ## _check_time < ioloop_time - MAILDIR_SYNC_SECS))
#define DIR_MTIME_CHANGED(st, hdr, name) \
((st).st_mtime != (time_t)(hdr)->name ## _mtime || \
!ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), (hdr)->name ## _mtime_nsecs))
struct maildir_index_header *hdr = &mbox->maildir_hdr;
struct stat new_st, cur_st;
bool refreshed = FALSE, check_new = FALSE, check_cur = FALSE;
if (mbox->maildir_hdr.new_mtime == 0) {
if (maildir_sync_header_refresh(mbox) < 0)
return -1;
if (mbox->maildir_hdr.new_mtime == 0) {
/* first sync */
*new_changed_r = *cur_changed_r = TRUE;
return 0;
}
}
*new_changed_r = *cur_changed_r = FALSE;
/* try to avoid stat()ing by first checking delayed changes */
if (DIR_DELAYED_REFRESH(hdr, new) ||
(DIR_DELAYED_REFRESH(hdr, cur) &&
!mbox->storage->set->maildir_very_dirty_syncs)) {
/* refresh index and try again */
if (maildir_sync_header_refresh(mbox) < 0)
return -1;
refreshed = TRUE;
if (DIR_DELAYED_REFRESH(hdr, new))
*new_changed_r = TRUE;
if (DIR_DELAYED_REFRESH(hdr, cur) &&
!mbox->storage->set->maildir_very_dirty_syncs)
*cur_changed_r = TRUE;
if (*new_changed_r && *cur_changed_r)
return 0;
}
if (!*new_changed_r) {
if (maildir_stat(mbox, new_dir, &new_st) < 0)
return -1;
check_new = TRUE;
}
if (!*cur_changed_r) {
if (maildir_stat(mbox, cur_dir, &cur_st) < 0)
return -1;
check_cur = TRUE;
}
for (;;) {
if (check_new)
*new_changed_r = DIR_MTIME_CHANGED(new_st, hdr, new);
if (check_cur)
*cur_changed_r = DIR_MTIME_CHANGED(cur_st, hdr, cur);
if ((!*new_changed_r && !*cur_changed_r) || refreshed)
break;
/* refresh index and try again */
if (maildir_sync_header_refresh(mbox) < 0)
return -1;
refreshed = TRUE;
}
return 0;
}
static void maildir_sync_update_next_uid(struct maildir_mailbox *mbox)
{
const struct mail_index_header *hdr;
uint32_t uid_validity, next_uid;
hdr = mail_index_get_header(mbox->ibox.view);
if (hdr->uid_validity == 0)
return;
uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
next_uid = maildir_uidlist_get_next_uid(mbox->uidlist);
if (uid_validity == hdr->uid_validity || uid_validity == 0) {
/* make sure uidlist's next_uid is at least as large as
index file's. typically this happens only if uidlist gets
deleted. */
maildir_uidlist_set_uid_validity(mbox->uidlist,
hdr->uid_validity);
maildir_uidlist_set_next_uid(mbox->uidlist,
hdr->next_uid, FALSE);
}
}
static bool have_recent_messages(struct maildir_sync_context *ctx)
{
const struct mail_index_header *hdr;
(void)maildir_uidlist_refresh(ctx->mbox->uidlist);
/* if there are files in new/, we'll need to move them. we'll check
this by checking if we have any recent messages */
hdr = mail_index_get_header(ctx->mbox->ibox.view);
return hdr->first_recent_uid <
maildir_uidlist_get_next_uid(ctx->mbox->uidlist);
}
static int maildir_sync_get_changes(struct maildir_sync_context *ctx,
bool *new_changed_r, bool *cur_changed_r)
{
enum mail_index_sync_flags flags = 0;
bool undirty = (ctx->flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0;
if (maildir_sync_quick_check(ctx->mbox, undirty,
ctx->new_dir, ctx->cur_dir,
new_changed_r, cur_changed_r) < 0)
return -1;
if (*new_changed_r || *cur_changed_r)
return 1;
if (have_recent_messages(ctx)) {
if (!ctx->mbox->ibox.keep_recent) {
*new_changed_r = TRUE;
return 1;
} else if (*new_changed_r) {
/* we have some recent messages and new/ has changed.
if messages had been externally deleted from new/,
we need to get them out of index. this requires that
we make sure they weren't just moved to cur/. */
*cur_changed_r = TRUE;
return 1;
}
}
if (!ctx->mbox->ibox.keep_recent)
flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
return mail_index_sync_have_any(ctx->mbox->ibox.index, flags) ? 1 : 0;
}
static int maildir_sync_context(struct maildir_sync_context *ctx, bool forced,
uint32_t *find_uid, bool *lost_files_r)
{
enum maildir_uidlist_sync_flags sync_flags;
enum maildir_uidlist_rec_flag flags;
bool new_changed, cur_changed, lock_failure;
const char *fname;
int ret;
*lost_files_r = FALSE;
if (forced)
new_changed = cur_changed = TRUE;
else {
ret = maildir_sync_get_changes(ctx, &new_changed, &cur_changed);
if (ret <= 0)
return ret;
}
/*
Locking, locking, locking.. Wasn't maildir supposed to be lockless?
We can get here either as beginning a real maildir sync, or when
committing changes to maildir but a file was lost (maybe renamed).
So, we're going to need two locks. One for index and one for
uidlist. To avoid deadlocking do the uidlist lock always first.
uidlist is needed only for figuring out UIDs for newly seen files,
so theoretically we wouldn't need to lock it unless there are new
files. It has a few problems though, assuming the index lock didn't
already protect it (eg. in-memory indexes):
1. Just because you see a new file which doesn't exist in uidlist
file, doesn't mean that the file really exists anymore, or that
your readdir() lists all new files. Meaning that this is possible:
A: opendir(), readdir() -> new file ...
-- new files are written to the maildir --
B: opendir(), readdir() -> new file, lock uidlist,
readdir() -> another new file, rewrite uidlist, unlock
A: ... lock uidlist, readdir() -> nothing left, rewrite uidlist,
unlock
The second time running A didn't see the two new files. To handle
this correctly, it must not remove the new unseen files from
uidlist. This is possible to do, but adds extra complexity.
2. If another process is rename()ing files while we are
readdir()ing, it's possible that readdir() never lists some files,
causing Dovecot to assume they were expunged. In next sync they
would show up again, but client could have already been notified of
that and they would show up under new UIDs, so the damage is
already done.
Both of the problems can be avoided if we simply lock the uidlist
before syncing and keep it until sync is finished. Typically this
would happen in any case, as there is the index lock..
The second case is still a problem with external changes though,
because maildir doesn't require any kind of locking. Luckily this
problem rarely happens except under high amount of modifications.
*/
if (!cur_changed) {
ctx->partial = TRUE;
sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL;
} else {
ctx->partial = FALSE;
sync_flags = 0;
if (forced)
sync_flags |= MAILDIR_UIDLIST_SYNC_FORCE;
if ((ctx->flags & MAILBOX_SYNC_FLAG_FAST) != 0)
sync_flags |= MAILDIR_UIDLIST_SYNC_TRYLOCK;
}
ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags,
&ctx->uidlist_sync_ctx);
lock_failure = ret <= 0;
if (ret <= 0) {
struct mail_storage *storage = ctx->mbox->ibox.box.storage;
if (ret == 0) {
/* timeout */
return 0;
}
/* locking failed. sync anyway without locking so that it's
possible to expunge messages when out of quota. */
if (forced) {
/* we're already forcing a sync, we're trying to find
a message that was probably already expunged, don't
loop for a long time trying to find it. */
return -1;
}
ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags |
MAILDIR_UIDLIST_SYNC_NOLOCK,
&ctx->uidlist_sync_ctx);
if (ret <= 0) {
i_assert(ret != 0);
return -1;
}
if (storage->callbacks.notify_no != NULL) {
storage->callbacks.notify_no(&ctx->mbox->ibox.box,
"Internal mailbox synchronization failure, "
"showing only old mails.",
storage->callback_context);
}
}
ctx->locked = maildir_uidlist_is_locked(ctx->mbox->uidlist);
if (!ctx->locked)
ctx->partial = TRUE;
if (!ctx->mbox->syncing_commit && (ctx->locked || lock_failure)) {
if (maildir_sync_index_begin(ctx->mbox, ctx,
&ctx->index_sync_ctx) < 0)
return -1;
}
if (new_changed || cur_changed) {
/* if we're going to check cur/ dir our current logic requires
that new/ dir is checked as well. it's a good idea anyway. */
unsigned int count = 0;
bool final = FALSE;
while ((ret = maildir_scan_dir(ctx, TRUE, final)) > 0) {
/* rename()d at least some files, which might have
caused some other files to be missed. check again
(see MAILDIR_RENAME_RESCAN_COUNT). */
if (++count >= MAILDIR_SCAN_DIR_MAX_COUNT)
final = TRUE;
}
if (ret < 0)
return -1;
if (cur_changed) {
if (maildir_scan_dir(ctx, FALSE, TRUE) < 0)
return -1;
}
maildir_sync_update_next_uid(ctx->mbox);
/* finish uidlist syncing, but keep it still locked */
maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
}
if (!ctx->locked) {
/* make sure we sync the maildir later */
ctx->mbox->maildir_hdr.new_mtime = 0;
ctx->mbox->maildir_hdr.cur_mtime = 0;
}
if (ctx->index_sync_ctx != NULL) {
/* NOTE: index syncing here might cause a re-sync due to
files getting lost, so this function might be called
re-entrantly. */
ret = maildir_sync_index(ctx->index_sync_ctx, ctx->partial);
if (ret < 0)
maildir_sync_index_rollback(&ctx->index_sync_ctx);
else if (maildir_sync_index_commit(&ctx->index_sync_ctx) < 0)
return -1;
if (ret < 0)
return -1;
if (ret == 0)
*lost_files_r = TRUE;
i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist) ||
lock_failure);
}
if (find_uid != NULL && *find_uid != 0) {
ret = maildir_uidlist_lookup_nosync(ctx->mbox->uidlist,
*find_uid, &flags, &fname);
if (ret < 0)
return -1;
if (ret == 0) {
/* UID is expunged */
*find_uid = 0;
} else if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) {
/* we didn't find it, possibly expunged? */
*find_uid = 0;
}
}
return maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, TRUE);
}
int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid)
{
struct maildir_sync_context *ctx;
bool lost_files;
int ret;
T_BEGIN {
ctx = maildir_sync_context_new(mbox, MAILBOX_SYNC_FLAG_FAST);
ret = maildir_sync_context(ctx, TRUE, &uid, &lost_files);
maildir_sync_deinit(ctx);
} T_END;
if (uid != 0) {
/* maybe it's expunged. check again. */
T_BEGIN {
ctx = maildir_sync_context_new(mbox, 0);
ret = maildir_sync_context(ctx, TRUE, NULL,
&lost_files);
maildir_sync_deinit(ctx);
} T_END;
}
return ret;
}
struct mailbox_sync_context *
maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
struct maildir_sync_context *ctx;
bool lost_files, force_resync;
int ret = 0;
if (!box->opened) {
if (mailbox_open(box) < 0)
return index_mailbox_sync_init(box, flags, TRUE);
}
force_resync = (flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0;
if (index_mailbox_want_full_sync(&mbox->ibox, flags)) {
T_BEGIN {
ctx = maildir_sync_context_new(mbox, flags);
ret = maildir_sync_context(ctx, force_resync, NULL,
&lost_files);
maildir_sync_deinit(ctx);
} T_END;
i_assert(!maildir_uidlist_is_locked(mbox->uidlist) ||
mbox->ibox.keep_locked);
if (lost_files) {
/* lost some files from new/, see if thery're in cur/ */
ret = maildir_storage_sync_force(mbox, 0);
}
}
if (mbox->storage->set->maildir_very_dirty_syncs) {
struct mail_index_view_sync_ctx *sync_ctx;
bool b;
if (mbox->flags_view == NULL) {
mbox->flags_view =
mail_index_view_open(mbox->ibox.index);
}
sync_ctx = mail_index_view_sync_begin(mbox->flags_view,
MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
if (mail_index_view_sync_commit(&sync_ctx, &b) < 0) {
mail_storage_set_index_error(&mbox->ibox);
ret = -1;
}
/* make sure the map stays in private memory */
if (mbox->flags_view->map->refcount > 1) {
struct mail_index_map *map;
map = mail_index_map_clone(mbox->flags_view->map);
mail_index_unmap(&mbox->flags_view->map);
mbox->flags_view->map = map;
}
mail_index_record_map_move_to_private(mbox->flags_view->map);
mail_index_map_move_to_memory(mbox->flags_view->map);
maildir_uidlist_set_all_nonsynced(mbox->uidlist);
}
return index_mailbox_sync_init(box, flags, ret < 0);
}
int maildir_sync_is_synced(struct maildir_mailbox *mbox)
{
bool new_changed, cur_changed;
int ret;
T_BEGIN {
const char *new_dir, *cur_dir;
new_dir = t_strconcat(mbox->ibox.box.path, "/new", NULL);
cur_dir = t_strconcat(mbox->ibox.box.path, "/cur", NULL);
ret = maildir_sync_quick_check(mbox, FALSE, new_dir, cur_dir,
&new_changed, &cur_changed);
} T_END;
return ret < 0 ? -1 : (!new_changed && !cur_changed);
}