maildir-sync.c revision b63284468d717737ecd63d78b6928c5d7f0d3634
2454dfa32c93c20a8522c6ed42fe057baaac9f9aStephan Bosch/* Copyright (c) 2004-2013 Dovecot authors, see the included COPYING file */
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen/*
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Here's a description of how we handle Maildir synchronization and
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen it's problems:
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen We want to be as efficient as we can. The most efficient way to
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen check if changes have occurred is to stat() the new/ and cur/
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen directories and uidlist file - if their mtimes haven't changed,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen there's no changes and we don't need to do anything.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Problem 1: Multiple changes can happen within a single second -
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen nothing guarantees that once we synced it, someone else didn't just
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen then make a modification. Such modifications wouldn't get noticed
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen until a new modification occurred later.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Problem 2: Syncing cur/ directory is much more costly than syncing
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch new/. Moving mails from new/ to cur/ will always change mtime of
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen cur/ causing us to sync it as well.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Problem 3: We may not be able to move mail from new/ to cur/
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen because we're out of quota, or simply because we're accessing a
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen read-only mailbox.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen MAILDIR_SYNC_SECS
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen -----------------
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Several checks below use MAILDIR_SYNC_SECS, which should be maximum
fe779565bda49a0ed0476724819c6e3c1340c94bTimo Sirainen clock drift between all computers accessing the maildir (eg. via
fe779565bda49a0ed0476724819c6e3c1340c94bTimo Sirainen NFS), rounded up to next second. Our default is 1 second, since
fe779565bda49a0ed0476724819c6e3c1340c94bTimo Sirainen everyone should be using NTP.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Note that setting it to 0 works only if there's only one computer
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen accessing the maildir. It's practically impossible to make two
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch clocks _exactly_ synchronized.
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen It might be possible to only use file server's clock by looking at
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen the atime field, but I don't know how well that would actually work.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen cur directory
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen -------------
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen We have dirty_cur_time variable which is set to cur/ directory's
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen synchronized the directory.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen When dirty_cur_time is non-zero, we don't synchronize the cur/
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch directory until
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen a) cur/'s mtime changes
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen b) opening a mail fails with ENOENT
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen c) time() > dirty_cur_time + MAILDIR_SYNC_SECS
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen This allows us to modify the maildir multiple times without having
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen to sync it at every change. The sync will eventually be done to
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen make sure we didn't miss any external changes.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen The dirty_cur_time is set when:
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen - we change message flags
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen - we expunge messages
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen - we move mail from new/ to cur/
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen It's unset when we do the final syncing, ie. when mtime is
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen older than time() - MAILDIR_SYNC_SECS.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen new directory
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen -------------
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen it. dirty_cur_time-like feature might save us a few syncs, but
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen that might break a client which saves a mail in one connection and
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch tries to fetch it in another one. new/ directory is almost always
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch empty, so syncing it should be very fast anyway. Actually this can
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch still happen if we sync only new/ dir while another client is also
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch moving mails from it to cur/ - it takes us a while to see them.
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch That's pretty unlikely to happen however, and only way to fix it
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen would be to always synchronize cur/ after new/.
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch Normally we move all mails from new/ to cur/ whenever we sync it. If
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch it's not possible for some reason, we mark the mail with "probably
0adc24c0c534944b55a185795e09dfaea2ca3131Stephan Bosch exists in new/ directory" flag.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen If rename() still fails because of ENOSPC or EDQUOT, we still save
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen the flag changes in index with dirty-flag on. When moving the mail
1c75bf24894a3fc0631caa4954e5130e9bb01d8dTimo Sirainen to cur/ directory, or when we notice it's already moved there, we
1c75bf24894a3fc0631caa4954e5130e9bb01d8dTimo Sirainen apply the flag changes to the filename, rename it and remove the
1c75bf24894a3fc0631caa4954e5130e9bb01d8dTimo Sirainen dirty flag. If there's dirty flags, this should be tried every time
1c75bf24894a3fc0631caa4954e5130e9bb01d8dTimo Sirainen after expunge or when closing the mailbox.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen uidlist
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen -------
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch This file contains UID <-> filename mappings. It's updated only when
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch new mail arrives, so it may contain filenames that have already been
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch deleted. Updating is done by getting uidlist.lock file, writing the
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch whole uidlist into it and rename()ing it over the old uidlist. This
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch means there's no need to lock the file for reading.
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Whenever uidlist is rewritten, it's mtime must be larger than the old
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen one's. Use utime() before rename() if needed. Note that inode checking
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen wouldn't have been sufficient as inode numbers can be reused.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen This file is usually read the first time you need to know filename for
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen given UID. After that it's not re-read unless new mails come that we
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen don't know about.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen broken clients
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen --------------
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Originally the middle identifier in Maildir filename was specified
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen only as <process id>_<delivery counter>. That however created a
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen problem with randomized PIDs which made it possible that the same
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen PID was reused within one second.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen So if within one second a mail was delivered, MUA moved it to cur/
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen and another mail was delivered by a new process using same PID as
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen the first one, we likely ended up overwriting the first mail when
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen the second mail was moved over it.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Nowadays everyone should be giving a bit more specific identifier,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen for example include microseconds in it which Dovecot does.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen There's a simple way to prevent this from happening in some cases:
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Don't move the mail from new/ to cur/ if it's mtime is >= time() -
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen MAILDIR_SYNC_SECS. The second delivery's link() call then fails
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody because the file is already in new/, and it will then use a
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen different filename. There's a few problems with this however:
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen - it requires extra stat() call which is unneeded extra I/O
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen - another MUA might still move the mail to cur/
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen - if first file's flags are modified by either Dovecot or another
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen but that'd be ugly)
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Because this is useful only for very few people and it requires
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen extra I/O, I decided not to implement this. It should be however
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch quite easy to do since we need to be able to deal with files in new/
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen in any case.
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen It's also possible to never accidentally overwrite a mail by using
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen link() + unlink() rather than rename(). This however isn't very
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen good idea as it introduces potential race conditions when multiple
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen clients are accessing the mailbox:
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen Trying to move the same mail from new/ to cur/ at the same time:
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen a) Client 1 uses slightly different filename than client 2,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen for example one sets read-flag on but the other doesn't.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen You have the same mail duplicated now.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen b) Client 3 sees the mail between Client 1's and 2's link() calls
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen and changes it's flag. You have the same mail duplicated now.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen And it gets worse when they're unlink()ing in cur/ directory:
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen c) Client 1 changes mails's flag and client 2 changes it back
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen between 1's link() and unlink(). The mail is now expunged.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen d) If you try to deal with the duplicates by unlink()ing another
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen one of them, you might end up unlinking both of them.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen So, what should we do then if we notice a duplicate? First of all,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen it might not be a duplicate at all, readdir() might have just
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen returned it twice because it was just renamed. What we should do is
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen create a completely new base name for it and rename() it to that.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen If the call fails with ENOENT, it only means that it wasn't a
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen duplicate after all.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen*/
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "lib.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "ioloop.h"
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch#include "array.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "buffer.h"
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch#include "hash.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "str.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "eacces-error.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "nfs-workarounds.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "maildir-storage.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "maildir-uidlist.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "maildir-filename.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include "maildir-sync.h"
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include <stdio.h>
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include <stddef.h>
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include <stdlib.h>
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include <unistd.h>
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include <dirent.h>
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#include <sys/stat.h>
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#define MAILDIR_FILENAME_FLAG_FOUND 128
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen/* When rename()ing many files from new/ to cur/, it's possible that next
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen readdir() skips some files. we don't of course wish to lose them, so we
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen go and rescan the new/ directory again from beginning until no files are
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen left. This value is just an optimization to avoid checking the directory
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen twice unneededly. usually only NFS is the problem case. 1 is the safest
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen bet here, but I guess 5 will do just fine too. */
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#define MAILDIR_RENAME_RESCAN_COUNT 5
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen/* This is mostly to avoid infinite looping when rename() destination already
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen exists as the hard link of the file itself. */
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#define MAILDIR_SCAN_DIR_MAX_COUNT 5
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen#define DUPE_LINKS_DELETE_SECS 30
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainenenum maildir_scan_why {
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen WHY_FORCED = 0x01,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen WHY_FIRSTSYNC = 0x02,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen WHY_NEWCHANGED = 0x04,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen WHY_CURCHANGED = 0x08,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen WHY_DROPRECENT = 0x10,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen WHY_FINDRECENT = 0x20,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen WHY_DELAYEDNEW = 0x40,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen WHY_DELAYEDCUR = 0x80
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch};
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainenstruct maildir_sync_context {
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen struct maildir_mailbox *mbox;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen const char *new_dir, *cur_dir;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen enum mailbox_sync_flags flags;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen time_t last_touch, last_notify;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen struct maildir_index_sync_context *index_sync_ctx;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen unsigned int partial:1;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen unsigned int locked:1;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen unsigned int racing:1;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen};
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainenvoid maildir_sync_set_racing(struct maildir_sync_context *ctx)
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen{
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen ctx->racing = TRUE;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen}
8855b8b57050fe3b6dc3f19283488512fae98648Timo Sirainen
8855b8b57050fe3b6dc3f19283488512fae98648Timo Sirainenvoid maildir_sync_notify(struct maildir_sync_context *ctx)
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen{
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen time_t now;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen if (ctx == NULL) {
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen /* we got here from maildir-save.c. it has no
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen maildir_sync_context, */
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen return;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch }
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch now = time(NULL);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch if (now - ctx->last_touch > MAILDIR_LOCK_TOUCH_SECS && ctx->locked) {
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch (void)maildir_uidlist_lock_touch(ctx->mbox->uidlist);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx->last_touch = now;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch }
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch if (now - ctx->last_notify > MAIL_STORAGE_STAYALIVE_SECS) {
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch struct mailbox *box = &ctx->mbox->box;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch if (box->storage->callbacks.notify_ok != NULL) {
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch box->storage->callbacks.
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch notify_ok(box, "Hang in there..",
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch box->storage->callback_context);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch }
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx->last_notify = now;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen }
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch}
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainenstatic struct maildir_sync_context *
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainenmaildir_sync_context_new(struct maildir_mailbox *mbox,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen enum mailbox_sync_flags flags)
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch{
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch struct maildir_sync_context *ctx;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx = t_new(struct maildir_sync_context, 1);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx->mbox = mbox;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx->new_dir = t_strconcat(mailbox_get_path(&mbox->box), "/new", NULL);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx->cur_dir = t_strconcat(mailbox_get_path(&mbox->box), "/cur", NULL);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx->last_touch = ioloop_time;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx->last_notify = ioloop_time;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch ctx->flags = flags;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch return ctx;
d8aa10df6d1dae56d3aa485708a34d74e9e31e79Stephan Bosch}
d8aa10df6d1dae56d3aa485708a34d74e9e31e79Stephan Bosch
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Boschstatic void maildir_sync_deinit(struct maildir_sync_context *ctx)
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch{
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch if (ctx->uidlist_sync_ctx != NULL)
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch (void)maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, FALSE);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch if (ctx->index_sync_ctx != NULL)
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch maildir_sync_index_rollback(&ctx->index_sync_ctx);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch}
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Boschstatic int maildir_fix_duplicate(struct maildir_sync_context *ctx,
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch const char *dir, const char *fname2)
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch{
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch const char *fname1, *path1, *path2;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch const char *new_fname, *new_path;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch struct stat st1, st2;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch uoff_t size;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch fname1 = maildir_uidlist_sync_get_full_filename(ctx->uidlist_sync_ctx,
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch fname2);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch i_assert(fname1 != NULL);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch path1 = t_strconcat(dir, "/", fname1, NULL);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch path2 = t_strconcat(dir, "/", fname2, NULL);
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch if (stat(path1, &st1) < 0 || stat(path2, &st2) < 0) {
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch /* most likely the files just don't exist anymore.
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch don't really care about other errors much. */
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch return 0;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch }
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch if (st1.st_ino == st2.st_ino &&
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch CMP_DEV_T(st1.st_dev, st2.st_dev)) {
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch /* Files are the same. this means either a race condition
61cf001f1944d92eb25f113ba4c08985d6e30d53Timo Sirainen between stat() calls, or that the files were link()ed. */
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch if (st1.st_nlink > 1 && st2.st_nlink == st1.st_nlink &&
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch st1.st_ctime == st2.st_ctime &&
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch st1.st_ctime < ioloop_time - DUPE_LINKS_DELETE_SECS) {
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch /* The file has hard links and it hasn't had any
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch changes (such as renames) for a while, so this
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch isn't a race condition.
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch rename()ing one file on top of the other would fix
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch this safely, except POSIX decided that rename()
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch doesn't work that way. So we'll have unlink() one
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen and hope that another process didn't just decide to
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen unlink() the other (uidlist lock prevents this from
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen happening) */
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen if (unlink(path2) == 0)
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen i_warning("Unlinked a duplicate: %s", path2);
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen else {
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen mail_storage_set_critical(
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen &ctx->mbox->storage->storage,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen "unlink(%s) failed: %m", path2);
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen }
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen }
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen return 0;
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen }
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen new_fname = maildir_filename_generate();
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen /* preserve S= and W= sizes if they're available.
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen (S=size is required for zlib plugin to work) */
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_FILE_SIZE, &size)) {
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen new_fname, MAILDIR_EXTRA_FILE_SIZE, size);
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen }
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_VIRTUAL_SIZE, &size)) {
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T,
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen new_fname, MAILDIR_EXTRA_VIRTUAL_SIZE, size);
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen }
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box),
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen "/new/", new_fname, NULL);
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen if (rename(path2, new_path) == 0)
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen i_warning("Fixed a duplicate: %s -> %s", path2, new_fname);
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen else if (errno != ENOENT) {
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen mail_storage_set_critical(&ctx->mbox->storage->storage,
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch "Couldn't fix a duplicate: rename(%s, %s) failed: %m",
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen path2, new_path);
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen return -1;
8ccdf195768afdfbc32088d7be77dfca7dddd829Stephan Bosch }
02c75e04c6ff80726bb59e3ea34a7995ad1f6f7cTimo Sirainen return 0;
}
static int
maildir_rename_empty_basename(struct maildir_sync_context *ctx,
const char *dir, const char *fname)
{
const char *old_path, *new_fname, *new_path;
old_path = t_strconcat(dir, "/", fname, NULL);
new_fname = maildir_filename_generate();
new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box),
"/new/", new_fname, NULL);
if (rename(old_path, new_path) == 0)
i_warning("Fixed broken filename: %s -> %s", old_path, new_fname);
else if (errno != ENOENT) {
mail_storage_set_critical(&ctx->mbox->storage->storage,
"Couldn't fix a broken filename: rename(%s, %s) failed: %m",
old_path, new_path);
return -1;
}
return 0;
}
static int
maildir_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st_r)
{
struct mailbox *box = &mbox->box;
int i;
for (i = 0;; i++) {
if (nfs_safe_stat(path, st_r) == 0)
return 0;
if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT)
break;
if (!maildir_set_deleted(box))
return -1;
/* try again */
}
mail_storage_set_critical(box->storage, "stat(%s) failed: %m", path);
return -1;
}
static int
maildir_scan_dir(struct maildir_sync_context *ctx, bool new_dir, bool final,
enum maildir_scan_why why)
{
struct mail_storage *storage = &ctx->mbox->storage->storage;
const char *path;
DIR *dirp;
string_t *src, *dest;
struct dirent *dp;
struct stat st;
enum maildir_uidlist_rec_flag flags;
unsigned int time_diff, i, readdir_count = 0, move_count = 0;
time_t start_time;
int ret = 1;
bool move_new, dir_changed = FALSE;
path = new_dir ? ctx->new_dir : ctx->cur_dir;
for (i = 0;; i++) {
dirp = opendir(path);
if (dirp != NULL)
break;
if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) {
if (errno == EACCES) {
mail_storage_set_critical(storage, "%s",
eacces_error_get("opendir", path));
} else {
mail_storage_set_critical(storage,
"opendir(%s) failed: %m", path);
}
return -1;
}
if (!maildir_set_deleted(&ctx->mbox->box))
return -1;
/* try again */
}
#ifdef HAVE_DIRFD
if (fstat(dirfd(dirp), &st) < 0) {
mail_storage_set_critical(storage,
"fstat(%s) failed: %m", path);
(void)closedir(dirp);
return -1;
}
#else
if (maildir_stat(ctx->mbox, path, &st) < 0) {
(void)closedir(dirp);
return -1;
}
#endif
start_time = time(NULL);
if (new_dir) {
ctx->mbox->maildir_hdr.new_check_time = start_time;
ctx->mbox->maildir_hdr.new_mtime = st.st_mtime;
ctx->mbox->maildir_hdr.new_mtime_nsecs = ST_MTIME_NSEC(st);
} else {
ctx->mbox->maildir_hdr.cur_check_time = start_time;
ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime;
ctx->mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st);
}
src = t_str_new(1024);
dest = t_str_new(1024);
move_new = new_dir && !mailbox_is_readonly(&ctx->mbox->box) &&
(ctx->mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0 &&
ctx->locked;
errno = 0;
for (; (dp = readdir(dirp)) != NULL; errno = 0) {
if (dp->d_name[0] == '.')
continue;
if (dp->d_name[0] == MAILDIR_INFO_SEP) {
/* don't even try to use file with empty base name */
if (maildir_rename_empty_basename(ctx, path,
dp->d_name) < 0)
break;
continue;
}
flags = 0;
if (move_new) {
i_assert(dp->d_name[0] != '\0');
str_truncate(src, 0);
str_truncate(dest, 0);
str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
if (strchr(dp->d_name, MAILDIR_INFO_SEP) == NULL) {
str_append(dest, MAILDIR_FLAGS_FULL_SEP);
}
if (rename(str_c(src), str_c(dest)) == 0) {
/* we moved it - it's \Recent for us */
dir_changed = TRUE;
move_count++;
flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
} else if (ENOTFOUND(errno)) {
/* someone else moved it already */
dir_changed = TRUE;
move_count++;
flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
} else if (ENOSPACE(errno) || errno == EACCES) {
/* not enough disk space / read-only maildir,
leave here */
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
move_new = FALSE;
} else {
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
mail_storage_set_critical(storage,
"rename(%s, %s) failed: %m",
str_c(src), str_c(dest));
}
if ((move_count % MAILDIR_SLOW_MOVE_COUNT) == 0)
maildir_sync_notify(ctx);
} else if (new_dir) {
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
}
readdir_count++;
if ((readdir_count % MAILDIR_SLOW_CHECK_COUNT) == 0)
maildir_sync_notify(ctx);
ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
dp->d_name, flags);
if (ret <= 0) {
if (ret < 0)
break;
/* possibly duplicate - try fixing it */
T_BEGIN {
ret = maildir_fix_duplicate(ctx, path,
dp->d_name);
} T_END;
if (ret < 0)
break;
}
}
#ifdef __APPLE__
if (errno == EINVAL && move_count > 0 && !final) {
/* OS X HFS+: readdir() fails sometimes when rename()
have been done. */
move_count = MAILDIR_RENAME_RESCAN_COUNT + 1;
} else
#endif
if (errno != 0) {
mail_storage_set_critical(storage,
"readdir(%s) failed: %m", path);
ret = -1;
}
if (closedir(dirp) < 0) {
mail_storage_set_critical(storage,
"closedir(%s) failed: %m", path);
ret = -1;
}
if (dir_changed) {
/* save the exact new times. the new mtimes should be >=
"start_time", but just in case something weird happens and
mtime doesn't update, use "start_time". */
if (stat(ctx->new_dir, &st) == 0) {
ctx->mbox->maildir_hdr.new_check_time =
I_MAX(st.st_mtime, start_time);
ctx->mbox->maildir_hdr.new_mtime = st.st_mtime;
ctx->mbox->maildir_hdr.new_mtime_nsecs =
ST_MTIME_NSEC(st);
}
if (stat(ctx->cur_dir, &st) == 0) {
ctx->mbox->maildir_hdr.new_check_time =
I_MAX(st.st_mtime, start_time);
ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime;
ctx->mbox->maildir_hdr.cur_mtime_nsecs =
ST_MTIME_NSEC(st);
}
}
time_diff = time(NULL) - start_time;
if (time_diff >= MAILDIR_SYNC_TIME_WARN_SECS) {
i_warning("Maildir: Scanning %s took %u seconds "
"(%u readdir()s, %u rename()s to cur/, why=0x%x)",
path, time_diff, readdir_count, move_count, why);
}
return ret < 0 ? -1 :
(move_count <= MAILDIR_RENAME_RESCAN_COUNT || final ? 0 : 1);
}
static void maildir_sync_get_header(struct maildir_mailbox *mbox)
{
const void *data;
size_t data_size;
mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id,
&data, &data_size);
if (data_size == 0) {
/* header doesn't exist */
} else {
memcpy(&mbox->maildir_hdr, data,
I_MIN(sizeof(mbox->maildir_hdr), data_size));
}
}
int maildir_sync_header_refresh(struct maildir_mailbox *mbox)
{
if (mail_index_refresh(mbox->box.index) < 0) {
mailbox_set_index_error(&mbox->box);
return -1;
}
maildir_sync_get_header(mbox);
return 0;
}
static int maildir_sync_quick_check(struct maildir_mailbox *mbox, bool undirty,
const char *new_dir, const char *cur_dir,
bool *new_changed_r, bool *cur_changed_r,
enum maildir_scan_why *why_r)
{
#define DIR_DELAYED_REFRESH(hdr, name) \
((hdr)->name ## _check_time <= \
(hdr)->name ## _mtime + MAILDIR_SYNC_SECS && \
(undirty || \
(time_t)(hdr)->name ## _check_time < ioloop_time - MAILDIR_SYNC_SECS))
#define DIR_MTIME_CHANGED(st, hdr, name) \
((st).st_mtime != (time_t)(hdr)->name ## _mtime || \
!ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), (hdr)->name ## _mtime_nsecs))
struct maildir_index_header *hdr = &mbox->maildir_hdr;
struct stat new_st, cur_st;
bool refreshed = FALSE, check_new = FALSE, check_cur = FALSE;
*why_r = 0;
if (mbox->maildir_hdr.new_mtime == 0) {
maildir_sync_get_header(mbox);
if (mbox->maildir_hdr.new_mtime == 0) {
/* first sync */
*why_r |= WHY_FIRSTSYNC;
*new_changed_r = *cur_changed_r = TRUE;
return 0;
}
}
*new_changed_r = *cur_changed_r = FALSE;
/* try to avoid stat()ing by first checking delayed changes */
if (DIR_DELAYED_REFRESH(hdr, new) ||
(DIR_DELAYED_REFRESH(hdr, cur) &&
!mbox->storage->set->maildir_very_dirty_syncs)) {
/* refresh index and try again */
if (maildir_sync_header_refresh(mbox) < 0)
return -1;
refreshed = TRUE;
if (DIR_DELAYED_REFRESH(hdr, new)) {
*why_r |= WHY_DELAYEDNEW;
*new_changed_r = TRUE;
}
if (DIR_DELAYED_REFRESH(hdr, cur) &&
!mbox->storage->set->maildir_very_dirty_syncs) {
*why_r |= WHY_DELAYEDCUR;
*cur_changed_r = TRUE;
}
if (*new_changed_r && *cur_changed_r)
return 0;
}
if (!*new_changed_r) {
if (maildir_stat(mbox, new_dir, &new_st) < 0)
return -1;
check_new = TRUE;
}
if (!*cur_changed_r) {
if (maildir_stat(mbox, cur_dir, &cur_st) < 0)
return -1;
check_cur = TRUE;
}
for (;;) {
if (check_new) {
*new_changed_r = DIR_MTIME_CHANGED(new_st, hdr, new);
if (*new_changed_r)
*why_r |= WHY_NEWCHANGED;
}
if (check_cur) {
*cur_changed_r = DIR_MTIME_CHANGED(cur_st, hdr, cur);
if (*cur_changed_r)
*why_r |= WHY_CURCHANGED;
}
if ((!*new_changed_r && !*cur_changed_r) || refreshed)
break;
/* refresh index and try again */
if (maildir_sync_header_refresh(mbox) < 0)
return -1;
refreshed = TRUE;
}
return 0;
}
static void maildir_sync_update_next_uid(struct maildir_mailbox *mbox)
{
const struct mail_index_header *hdr;
uint32_t uid_validity;
hdr = mail_index_get_header(mbox->box.view);
if (hdr->uid_validity == 0)
return;
uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
if (uid_validity == hdr->uid_validity || uid_validity == 0) {
/* make sure uidlist's next_uid is at least as large as
index file's. typically this happens only if uidlist gets
deleted. */
maildir_uidlist_set_uid_validity(mbox->uidlist,
hdr->uid_validity);
maildir_uidlist_set_next_uid(mbox->uidlist,
hdr->next_uid, FALSE);
}
}
static bool
have_recent_messages(struct maildir_sync_context *ctx, bool seen_changes)
{
const struct mail_index_header *hdr;
uint32_t next_uid;
hdr = mail_index_get_header(ctx->mbox->box.view);
if (!seen_changes) {
/* index is up to date. get the next-uid from it */
next_uid = hdr->next_uid;
} else {
(void)maildir_uidlist_refresh(ctx->mbox->uidlist);
next_uid = maildir_uidlist_get_next_uid(ctx->mbox->uidlist);
}
return hdr->first_recent_uid < next_uid;
}
static int maildir_sync_get_changes(struct maildir_sync_context *ctx,
bool *new_changed_r, bool *cur_changed_r,
enum maildir_scan_why *why_r)
{
struct maildir_mailbox *mbox = ctx->mbox;
enum mail_index_sync_flags flags = 0;
bool undirty = (ctx->flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0;
*why_r = 0;
if (maildir_sync_quick_check(mbox, undirty, ctx->new_dir, ctx->cur_dir,
new_changed_r, cur_changed_r, why_r) < 0)
return -1;
/* if there are files in new/, we'll need to move them. we'll check
this by seeing if we have any recent messages */
if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) {
if (!*new_changed_r && have_recent_messages(ctx, FALSE)) {
*new_changed_r = TRUE;
*why_r |= WHY_DROPRECENT;
}
} else if (*new_changed_r) {
/* if recent messages have been externally deleted from new/,
we need to get them out of index. this requires that
we make sure they weren't just moved to cur/. */
if (!*cur_changed_r && have_recent_messages(ctx, TRUE)) {
*cur_changed_r = TRUE;
*why_r |= WHY_FINDRECENT;
}
}
if (*new_changed_r || *cur_changed_r)
return 1;
if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0)
flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
if (mbox->synced) {
/* refresh index only after the first sync, i.e. avoid wasting
time on refreshing it immediately after it was just opened */
mail_index_refresh(mbox->box.index);
}
return mail_index_sync_have_any(mbox->box.index, flags) ? 1 : 0;
}
static int ATTR_NULL(3)
maildir_sync_context(struct maildir_sync_context *ctx, bool forced,
uint32_t *find_uid, bool *lost_files_r)
{
enum maildir_uidlist_sync_flags sync_flags;
enum maildir_uidlist_rec_flag flags;
bool new_changed, cur_changed, lock_failure;
const char *fname;
enum maildir_scan_why why;
int ret;
*lost_files_r = FALSE;
if (forced) {
new_changed = cur_changed = TRUE;
why = WHY_FORCED;
} else {
ret = maildir_sync_get_changes(ctx, &new_changed, &cur_changed,
&why);
if (ret <= 0)
return ret;
}
/*
Locking, locking, locking.. Wasn't maildir supposed to be lockless?
We can get here either as beginning a real maildir sync, or when
committing changes to maildir but a file was lost (maybe renamed).
So, we're going to need two locks. One for index and one for
uidlist. To avoid deadlocking do the uidlist lock always first.
uidlist is needed only for figuring out UIDs for newly seen files,
so theoretically we wouldn't need to lock it unless there are new
files. It has a few problems though, assuming the index lock didn't
already protect it (eg. in-memory indexes):
1. Just because you see a new file which doesn't exist in uidlist
file, doesn't mean that the file really exists anymore, or that
your readdir() lists all new files. Meaning that this is possible:
A: opendir(), readdir() -> new file ...
-- new files are written to the maildir --
B: opendir(), readdir() -> new file, lock uidlist,
readdir() -> another new file, rewrite uidlist, unlock
A: ... lock uidlist, readdir() -> nothing left, rewrite uidlist,
unlock
The second time running A didn't see the two new files. To handle
this correctly, it must not remove the new unseen files from
uidlist. This is possible to do, but adds extra complexity.
2. If another process is rename()ing files while we are
readdir()ing, it's possible that readdir() never lists some files,
causing Dovecot to assume they were expunged. In next sync they
would show up again, but client could have already been notified of
that and they would show up under new UIDs, so the damage is
already done.
Both of the problems can be avoided if we simply lock the uidlist
before syncing and keep it until sync is finished. Typically this
would happen in any case, as there is the index lock..
The second case is still a problem with external changes though,
because maildir doesn't require any kind of locking. Luckily this
problem rarely happens except under high amount of modifications.
*/
if (!cur_changed) {
ctx->partial = TRUE;
sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL;
} else {
ctx->partial = FALSE;
sync_flags = 0;
if (forced)
sync_flags |= MAILDIR_UIDLIST_SYNC_FORCE;
if ((ctx->flags & MAILBOX_SYNC_FLAG_FAST) != 0)
sync_flags |= MAILDIR_UIDLIST_SYNC_TRYLOCK;
}
ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags,
&ctx->uidlist_sync_ctx);
lock_failure = ret <= 0;
if (ret <= 0) {
struct mail_storage *storage = ctx->mbox->box.storage;
if (ret == 0) {
/* timeout */
return 0;
}
/* locking failed. sync anyway without locking so that it's
possible to expunge messages when out of quota. */
if (forced) {
/* we're already forcing a sync, we're trying to find
a message that was probably already expunged, don't
loop for a long time trying to find it. */
return -1;
}
ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags |
MAILDIR_UIDLIST_SYNC_NOLOCK,
&ctx->uidlist_sync_ctx);
if (ret <= 0) {
i_assert(ret != 0);
return -1;
}
if (storage->callbacks.notify_no != NULL) {
storage->callbacks.notify_no(&ctx->mbox->box,
"Internal mailbox synchronization failure, "
"showing only old mails.",
storage->callback_context);
}
}
ctx->locked = maildir_uidlist_is_locked(ctx->mbox->uidlist);
if (!ctx->locked)
ctx->partial = TRUE;
if (!ctx->mbox->syncing_commit && (ctx->locked || lock_failure)) {
if (maildir_sync_index_begin(ctx->mbox, ctx,
&ctx->index_sync_ctx) < 0)
return -1;
}
if (new_changed || cur_changed) {
/* if we're going to check cur/ dir our current logic requires
that new/ dir is checked as well. it's a good idea anyway. */
unsigned int count = 0;
bool final = FALSE;
while ((ret = maildir_scan_dir(ctx, TRUE, final, why)) > 0) {
/* rename()d at least some files, which might have
caused some other files to be missed. check again
(see MAILDIR_RENAME_RESCAN_COUNT). */
if (++count >= MAILDIR_SCAN_DIR_MAX_COUNT)
final = TRUE;
}
if (ret < 0)
return -1;
if (cur_changed) {
if (maildir_scan_dir(ctx, FALSE, TRUE, why) < 0)
return -1;
}
maildir_sync_update_next_uid(ctx->mbox);
/* finish uidlist syncing, but keep it still locked */
maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
}
if (!ctx->locked) {
/* make sure we sync the maildir later */
ctx->mbox->maildir_hdr.new_mtime = 0;
ctx->mbox->maildir_hdr.cur_mtime = 0;
}
if (ctx->index_sync_ctx != NULL) {
/* NOTE: index syncing here might cause a re-sync due to
files getting lost, so this function might be called
re-entrantly. */
ret = maildir_sync_index(ctx->index_sync_ctx, ctx->partial);
if (ret < 0)
maildir_sync_index_rollback(&ctx->index_sync_ctx);
else if (maildir_sync_index_commit(&ctx->index_sync_ctx) < 0)
return -1;
if (ret < 0)
return -1;
if (ret == 0)
*lost_files_r = TRUE;
i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist) ||
lock_failure);
}
if (find_uid != NULL && *find_uid != 0) {
ret = maildir_uidlist_lookup(ctx->mbox->uidlist,
*find_uid, &flags, &fname);
if (ret < 0)
return -1;
if (ret == 0) {
/* UID is expunged */
*find_uid = 0;
} else if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) {
/* we didn't find it, possibly expunged? */
*find_uid = 0;
}
}
return maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, TRUE);
}
int maildir_sync_lookup(struct maildir_mailbox *mbox, uint32_t uid,
enum maildir_uidlist_rec_flag *flags_r,
const char **fname_r)
{
int ret;
ret = maildir_uidlist_lookup(mbox->uidlist, uid, flags_r, fname_r);
if (ret != 0)
return ret;
if (maildir_uidlist_is_open(mbox->uidlist)) {
/* refresh uidlist and check again in case it was added
after the last mailbox sync */
if (mbox->sync_uidlist_refreshed) {
/* we've already refreshed it, don't bother again */
return ret;
}
mbox->sync_uidlist_refreshed = TRUE;
if (maildir_uidlist_refresh(mbox->uidlist) < 0)
return -1;
} else {
/* the uidlist doesn't exist. */
if (maildir_storage_sync_force(mbox, uid) < 0)
return -1;
}
/* try again */
return maildir_uidlist_lookup(mbox->uidlist, uid, flags_r, fname_r);
}
static int maildir_sync_run(struct maildir_mailbox *mbox,
enum mailbox_sync_flags flags, bool force_resync,
uint32_t *uid, bool *lost_files_r)
{
struct maildir_sync_context *ctx;
bool retry, lost_files;
int ret;
T_BEGIN {
ctx = maildir_sync_context_new(mbox, flags);
ret = maildir_sync_context(ctx, force_resync, uid, lost_files_r);
retry = ctx->racing;
maildir_sync_deinit(ctx);
} T_END;
if (retry) T_BEGIN {
/* we're racing some file. retry the sync again to see if the
file is really gone or not. if it is, this is a bit of
unnecessary work, but if it's not, this is necessary for
e.g. doveadm force-resync to work. */
ctx = maildir_sync_context_new(mbox, 0);
ret = maildir_sync_context(ctx, TRUE, NULL, &lost_files);
maildir_sync_deinit(ctx);
} T_END;
return ret;
}
int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid)
{
bool lost_files;
int ret;
ret = maildir_sync_run(mbox, MAILBOX_SYNC_FLAG_FAST,
TRUE, &uid, &lost_files);
if (uid != 0) {
/* maybe it's expunged. check again. */
ret = maildir_sync_run(mbox, 0, TRUE, NULL, &lost_files);
}
return ret;
}
int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox)
{
struct mail_index_view_sync_ctx *sync_ctx;
bool delayed_expunges;
mail_index_refresh(mbox->box.index);
if (mbox->flags_view == NULL)
mbox->flags_view = mail_index_view_open(mbox->box.index);
sync_ctx = mail_index_view_sync_begin(mbox->flags_view,
MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
if (mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) < 0) {
mailbox_set_index_error(&mbox->box);
return -1;
}
/* make sure the map stays in private memory */
if (mbox->flags_view->map->refcount > 1) {
struct mail_index_map *map;
map = mail_index_map_clone(mbox->flags_view->map);
mail_index_unmap(&mbox->flags_view->map);
mbox->flags_view->map = map;
}
mail_index_record_map_move_to_private(mbox->flags_view->map);
mail_index_map_move_to_memory(mbox->flags_view->map);
return 0;
}
struct mailbox_sync_context *
maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
bool lost_files, force_resync;
int ret = 0;
if (!box->opened) {
if (mailbox_open(box) < 0)
return index_mailbox_sync_init(box, flags, TRUE);
}
force_resync = (flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0;
if (index_mailbox_want_full_sync(&mbox->box, flags)) {
ret = maildir_sync_run(mbox, flags, force_resync,
NULL, &lost_files);
i_assert(!maildir_uidlist_is_locked(mbox->uidlist) ||
(box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0);
if (lost_files) {
/* lost some files from new/, see if thery're in cur/ */
ret = maildir_storage_sync_force(mbox, 0);
}
}
if (mbox->storage->set->maildir_very_dirty_syncs) {
if (maildir_sync_refresh_flags_view(mbox) < 0)
ret = -1;
maildir_uidlist_set_all_nonsynced(mbox->uidlist);
}
mbox->synced = TRUE;
mbox->sync_uidlist_refreshed = FALSE;
return index_mailbox_sync_init(box, flags, ret < 0);
}
int maildir_sync_is_synced(struct maildir_mailbox *mbox)
{
bool new_changed, cur_changed;
enum maildir_scan_why why;
int ret;
T_BEGIN {
const char *box_path = mailbox_get_path(&mbox->box);
const char *new_dir, *cur_dir;
new_dir = t_strconcat(box_path, "/new", NULL);
cur_dir = t_strconcat(box_path, "/cur", NULL);
ret = maildir_sync_quick_check(mbox, FALSE, new_dir, cur_dir,
&new_changed, &cur_changed,
&why);
} T_END;
return ret < 0 ? -1 : (!new_changed && !cur_changed);
}