maildir-sync.c revision 0add8c99ca65e56dbf613595fc37c41aafff3f7f
3853N/A/*
3853N/A 1. read files in maildir
3853N/A 2. see if they're all found in uidlist read in memory
3853N/A 3. if not, check if uidlist's mtime has changed and read it if so
3853N/A 4. if still not, lock uidlist, sync it once more and generate UIDs for new
3853N/A files
3853N/A 5. apply changes in transaction log
3853N/A 6. apply changes in maildir to index
6983N/A*/
6983N/A
3853N/A/* Copyright (C) 2004 Timo Sirainen */
3853N/A
3853N/A#include "lib.h"
3853N/A#include "ioloop.h"
6983N/A#include "buffer.h"
6983N/A#include "hash.h"
6983N/A#include "str.h"
6983N/A#include "maildir-storage.h"
3853N/A#include "maildir-uidlist.h"
3853N/A
3853N/A#include <stdio.h>
3853N/A#include <unistd.h>
3853N/A#include <dirent.h>
3853N/A#include <sys/stat.h>
3853N/A
3853N/A#define MAILDIR_SYNC_SECS 1
3853N/A
3853N/A#define MAILDIR_FILENAME_FLAG_FOUND 128
3853N/A
3853N/Astruct maildir_sync_context {
3853N/A struct index_mailbox *ibox;
3853N/A const char *new_dir, *cur_dir;
3853N/A
3853N/A struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
3853N/A};
3853N/A
3853N/Astatic int maildir_expunge(struct index_mailbox *ibox, const char *path,
3853N/A void *context __attr_unused__)
3853N/A{
3853N/A if (unlink(path) == 0)
3853N/A return 1;
3853N/A if (errno == ENOENT)
3853N/A return 0;
3853N/A
3853N/A mail_storage_set_critical(ibox->box.storage,
3853N/A "unlink(%s) failed: %m", path);
3853N/A return -1;
3853N/A}
3853N/A
3853N/Astatic int maildir_sync_flags(struct index_mailbox *ibox, const char *path,
3853N/A void *context)
3853N/A{
3853N/A struct mail_index_sync_rec *syncrec = context;
3853N/A const char *newpath;
3853N/A enum mail_flags flags;
3853N/A uint8_t flags8;
3853N/A custom_flags_mask_t custom_flags;
3853N/A
3853N/A (void)maildir_filename_get_flags(path, &flags, custom_flags);
3853N/A
3853N/A flags8 = flags;
3853N/A mail_index_sync_flags_apply(syncrec, &flags8, custom_flags);
3853N/A
3853N/A newpath = maildir_filename_set_flags(path, flags8, custom_flags);
3853N/A if (rename(path, newpath) == 0)
3853N/A return 1;
3853N/A if (errno == ENOENT)
3853N/A return 0;
3853N/A
3853N/A mail_storage_set_critical(ibox->box.storage,
3853N/A "rename(%s, %s) failed: %m", path, newpath);
3853N/A return -1;
3853N/A}
3853N/A
3853N/Astatic int maildir_sync_record(struct index_mailbox *ibox,
3853N/A struct mail_index_view *view,
struct mail_index_sync_rec *syncrec)
{
uint32_t seq, uid;
switch (syncrec->type) {
case MAIL_INDEX_SYNC_TYPE_APPEND:
break;
case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
for (seq = syncrec->seq1; seq <= syncrec->seq2; seq++) {
if (mail_index_lookup_uid(view, seq, &uid) < 0)
return -1;
if (maildir_file_do(ibox, uid, maildir_expunge,
NULL) < 0)
return -1;
}
break;
case MAIL_INDEX_SYNC_TYPE_FLAGS:
for (seq = syncrec->seq1; seq <= syncrec->seq2; seq++) {
if (mail_index_lookup_uid(view, seq, &uid) < 0)
return -1;
if (maildir_file_do(ibox, uid, maildir_sync_flags,
syncrec) < 0)
return -1;
}
break;
}
return 0;
}
int maildir_sync_last_commit(struct index_mailbox *ibox)
{
struct mail_index_view *view;
struct mail_index_sync_ctx *sync_ctx;
struct mail_index_sync_rec sync_rec;
int ret;
if (ibox->commit_log_file_seq == 0)
return 0;
ret = mail_index_sync_begin(ibox->index, &sync_ctx, &view,
ibox->commit_log_file_seq,
ibox->commit_log_file_offset);
if (ret > 0) {
while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) {
if (maildir_sync_record(ibox, view, &sync_rec) < 0) {
ret = -1;
break;
}
}
if (mail_index_sync_end(sync_ctx) < 0)
ret = -1;
}
if (ret == 0) {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
} else {
mail_storage_set_index_error(ibox);
}
return ret;
}
static struct maildir_sync_context *
maildir_sync_context_new(struct index_mailbox *ibox)
{
struct maildir_sync_context *ctx;
ctx = t_new(struct maildir_sync_context, 1);
ctx->ibox = ibox;
ctx->new_dir = t_strconcat(ibox->path, "/new", NULL);
ctx->cur_dir = t_strconcat(ibox->path, "/cur", NULL);
return ctx;
}
static void maildir_sync_deinit(struct maildir_sync_context *ctx)
{
if (ctx->uidlist_sync_ctx != NULL)
(void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
}
static int maildir_fix_duplicate(struct index_mailbox *ibox, const char *dir,
const char *old_fname)
{
const char *new_fname, *old_path, *new_path;
int ret = 0;
t_push();
old_path = t_strconcat(dir, "/", old_fname, NULL);
new_fname = maildir_generate_tmp_filename(&ioloop_timeval);
new_path = t_strconcat(ibox->path, "/new/", new_fname, NULL);
if (rename(old_path, new_path) == 0) {
i_warning("Fixed duplicate in %s: %s -> %s",
ibox->path, old_fname, new_fname);
} else if (errno != ENOENT) {
mail_storage_set_critical(ibox->box.storage,
"rename(%s, %s) failed: %m", old_path, new_path);
ret = -1;
}
t_pop();
return ret;
}
static int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
{
struct mail_storage *storage = ctx->ibox->box.storage;
const char *dir;
DIR *dirp;
string_t *src, *dest;
struct dirent *dp;
int move_new, this_new, ret = 1;
src = t_str_new(1024);
dest = t_str_new(1024);
dir = new_dir ? ctx->new_dir : ctx->cur_dir;
dirp = opendir(dir);
if (dirp == NULL) {
mail_storage_set_critical(storage,
"opendir(%s) failed: %m", dir);
return -1;
}
move_new = new_dir;
while ((dp = readdir(dirp)) != NULL) {
if (dp->d_name[0] == '.')
continue;
this_new = new_dir;
if (move_new) {
str_truncate(src, 0);
str_truncate(dest, 0);
str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
if (rename(str_c(src), str_c(dest)) == 0 ||
ENOTFOUND(errno)) {
/* moved - we'll look at it later in cur/ dir */
this_new = FALSE;
continue;
} else if (ENOSPACE(errno)) {
/* not enough disk space, leave here */
move_new = FALSE;
} else {
mail_storage_set_critical(storage,
"rename(%s, %s) failed: %m",
str_c(src), str_c(dest));
}
}
ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
dp->d_name, this_new);
if (ret <= 0) {
if (ret < 0)
break;
/* possibly duplicate - try fixing it */
if (maildir_fix_duplicate(ctx->ibox,
dir, dp->d_name) < 0) {
ret = -1;
break;
}
}
}
if (closedir(dirp) < 0) {
mail_storage_set_critical(storage,
"closedir(%s) failed: %m", dir);
}
return ret < 0 ? -1 : 0;
}
static int maildir_sync_quick_check(struct maildir_sync_context *ctx,
int *new_changed_r, int *cur_changed_r)
{
struct index_mailbox *ibox = ctx->ibox;
struct stat st;
time_t new_mtime, cur_mtime;
*new_changed_r = *cur_changed_r = FALSE;
if (stat(ctx->new_dir, &st) < 0) {
mail_storage_set_critical(ibox->box.storage,
"stat(%s) failed: %m", ctx->new_dir);
return -1;
}
new_mtime = st.st_mtime;
if (stat(ctx->cur_dir, &st) < 0) {
mail_storage_set_critical(ibox->box.storage,
"stat(%s) failed: %m", ctx->cur_dir);
return -1;
}
cur_mtime = st.st_mtime;
if (new_mtime != ibox->last_new_mtime ||
new_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS) {
*new_changed_r = TRUE;
ibox->last_new_mtime = new_mtime;
}
if (cur_mtime != ibox->last_cur_mtime ||
(cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS &&
ioloop_time - ibox->last_sync > MAILDIR_SYNC_SECS)) {
/* cur/ changed, or delayed cur/ check */
*cur_changed_r = TRUE;
ibox->last_cur_mtime = cur_mtime;
}
ibox->last_sync = ioloop_time;
return 0;
}
static int maildir_sync_index(struct maildir_sync_context *ctx)
{
struct index_mailbox *ibox = ctx->ibox;
struct mail_index_sync_ctx *sync_ctx;
struct mail_index_sync_rec sync_rec;
struct maildir_uidlist_iter_ctx *iter;
struct mail_index_transaction *trans;
struct mail_index_view *view;
const struct mail_index_header *hdr;
const struct mail_index_record *rec;
uint32_t seq, uid, uflags;
const char *filename;
enum mail_flags flags;
custom_flags_mask_t custom_flags;
int ret;
if (mail_index_sync_begin(ibox->index, &sync_ctx, &view,
(uint32_t)-1, (uoff_t)-1) <= 0) {
mail_storage_set_index_error(ibox);
return -1;
}
ret = mail_index_get_header(view, &hdr);
i_assert(ret == 0); /* view is locked, can't happen */
trans = mail_index_transaction_begin(view, FALSE);
seq = 0;
iter = maildir_uidlist_iter_init(ibox->uidlist);
while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
maildir_filename_get_flags(filename, &flags, custom_flags);
__again:
seq++;
if (seq > hdr->messages_count) {
mail_index_append(trans, uid, &seq);
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
flags, custom_flags);
continue;
}
if (mail_index_lookup(view, seq, &rec) < 0) {
mail_storage_set_index_error(ibox);
ret = -1;
break;
}
if (rec->uid < uid) {
/* expunged */
mail_index_expunge(trans, seq);
goto __again;
}
if (rec->uid > uid) {
/* new UID in the middle of the mailbox -
shouldn't happen */
mail_storage_set_critical(ibox->box.storage,
"Maildir sync: UID inserted in the middle "
"of mailbox (%u > %u)", rec->uid, uid);
(void)mail_index_reset(ibox->index);
ret = -1;
break;
}
maildir_filename_get_flags(filename, &flags, custom_flags);
if (rec->flags & MAIL_RECENT)
flags |= MAIL_RECENT;
if ((uint8_t)flags != (rec->flags & MAIL_FLAGS_MASK) ||
memcmp(custom_flags, rec->custom_flags,
INDEX_CUSTOM_FLAGS_BYTE_COUNT) != 0) {
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
flags, custom_flags);
}
}
maildir_uidlist_iter_deinit(iter);
if (ret < 0)
mail_index_transaction_rollback(trans);
else {
uint32_t seq;
uoff_t offset;
if (mail_index_transaction_commit(trans, &seq, &offset) < 0)
mail_storage_set_index_error(ibox);
else {
ibox->commit_log_file_seq = seq;
ibox->commit_log_file_offset = offset;
}
}
/* now, sync the index */
while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) {
if (maildir_sync_record(ibox, view, &sync_rec) < 0) {
ret = -1;
break;
}
}
if (mail_index_sync_end(sync_ctx) < 0)
ret = -1;
if (ret == 0) {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
} else {
mail_storage_set_index_error(ibox);
}
return ret;
}
static int maildir_sync_context(struct maildir_sync_context *ctx,
int *changes_r)
{
int ret, new_changed, cur_changed;
if (maildir_sync_quick_check(ctx, &new_changed, &cur_changed) < 0)
return -1;
ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
if (maildir_scan_dir(ctx, FALSE) < 0)
return -1;
ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
ctx->uidlist_sync_ctx = NULL;
if (ret == 0)
ret = maildir_sync_index(ctx);
return ret;
}
static int maildir_sync_context_readonly(struct maildir_sync_context *ctx)
{
int ret;
ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
if (maildir_scan_dir(ctx, FALSE) < 0)
return -1;
ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
ctx->uidlist_sync_ctx = NULL;
return ret;
}
int maildir_storage_sync_readonly(struct index_mailbox *ibox)
{
struct maildir_sync_context *ctx;
int ret;
ctx = maildir_sync_context_new(ibox);
ret = maildir_sync_context_readonly(ctx);
maildir_sync_deinit(ctx);
return ret;
}
int maildir_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
struct maildir_sync_context *ctx;
int changes, ret;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
ctx = maildir_sync_context_new(ibox);
ret = maildir_sync_context(ctx, &changes);
maildir_sync_deinit(ctx);
if (ret < 0)
return -1;
}
return index_storage_sync(box, flags);
}