55a7410569737197afb302b07b488973324b0cc5Timo Sirainen

#include "lib.h"

#include "ioloop.h"

#include "array.h"

#include "buffer.h"

#include "hash.h"

#include "str.h"

#include "maildir-storage.h"

#include "maildir-uidlist.h"

#include "maildir-keywords.h"

#include <stdio.h>

#include <stddef.h>

#include <unistd.h>

#include <dirent.h>

#include <sys/stat.h>

#define MAILDIR_SYNC_SECS 1

#define MAILDIR_FILENAME_FLAG_FOUND 128

#define MAILDIR_RENAME_RESCAN_COUNT 5

struct maildir_sync_context {

struct maildir_mailbox *mbox;

const char *new_dir, *cur_dir;

bool partial;

struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;

struct maildir_index_sync_context *index_sync_ctx;

};

struct maildir_index_sync_context {

struct maildir_mailbox *mbox;

struct mail_index_view *view;

struct mail_index_sync_ctx *sync_ctx;

struct maildir_keywords_sync_ctx *keywords_sync_ctx;

struct mail_index_transaction *trans;

array_t ARRAY_DEFINE(sync_recs, struct mail_index_sync_rec);

uint32_t seq;

int dirty_state;

};

struct maildir_keywords_sync_ctx *

maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx)

{

return ctx->keywords_sync_ctx;

}

int maildir_filename_get_flags(struct maildir_keywords_sync_ctx *ctx,

const char *fname, enum mail_flags *flags_r,

array_t *keywords_r)

{

ARRAY_SET_TYPE(keywords_r, unsigned int);

const char *info;

array_clear(keywords_r);

*flags_r = 0;

info = strchr(fname, MAILDIR_INFO_SEP);

if (info == NULL || info[1] != '2' || info[2] != MAILDIR_FLAGS_SEP)

return 0;

for (info += 3; *info != '\0' && *info != MAILDIR_FLAGS_SEP; info++) {

switch (*info) {

case 'R': /* replied */

*flags_r |= MAIL_ANSWERED;

break;

case 'S': /* seen */

*flags_r |= MAIL_SEEN;

break;

case 'T': /* trashed */

*flags_r |= MAIL_DELETED;

break;

case 'D': /* draft */

*flags_r |= MAIL_DRAFT;

break;

case 'F': /* flagged */

*flags_r |= MAIL_FLAGGED;

break;

default:

if (*info >= MAILDIR_KEYWORD_FIRST &&

*info <= MAILDIR_KEYWORD_LAST) {

int idx;

idx = maildir_keywords_char_idx(ctx, *info);

if (idx < 0) {

/* unknown keyword. */

break;

}

array_append(keywords_r,

(unsigned int *)&idx, 1);

break;

}

/* unknown flag - ignore */

break;

}

}

return 1;

}

static void

maildir_filename_append_keywords(struct maildir_keywords_sync_ctx *ctx,

array_t *keywords, string_t *str)

{

ARRAY_SET_TYPE(keywords, unsigned int);

const unsigned int *indexes;

unsigned int i, count;

char chr;

indexes = array_get(keywords, &count);

for (i = 0; i < count; i++) {

chr = maildir_keywords_idx_char(ctx, indexes[i]);

if (chr != '\0')

str_append_c(str, chr);

}

}

const char *maildir_filename_set_flags(struct maildir_keywords_sync_ctx *ctx,

const char *fname, enum mail_flags flags,

array_t *keywords)

{

string_t *flags_str;

enum mail_flags flags_left;

const char *info, *oldflags;

int nextflag;

/* remove the old :info from file name, and get the old flags */

info = strrchr(fname, MAILDIR_INFO_SEP);

if (info != NULL && strrchr(fname, '/') > info)

info = NULL;

oldflags = "";

if (info != NULL) {

fname = t_strdup_until(fname, info);

if (info[1] == '2' && info[2] == MAILDIR_FLAGS_SEP)

oldflags = info+3;

}

/* insert the new flags between old flags. flags must be sorted by

flags_str = t_str_new(256);

str_append(flags_str, fname);

str_append(flags_str, MAILDIR_FLAGS_FULL_SEP);

flags_left = flags;

for (;;) {

/* skip all known flags */

while (*oldflags == 'D' || *oldflags == 'F' ||

*oldflags == 'R' || *oldflags == 'S' ||

*oldflags == 'T' ||

(*oldflags >= MAILDIR_KEYWORD_FIRST &&

*oldflags <= MAILDIR_KEYWORD_LAST))

oldflags++;

nextflag = *oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP ?

256 : (unsigned char) *oldflags;

if ((flags_left & MAIL_DRAFT) && nextflag > 'D') {

str_append_c(flags_str, 'D');

flags_left &= ~MAIL_DRAFT;

}

if ((flags_left & MAIL_FLAGGED) && nextflag > 'F') {

str_append_c(flags_str, 'F');

flags_left &= ~MAIL_FLAGGED;

}

if ((flags_left & MAIL_ANSWERED) && nextflag > 'R') {

str_append_c(flags_str, 'R');

flags_left &= ~MAIL_ANSWERED;

}

if ((flags_left & MAIL_SEEN) && nextflag > 'S') {

str_append_c(flags_str, 'S');

flags_left &= ~MAIL_SEEN;

}

if ((flags_left & MAIL_DELETED) && nextflag > 'T') {

str_append_c(flags_str, 'T');

flags_left &= ~MAIL_DELETED;

}

if (keywords != NULL && array_is_created(keywords) &&

nextflag > MAILDIR_KEYWORD_FIRST) {

maildir_filename_append_keywords(ctx, keywords,

flags_str);

keywords = NULL;

}

if (*oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP)

break;

str_append_c(flags_str, *oldflags);

oldflags++;

}

if (*oldflags == MAILDIR_FLAGS_SEP) {

/* another flagset, we don't know about these, just keep them */

while (*oldflags != '\0')

str_append_c(flags_str, *oldflags++);

}

return str_c(flags_str);

}

static int maildir_expunge(struct maildir_mailbox *mbox, const char *path,

void *context __attr_unused__)

{

if (unlink(path) == 0) {

mbox->dirty_cur_time = ioloop_time;

return 1;

}

if (errno == ENOENT)

return 0;

mail_storage_set_critical(STORAGE(mbox->storage),

"unlink(%s) failed: %m", path);

return -1;

}

static int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path,

void *context)

{

struct maildir_index_sync_context *ctx = context;

const struct mail_index_sync_rec *recs;

const char *newpath;

enum mail_flags flags;

array_t ARRAY_DEFINE(keywords, unsigned int);

unsigned int i, count;

uint8_t flags8;

ctx->dirty_state = 0;

ARRAY_CREATE(&keywords, pool_datastack_create(), unsigned int, 16);

(void)maildir_filename_get_flags(ctx->keywords_sync_ctx,

path, &flags, &keywords);

flags8 = flags;

recs = array_get_modifyable(&ctx->sync_recs, &count);

for (i = 0; i < count; i++) {

if (recs[i].uid1 != ctx->seq)

break;

switch (recs[i].type) {

case MAIL_INDEX_SYNC_TYPE_FLAGS:

mail_index_sync_flags_apply(&recs[i], &flags8);

break;

case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:

case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:

case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:

mail_index_sync_keywords_apply(&recs[i], &keywords);

break;

case MAIL_INDEX_SYNC_TYPE_APPEND:

case MAIL_INDEX_SYNC_TYPE_EXPUNGE:

i_unreached();

break;

}

}

newpath = maildir_filename_set_flags(ctx->keywords_sync_ctx,

path, flags8, &keywords);

if (rename(path, newpath) == 0) {

if ((flags8 & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)

ctx->dirty_state = -1;

mbox->dirty_cur_time = ioloop_time;

return 1;

}

if (errno == ENOENT)

return 0;

if (ENOSPACE(errno) || errno == EACCES) {

mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_ADD,

MAIL_INDEX_MAIL_FLAG_DIRTY);

ctx->dirty_state = 1;

return 1;

}

mail_storage_set_critical(STORAGE(mbox->storage),

"rename(%s, %s) failed: %m", path, newpath);

return -1;

}

static int

maildir_sync_record_commit_until(struct maildir_index_sync_context *ctx,

uint32_t last_seq)

{

struct mail_index_sync_rec *recs;

unsigned int i, count;

uint32_t seq, uid;

bool expunged, flag_changed;

recs = array_get_modifyable(&ctx->sync_recs, &count);

for (seq = recs[0].uid1; seq <= last_seq; seq++) {

expunged = flag_changed = FALSE;

for (i = 0; i < count; i++) {

if (recs[i].uid1 > seq)

break;

i_assert(recs[i].uid1 == seq);

switch (recs[i].type) {

case MAIL_INDEX_SYNC_TYPE_EXPUNGE:

expunged = TRUE;

break;

case MAIL_INDEX_SYNC_TYPE_FLAGS:

case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:

case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:

case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:

flag_changed = TRUE;

break;

case MAIL_INDEX_SYNC_TYPE_APPEND:

i_unreached();

break;

}

}

if (mail_index_lookup_uid(ctx->view, seq, &uid) < 0)

return -1;

ctx->seq = seq;

if (expunged) {

if (maildir_file_do(ctx->mbox, uid,

maildir_expunge, ctx) < 0)

return -1;

} else if (flag_changed) {

if (maildir_file_do(ctx->mbox, uid,

maildir_sync_flags, ctx) < 0)

return -1;

}

for (i = count; i > 0; i--) {

if (++recs[i-1].uid1 > recs[i-1].uid2) {

array_delete(&ctx->sync_recs, i-1, 1);

recs = array_get_modifyable(&ctx->sync_recs,

&count);

if (count == 0) {

/* all sync_recs committed */

return 0;

}

}

}

}

return 0;

}

static int maildir_sync_record(struct maildir_index_sync_context *ctx,

const struct mail_index_sync_rec *sync_rec)

{

struct mail_index_view *view = ctx->view;

struct mail_index_sync_rec sync_copy;

if (sync_rec == NULL) {

/* deinit */

while (array_count(&ctx->sync_recs) > 0) {

if (maildir_sync_record_commit_until(ctx,

(uint32_t)-1) < 0)

return -1;

}

return 0;

}

if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND)

return 0; /* ignore */

/* convert to sequences to avoid looping through huge holes in

sync_copy = *sync_rec;

if (mail_index_lookup_uid_range(view, sync_rec->uid1,

sync_rec->uid2,

&sync_copy.uid1,

&sync_copy.uid2) < 0)

return -1;

if (sync_copy.uid1 == 0) {

/* UIDs were expunged */

return 0;

}

while (array_count(&ctx->sync_recs) > 0) {

const struct mail_index_sync_rec *rec =

array_idx(&ctx->sync_recs, 0);

i_assert(rec->uid1 <= sync_copy.uid1);

if (rec->uid1 == sync_copy.uid1)

break;

if (maildir_sync_record_commit_until(ctx, sync_copy.uid1-1) < 0)

return -1;

}

array_append(&ctx->sync_recs, &sync_copy, 1);

return 0;

}

static int maildir_sync_index_records(struct maildir_index_sync_context *ctx)

{

struct mail_index_sync_rec sync_rec;

int ret;

ret = mail_index_sync_next(ctx->sync_ctx, &sync_rec);

if (ret <= 0)

return ret;

ARRAY_CREATE(&ctx->sync_recs, pool_datastack_create(),

struct mail_index_sync_rec, 32);

do {

if (maildir_sync_record(ctx, &sync_rec) < 0)

return -1;

ret = mail_index_sync_next(ctx->sync_ctx, &sync_rec);

} while (ret > 0);

if (maildir_sync_record(ctx, NULL) < 0)

return -1;

return ret;

}

static struct maildir_sync_context *

maildir_sync_context_new(struct maildir_mailbox *mbox)

{

struct maildir_sync_context *ctx;

ctx = t_new(struct maildir_sync_context, 1);

ctx->mbox = mbox;

ctx->new_dir = t_strconcat(mbox->path, "/new", NULL);

ctx->cur_dir = t_strconcat(mbox->path, "/cur", NULL);

return ctx;

}

static void maildir_sync_deinit(struct maildir_sync_context *ctx)

{

if (ctx->uidlist_sync_ctx != NULL)

(void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);

if (ctx->index_sync_ctx != NULL)

maildir_sync_index_abort(ctx->index_sync_ctx);

}

static int maildir_fix_duplicate(struct maildir_mailbox *mbox, const char *dir,

const char *old_fname)

{

const char *new_fname, *old_path, *new_path;

int ret = 0;

t_push();

old_path = t_strconcat(dir, "/", old_fname, NULL);

new_fname = maildir_generate_tmp_filename(&ioloop_timeval);

new_path = t_strconcat(mbox->path, "/new/", new_fname, NULL);

if (rename(old_path, new_path) == 0) {

i_warning("Fixed duplicate in %s: %s -> %s",

mbox->path, old_fname, new_fname);

} else if (errno != ENOENT) {

mail_storage_set_critical(STORAGE(mbox->storage),

"rename(%s, %s) failed: %m", old_path, new_path);

ret = -1;

}

t_pop();

return ret;

}

static int maildir_scan_dir(struct maildir_sync_context *ctx, bool new_dir)

{

struct mail_storage *storage = STORAGE(ctx->mbox->storage);

const char *dir;

DIR *dirp;

string_t *src, *dest;

struct dirent *dp;

enum maildir_uidlist_rec_flag flags;

unsigned int moves = 0;

int ret = 1;

bool move_new;

dir = new_dir ? ctx->new_dir : ctx->cur_dir;

dirp = opendir(dir);

if (dirp == NULL) {

mail_storage_set_critical(storage,

"opendir(%s) failed: %m", dir);

return -1;

}

t_push();

src = t_str_new(1024);

dest = t_str_new(1024);

move_new = new_dir && !mailbox_is_readonly(&ctx->mbox->ibox.box) &&

!ctx->mbox->ibox.keep_recent;

while ((dp = readdir(dirp)) != NULL) {

if (dp->d_name[0] == '.')

continue;

ret = maildir_uidlist_sync_next_pre(ctx->uidlist_sync_ctx,

dp->d_name);

if (ret == 0) {

/* new file and we couldn't lock uidlist, check this

if (new_dir)

ctx->mbox->last_new_mtime = 0;

else

ctx->mbox->dirty_cur_time = ioloop_time;

continue;

}

if (ret < 0)

break;

flags = 0;

if (move_new) {

str_truncate(src, 0);

str_truncate(dest, 0);

str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);

str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);

if (strchr(dp->d_name, MAILDIR_INFO_SEP) == NULL) {

str_append(dest, MAILDIR_FLAGS_FULL_SEP);

}

if (rename(str_c(src), str_c(dest)) == 0) {

/* we moved it - it's \Recent for us */

moves++;

ctx->mbox->dirty_cur_time = ioloop_time;

flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |

MAILDIR_UIDLIST_REC_FLAG_RECENT;

} else if (ENOTFOUND(errno)) {

/* someone else moved it already */

moves++;

flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED;

} else if (ENOSPACE(errno) || errno == EACCES) {

/* not enough disk space / read-only maildir,

flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |

MAILDIR_UIDLIST_REC_FLAG_RECENT;

move_new = FALSE;

} else {

flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |

MAILDIR_UIDLIST_REC_FLAG_RECENT;

mail_storage_set_critical(storage,

"rename(%s, %s) failed: %m",

str_c(src), str_c(dest));

}

} else if (new_dir) {

flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |

MAILDIR_UIDLIST_REC_FLAG_RECENT;

}

ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,

dp->d_name, flags);

if (ret <= 0) {

if (ret < 0)

break;

/* possibly duplicate - try fixing it */

if (maildir_fix_duplicate(ctx->mbox,

dir, dp->d_name) < 0) {

ret = -1;

break;

}

}

}

if (closedir(dirp) < 0) {

mail_storage_set_critical(storage,

"closedir(%s) failed: %m", dir);

}

t_pop();

return ret < 0 ? -1 : (moves <= MAILDIR_RENAME_RESCAN_COUNT ? 0 : 1);

}

static void

maildir_sync_update_from_header(struct maildir_mailbox *mbox)

{

uint64_t value;

/* FIXME: ugly, replace with extension header */

value = mail_index_get_header(mbox->ibox.view)->sync_size;

mbox->last_new_mtime = value & 0xffffffff;

mbox->last_new_sync_time = value >> 32;

mbox->last_cur_mtime =

mail_index_get_header(mbox->ibox.view)->sync_stamp;

}

static int

maildir_sync_quick_check(struct maildir_mailbox *mbox,

const char *new_dir, const char *cur_dir,

bool *new_changed_r, bool *cur_changed_r)

{

struct index_mailbox *ibox = &mbox->ibox;

struct stat st;

time_t new_mtime, cur_mtime;

*new_changed_r = *cur_changed_r = FALSE;

if (stat(new_dir, &st) < 0) {

mail_storage_set_critical(STORAGE(mbox->storage),

"stat(%s) failed: %m", new_dir);

return -1;

}

new_mtime = st.st_mtime;

if (stat(cur_dir, &st) < 0) {

mail_storage_set_critical(STORAGE(mbox->storage),

"stat(%s) failed: %m", cur_dir);

return -1;

}

cur_mtime = st.st_mtime;

/* cur stamp is kept in index, we don't have to sync if

maildir_sync_update_from_header(mbox);

if ((mbox->dirty_cur_time == 0 && cur_mtime != mbox->last_cur_mtime) ||

(new_mtime != mbox->last_new_mtime)) {

/* check if the index has been updated.. */

if (mail_index_refresh(ibox->index) < 0) {

mail_storage_set_index_error(ibox);

return -1;

}

maildir_sync_update_from_header(mbox);

}

/* If we're removing recent flags, always sync new/ directory if

if (new_mtime != mbox->last_new_mtime ||

new_mtime >= mbox->last_new_sync_time - MAILDIR_SYNC_SECS ||

(!ibox->keep_recent &&

mail_index_get_header(ibox->view)->recent_messages_count > 0)) {

*new_changed_r = TRUE;

mbox->last_new_mtime = new_mtime;

mbox->last_new_sync_time = ioloop_time;

}

if (cur_mtime != mbox->last_cur_mtime ||

(mbox->dirty_cur_time != 0 &&

ioloop_time - mbox->dirty_cur_time > MAILDIR_SYNC_SECS)) {

/* cur/ changed, or delayed cur/ check */

*cur_changed_r = TRUE;

mbox->last_cur_mtime = cur_mtime;

mbox->dirty_cur_time =

cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?

cur_mtime : 0;

}

return 0;

}

struct maildir_index_sync_context *

maildir_sync_index_begin(struct maildir_mailbox *mbox)

{

struct maildir_index_sync_context *sync_ctx;

sync_ctx = i_new(struct maildir_index_sync_context, 1);

sync_ctx->mbox = mbox;

if (mail_index_sync_begin(mbox->ibox.index, &sync_ctx->sync_ctx,

&sync_ctx->view, (uint32_t)-1, (uoff_t)-1,

FALSE, FALSE) <= 0) {

mail_storage_set_index_error(&mbox->ibox);

i_free(sync_ctx);

return NULL;

}

sync_ctx->keywords_sync_ctx =

maildir_keywords_sync_init(mbox->keywords, mbox->ibox.index);

return sync_ctx;

}

void maildir_sync_index_abort(struct maildir_index_sync_context *sync_ctx)

{

mail_index_sync_rollback(&sync_ctx->sync_ctx);

maildir_keywords_sync_deinit(sync_ctx->keywords_sync_ctx);

i_free(sync_ctx);

}

int maildir_sync_index_finish(struct maildir_index_sync_context *sync_ctx,

bool partial)

{

struct maildir_mailbox *mbox = sync_ctx->mbox;

struct mail_index_view *view = sync_ctx->view;

struct maildir_uidlist_iter_ctx *iter;

struct mail_index_transaction *trans;

const struct mail_index_header *hdr;

const struct mail_index_record *rec;

uint32_t seq, uid;

enum maildir_uidlist_rec_flag uflags;

const char *filename;

enum mail_flags flags;

array_t ARRAY_DEFINE(keywords, unsigned int);

array_t ARRAY_DEFINE(idx_keywords, unsigned int);

uint32_t uid_validity, next_uid;

uint64_t value;

time_t old_new_sync_time;

int ret = 0;

bool full_rescan = FALSE;

i_assert(maildir_uidlist_is_locked(sync_ctx->mbox->uidlist));

trans = mail_index_transaction_begin(view, FALSE, TRUE);

sync_ctx->trans = trans;

hdr = mail_index_get_header(view);

uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);

if (uid_validity != hdr->uid_validity &&

uid_validity != 0 && hdr->uid_validity != 0) {

/* uidvalidity changed and mailbox isn't being initialized,

mail_storage_set_critical(STORAGE(mbox->storage),

"Maildir %s sync: UIDVALIDITY changed (%u -> %u)",

mbox->path, hdr->uid_validity, uid_validity);

mail_index_mark_corrupted(mbox->ibox.index);

return -1;

}

seq = 0;

ARRAY_CREATE(&keywords, pool_datastack_create(),

unsigned int, MAILDIR_MAX_KEYWORDS);

ARRAY_CREATE(&idx_keywords, pool_datastack_create(),

unsigned int, MAILDIR_MAX_KEYWORDS);

iter = maildir_uidlist_iter_init(mbox->uidlist);

while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {

maildir_filename_get_flags(sync_ctx->keywords_sync_ctx,

filename, &flags, &keywords);

/* the private flags are kept only in indexes. don't use them

flags &= ~mbox->private_flags_mask;

if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0 &&

(uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 &&

(uflags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0) {

/* mail is recent for next session as well */

flags |= MAIL_RECENT;

}

__again:

seq++;

if (seq > hdr->messages_count) {

if (uid < hdr->next_uid) {

/* most likely a race condition: we read the

if ((uflags &

MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {

/* partial syncing */

continue;

}

if ((uflags &

MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {

mail_storage_set_critical(

STORAGE(mbox->storage),

"Maildir %s sync: "

"UID < next_uid "

"(%u < %u, file = %s)",

mbox->path, uid, hdr->next_uid,

filename);

mail_index_mark_corrupted(

mbox->ibox.index);

ret = -1;

break;

}

mbox->dirty_cur_time = ioloop_time;

maildir_uidlist_add_flags(mbox->uidlist,

filename,

MAILDIR_UIDLIST_REC_FLAG_RACING);

seq--;

continue;

}

mail_index_append(trans, uid, &seq);

mail_index_update_flags(trans, seq, MODIFY_REPLACE,

flags);

if (array_count(&keywords) > 0) {

struct mail_keywords *kw;

kw = mail_index_keywords_create_from_indexes(

trans, &keywords);

mail_index_update_keywords(trans, seq,

MODIFY_REPLACE, kw);

mail_index_keywords_free(&kw);

}

continue;

}

if (mail_index_lookup(view, seq, &rec) < 0) {

ret = -1;

break;

}

if (rec->uid < uid) {

/* expunged */

mail_index_expunge(trans, seq);

goto __again;

}

if (rec->uid > uid) {

/* most likely a race condition: we read the

if ((uflags &

MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {

/* partial syncing */

seq--;

continue;

}

if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {

mail_storage_set_critical(

STORAGE(mbox->storage),

"Maildir %s sync: "

"UID inserted in the middle of mailbox "

"(%u > %u, file = %s)",

mbox->path, rec->uid, uid, filename);

mail_index_mark_corrupted(mbox->ibox.index);

ret = -1;

break;

}

mbox->dirty_cur_time = ioloop_time;

maildir_uidlist_add_flags(mbox->uidlist, filename,

MAILDIR_UIDLIST_REC_FLAG_RACING);

seq--;

continue;

}

/* the private flags are stored only in indexes, keep them */

flags |= rec->flags & mbox->private_flags_mask;

if ((rec->flags & MAIL_RECENT) != 0) {

index_mailbox_set_recent(&mbox->ibox, seq);

if (mbox->ibox.keep_recent) {

flags |= MAIL_RECENT;

} else {

mail_index_update_flags(trans, seq,

MODIFY_REMOVE,

MAIL_RECENT);

}

}

if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {

/* partial syncing */

if ((flags & MAIL_RECENT) != 0) {

/* we last saw this mail in new/, but it's

full_rescan = TRUE;

}

continue;

}

if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {

/* we haven't been able to update maildir with this

continue;

}

if (((uint8_t)flags & ~MAIL_RECENT) !=

(rec->flags & (MAIL_FLAGS_MASK^MAIL_RECENT))) {

/* FIXME: this is wrong if there's pending changes in

mail_index_update_flags(trans, seq, MODIFY_REPLACE,

flags);

} else if ((flags & MAIL_RECENT) == 0 &&

(rec->flags & MAIL_RECENT) != 0) {

/* just remove recent flag */

mail_index_update_flags(trans, seq, MODIFY_REMOVE,

MAIL_RECENT);

}

/* update keywords if they have changed */

if (mail_index_lookup_keywords(view, seq, &idx_keywords) < 0) {

ret = -1;

break;

}

if (!index_keyword_array_cmp(&keywords, &idx_keywords)) {

struct mail_keywords *kw;

kw = mail_index_keywords_create_from_indexes(

trans, &keywords);

mail_index_update_keywords(trans, seq,

MODIFY_REPLACE, kw);

mail_index_keywords_free(&kw);

}

}

maildir_uidlist_iter_deinit(iter);

array_free(&keywords);

if (!partial) {

/* expunge the rest */

for (seq++; seq <= hdr->messages_count; seq++)

mail_index_expunge(trans, seq);

/* next_uid must be updated only in non-partial syncs since

next_uid = maildir_uidlist_get_next_uid(mbox->uidlist);

if (next_uid != 0 && hdr->next_uid != next_uid) {

mail_index_update_header(trans,

offsetof(struct mail_index_header, next_uid),

&next_uid, sizeof(next_uid), FALSE);

}

}

if (!mbox->syncing_commit) {

/* now, sync the index. NOTE: may recurse back to here with

mbox->syncing_commit = TRUE;

if (maildir_sync_index_records(sync_ctx) < 0)

ret = -1;

mbox->syncing_commit = FALSE;

}

if (mbox->dirty_cur_time == 0 &&

mbox->last_cur_mtime != (time_t)hdr->sync_stamp) {

uint32_t sync_stamp = mbox->last_cur_mtime;

mail_index_update_header(trans,

offsetof(struct mail_index_header, sync_stamp),

&sync_stamp, sizeof(sync_stamp), TRUE);

}

/* FIXME: use a header extension instead of sync_size.. */

value = mbox->last_new_mtime;

old_new_sync_time = hdr->sync_size >> 32;

if (mbox->last_new_mtime >= old_new_sync_time - MAILDIR_SYNC_SECS) {

value |= (uint64_t)mbox->last_new_sync_time << 32;

} else {

value |= (uint64_t)old_new_sync_time << 32;

}

if (value != hdr->sync_size) {

uint64_t sync_stamp = mbox->last_new_mtime |

((uint64_t)mbox->last_new_sync_time << 32);

mail_index_update_header(trans,

offsetof(struct mail_index_header, sync_size),

&sync_stamp, sizeof(sync_stamp), TRUE);

}

if (hdr->uid_validity == 0) {

/* get the initial uidvalidity */

if (maildir_uidlist_update(mbox->uidlist) < 0)

ret = -1;

uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);

if (uid_validity == 0) {

uid_validity = ioloop_time;

maildir_uidlist_set_uid_validity(mbox->uidlist,

uid_validity);

}

} else if (uid_validity == 0) {

maildir_uidlist_set_uid_validity(mbox->uidlist,

hdr->uid_validity);

}

if (uid_validity != hdr->uid_validity && uid_validity != 0) {

mail_index_update_header(trans,

offsetof(struct mail_index_header, uid_validity),

&uid_validity, sizeof(uid_validity), TRUE);

}

if (ret < 0) {

mail_index_transaction_rollback(&trans);

mail_index_sync_rollback(&sync_ctx->sync_ctx);

} else {

uint32_t seq;

uoff_t offset;

if (mail_index_transaction_commit(&trans, &seq, &offset) < 0)

ret = -1;

else if (seq != 0) {

mbox->ibox.commit_log_file_seq = seq;

mbox->ibox.commit_log_file_offset = offset;

}

if (mail_index_sync_commit(&sync_ctx->sync_ctx) < 0)

ret = -1;

}

maildir_keywords_sync_deinit(sync_ctx->keywords_sync_ctx);

sync_ctx->keywords_sync_ctx = NULL;

if (ret == 0) {

mbox->ibox.commit_log_file_seq = 0;

mbox->ibox.commit_log_file_offset = 0;

} else {

mail_storage_set_index_error(&mbox->ibox);

}

i_free(sync_ctx);

return ret < 0 ? -1 : (full_rescan ? 0 : 1);

}

static int maildir_sync_context(struct maildir_sync_context *ctx, bool forced,

bool sync_last_commit)

{

bool new_changed, cur_changed, full_rescan = FALSE;

int ret;

if (sync_last_commit) {

new_changed = cur_changed = FALSE;

} else if (!forced) {

if (maildir_sync_quick_check(ctx->mbox,

ctx->new_dir, ctx->cur_dir,

&new_changed, &cur_changed) < 0)

return -1;

if (!new_changed && !cur_changed)

return 1;

} else {

new_changed = cur_changed = TRUE;

}

/*

if (!ctx->mbox->syncing_commit) {

ctx->index_sync_ctx = maildir_sync_index_begin(ctx->mbox);

if (ctx->index_sync_ctx == NULL)

return -1;

}

ctx->partial = !cur_changed;

ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, ctx->partial,

&ctx->uidlist_sync_ctx);

if (ret <= 0) {

/* failure / timeout. if forced is TRUE, we could still go

return ret;

}

if (new_changed || cur_changed) {

/* if we're going to check cur/ dir our current logic requires

while ((ret = maildir_scan_dir(ctx, TRUE)) > 0) {

/* rename()d at least some files, which might have

}

if (ret < 0)

return -1;

if (cur_changed) {

if (maildir_scan_dir(ctx, FALSE) < 0)

return -1;

}

/* finish uidlist syncing, but keep it still locked */

maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);

}

if (!ctx->mbox->syncing_commit) {

/* NOTE: index syncing here might cause a re-sync due to

ret = maildir_sync_index_finish(ctx->index_sync_ctx,

ctx->partial);

if (ret < 0) {

ctx->index_sync_ctx = NULL;

return -1;

}

if (ret == 0)

full_rescan = TRUE;

i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist));

ctx->index_sync_ctx = NULL;

}

ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);

ctx->uidlist_sync_ctx = NULL;

return ret < 0 ? -1 : (full_rescan ? 0 : 1);

}

int maildir_storage_sync_force(struct maildir_mailbox *mbox)

{

struct maildir_sync_context *ctx;

int ret;

ctx = maildir_sync_context_new(mbox);

ret = maildir_sync_context(ctx, TRUE, FALSE);

maildir_sync_deinit(ctx);

return ret < 0 ? -1 : 0;

}

int maildir_sync_last_commit(struct maildir_mailbox *mbox)

{

struct maildir_sync_context *ctx;

int ret;

if (mbox->ibox.commit_log_file_seq == 0)

return 0;

ctx = maildir_sync_context_new(mbox);

ret = maildir_sync_context(ctx, FALSE, TRUE);

maildir_sync_deinit(ctx);

return ret < 0 ? -1 : 0;

}

struct mailbox_sync_context *

maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)

{

struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;

struct maildir_sync_context *ctx;

int ret = 0;

if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||

mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=

ioloop_time) {

mbox->ibox.sync_last_check = ioloop_time;

ctx = maildir_sync_context_new(mbox);

ret = maildir_sync_context(ctx, FALSE, FALSE);

maildir_sync_deinit(ctx);

i_assert(!maildir_uidlist_is_locked(mbox->uidlist) ||

mbox->ibox.keep_locked);