index/maildir/maildir-sync-index.c

	maildir-sync-index.c revision 4a6c27b85b71b535b827ef512ec88416a982dc38
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "lib.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "ioloop.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "array.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "hex-binary.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "maildir-storage.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "index-sync-changes.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "maildir-uidlist.h"
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen#include "maildir-keywords.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "maildir-filename.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include "maildir-sync.h"
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include <stdio.h>
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include <stdlib.h>
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#include <unistd.h>
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstruct maildir_index_sync_context {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        struct maildir_mailbox *mbox;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct maildir_sync_context *maildir_sync_ctx;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct mail_index_view *view;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct mail_index_sync_ctx *sync_ctx;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        struct maildir_keywords_sync_ctx *keywords_sync_ctx;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen    struct mail_index_transaction *trans;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct index_sync_changes_context *sync_changes;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    enum mail_flags flags;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    ARRAY_TYPE(keyword_indexes) keywords, idx_keywords;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    uint32_t /*seq, */uid;
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen    bool update_maildir_hdr_cur;
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen};
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstruct maildir_keywords_sync_ctx *
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenmaildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen{
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen    return ctx->keywords_sync_ctx;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen}
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstatic void
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenmaildir_index_expunge(struct maildir_index_sync_context *ctx, uint32_t seq)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen{
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    enum maildir_uidlist_rec_flag flags;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    uint8_t guid_128[MAIL_GUID_128_SIZE];
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    const char *fname;
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen    uint32_t uid;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    mail_index_lookup_uid(ctx->view, seq, &uid);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    if (maildir_uidlist_lookup(ctx->mbox->uidlist, uid,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                   &flags, &fname) <= 0)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        memset(guid_128, 0, sizeof(guid_128));
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    else T_BEGIN {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        mail_generate_guid_128_hash(t_strcut(fname, ':'),
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                        guid_128);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    } T_END;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    mail_index_expunge_guid(ctx->trans, seq, guid_128);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen}
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstatic bool
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenmaildir_expunge_is_valid_guid(struct maildir_index_sync_context *ctx,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                  const char *filename,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                  uint8_t expunged_guid_128[MAIL_GUID_128_SIZE])
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen{
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    uint8_t guid_128[MAIL_GUID_128_SIZE];
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    if (mail_guid_128_is_empty(expunged_guid_128)) {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        /* no GUID associated with expunge */
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        return TRUE;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    T_BEGIN {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        mail_generate_guid_128_hash(t_strcut(filename, ':'),
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                        guid_128);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    } T_END;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    if (memcmp(guid_128, expunged_guid_128, sizeof(guid_128)) == 0)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        return TRUE;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    mail_storage_set_critical(&ctx->mbox->storage->storage,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        "Mailbox %s: Expunged GUID mismatch for UID %u: %s vs %s",
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen        ctx->mbox->box.vname, ctx->uid,
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen        binary_to_hex(guid_128, sizeof(guid_128)),
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        binary_to_hex(expunged_guid_128, MAIL_GUID_128_SIZE));
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    return FALSE;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen}
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainenstatic int maildir_expunge(struct maildir_mailbox *mbox, const char *path,
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen               struct maildir_index_sync_context *ctx)
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen{
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen    struct mailbox *box = &mbox->box;
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen    if (unlink(path) == 0) {
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen        if (box->v.sync_notify != NULL) {
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen            box->v.sync_notify(box, ctx->uid,
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen                       MAILBOX_SYNC_TYPE_EXPUNGE);
7d359c1719bf6ff228a96c66d27f5cfa239cb31dTimo Sirainen        }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        return 1;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    if (errno == ENOENT)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        return 0;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    mail_storage_set_critical(&mbox->storage->storage,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                  "unlink(%s) failed: %m", path);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    return -1;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen}
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstatic int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                  struct maildir_index_sync_context *ctx)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen{
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct mailbox *box = &mbox->box;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct stat st;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    const char *dir, *fname, *newfname, *newpath;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    enum mail_index_sync_type sync_type;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    uint8_t flags8;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    fname = strrchr(path, '/');
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    i_assert(fname != NULL);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    fname++;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    dir = t_strdup_until(path, fname);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    /* get the current flags and keywords */
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    maildir_filename_get_flags(ctx->keywords_sync_ctx,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                   fname, &ctx->flags, &ctx->keywords);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    /* apply changes */
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    flags8 = ctx->flags;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    index_sync_changes_apply(ctx->sync_changes, NULL,
d03a871a77f8ec36f48f5fea98d810e51b186fdbTimo Sirainen                 &flags8, &ctx->keywords, &sync_type);
2cc88ff507e244faa63683f804833b321a62c665Timo Sirainen    ctx->flags = flags8;
d03a871a77f8ec36f48f5fea98d810e51b186fdbTimo Sirainen
10b8040903b1d1591f1d44552ff466c8789b8814Timo Sirainen    /* and try renaming with the new name */
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    newfname = maildir_filename_set_flags(ctx->keywords_sync_ctx, fname,
d03a871a77f8ec36f48f5fea98d810e51b186fdbTimo Sirainen                          ctx->flags, &ctx->keywords);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    newpath = t_strconcat(dir, newfname, NULL);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    if (strcmp(path, newpath) == 0) {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        /* just make sure that the file still exists. avoid rename()
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen           here because it's slow on HFS. */
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        if (stat(path, &st) < 0) {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen            if (errno == ENOENT)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                return 0;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen            mail_storage_set_critical(box->storage,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                "stat(%s) failed: %m", path);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen            return -1;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    } else {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        if (rename(path, newpath) < 0) {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen            if (errno == ENOENT)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                return 0;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen            if (!ENOSPACE(errno) && errno != EACCES) {
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen                mail_storage_set_critical(box->storage,
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen                    "rename(%s, %s) failed: %m",
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen                    path, newpath);
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen            }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen            return -1;
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        }
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    }
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    if (box->v.sync_notify != NULL) {
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        box->v.sync_notify(box, ctx->uid,
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen                   index_sync_type_convert(sync_type));
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    }
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    return 1;
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen}
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainenstatic int maildir_handle_uid_insertion(struct maildir_index_sync_context *ctx,
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen                    enum maildir_uidlist_rec_flag uflags,
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen                    const char *filename, uint32_t uid)
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen{
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    int ret;
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        /* partial syncing */
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        return 0;
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    }
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    /* most likely a race condition: we read the maildir, then someone else
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen       expunged messages and committed changes to index. so, this message
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen       shouldn't actually exist. */
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) == 0) {
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        /* mark it racy and check in next sync */
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        ctx->mbox->maildir_hdr.cur_check_time = 0;
3a8c3cd1847aea924c6dbf4dee17074ae2039b9dTimo Sirainen        maildir_uidlist_add_flags(ctx->mbox->uidlist, filename,
3a8c3cd1847aea924c6dbf4dee17074ae2039b9dTimo Sirainen                      MAILDIR_UIDLIST_REC_FLAG_RACING);
3a8c3cd1847aea924c6dbf4dee17074ae2039b9dTimo Sirainen        return 0;
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    }
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    if (ctx->uidlist_sync_ctx == NULL) {
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        ret = maildir_uidlist_sync_init(ctx->mbox->uidlist,
06972d9902a012a12eeb5155116410a9a421e1d2Timo Sirainen                        MAILDIR_UIDLIST_SYNC_PARTIAL |
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen                        MAILDIR_UIDLIST_SYNC_KEEP_STATE,
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen                        &ctx->uidlist_sync_ctx);
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        if (ret <= 0)
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen            return -1;
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    }
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    uflags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    maildir_uidlist_sync_remove(ctx->uidlist_sync_ctx, filename);
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen                    filename, uflags);
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    i_assert(ret > 0);
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    /* give the new UID to it immediately */
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    i_warning("Maildir %s: Expunged message reappeared, giving a new UID "
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen          "(old uid=%u, file=%s)", ctx->mbox->box.path,
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen          uid, filename);
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    return 0;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen}
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenint maildir_sync_index_begin(struct maildir_mailbox *mbox,
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen                 struct maildir_sync_context *maildir_sync_ctx,
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen                 struct maildir_index_sync_context **ctx_r)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen{
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct mailbox *_box = &mbox->box;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct maildir_index_sync_context *ctx;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct mail_index_sync_ctx *sync_ctx;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct mail_index_view *view;
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    struct mail_index_transaction *trans;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    enum mail_index_sync_flags sync_flags;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    sync_flags = index_storage_get_sync_flags(&mbox->box);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    /* don't drop recent messages if we're saving messages */
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen    if (maildir_sync_ctx == NULL)
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        sync_flags &= ~MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen    if (mail_index_sync_begin(_box->index, &sync_ctx, &view,
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen                  &trans, sync_flags) < 0) {
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        mail_storage_set_index_error(_box);
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        return -1;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
db4cad7df8f3a1a584b1f72a06995b188871dc5aTimo Sirainen    ctx = i_new(struct maildir_index_sync_context, 1);
db4cad7df8f3a1a584b1f72a06995b188871dc5aTimo Sirainen    ctx->mbox = mbox;
db4cad7df8f3a1a584b1f72a06995b188871dc5aTimo Sirainen    ctx->maildir_sync_ctx = maildir_sync_ctx;
db4cad7df8f3a1a584b1f72a06995b188871dc5aTimo Sirainen    ctx->sync_ctx = sync_ctx;
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen    ctx->view = view;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    ctx->trans = trans;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    ctx->keywords_sync_ctx =
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen        maildir_keywords_sync_init(mbox->keywords, _box->index);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    ctx->sync_changes =
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen        index_sync_changes_init(ctx->sync_ctx, ctx->view, ctx->trans,
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen                    mbox->box.backend_readonly);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen    *ctx_r = ctx;
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen    return 0;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen}
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstatic bool
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenmaildir_index_header_has_changed(const struct maildir_index_header *old_hdr,
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen                 const struct maildir_index_header *new_hdr)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen{
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen#define DIR_DELAYED_REFRESH(hdr, name) \
2cc88ff507e244faa63683f804833b321a62c665Timo Sirainen    ((hdr)->name ## _check_time <= \
67cc1d2d8495c3effad6f639cd8468d0ac7ff198Timo Sirainen        (hdr)->name ## _mtime + MAILDIR_SYNC_SECS)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    if (old_hdr->new_mtime != new_hdr->new_mtime ||
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs ||
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        old_hdr->cur_mtime != new_hdr->cur_mtime ||
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs ||
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        old_hdr->uidlist_mtime != new_hdr->uidlist_mtime ||
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen        old_hdr->uidlist_mtime_nsecs != new_hdr->uidlist_mtime_nsecs ||
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen        old_hdr->uidlist_size != new_hdr->uidlist_size)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        return TRUE;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen    return DIR_DELAYED_REFRESH(old_hdr, new) !=
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        DIR_DELAYED_REFRESH(new_hdr, new) ||
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        DIR_DELAYED_REFRESH(old_hdr, cur) !=
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        DIR_DELAYED_REFRESH(new_hdr, cur);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen}
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstatic void
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenmaildir_sync_index_update_ext_header(struct maildir_index_sync_context *ctx)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen{
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct maildir_mailbox *mbox = ctx->mbox;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    const void *data;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    size_t data_size;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct stat st;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen    if (ctx->update_maildir_hdr_cur &&
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        stat(t_strconcat(mbox->box.path, "/cur", NULL), &st) == 0) {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        if ((time_t)mbox->maildir_hdr.cur_check_time < st.st_mtime)
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen            mbox->maildir_hdr.cur_check_time = st.st_mtime;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        mbox->maildir_hdr.cur_mtime = st.st_mtime;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                  &data, &data_size);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    if (data_size != sizeof(mbox->maildir_hdr) ||
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        maildir_index_header_has_changed(data, &mbox->maildir_hdr)) {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        mail_index_update_header_ext(ctx->trans, mbox->maildir_ext_id,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                         0, &mbox->maildir_hdr,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                         sizeof(mbox->maildir_hdr));
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen}
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstatic int maildir_sync_index_finish(struct maildir_index_sync_context *ctx,
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen                     bool success)
584c79069470319ce9e0d4ed34d9604726cfefb1Timo Sirainen{
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct maildir_mailbox *mbox = ctx->mbox;
584c79069470319ce9e0d4ed34d9604726cfefb1Timo Sirainen    int ret = success ? 0 : -1;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    if (ret < 0)
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        mail_index_sync_rollback(&ctx->sync_ctx);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    else {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        maildir_sync_index_update_ext_header(ctx);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        /* Set syncing_commit=TRUE so that if any sync callbacks try
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen           to access mails which got lost (eg. expunge callback trying
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen           to open the file which was just unlinked) we don't try to
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen           start a second index sync and crash. */
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        mbox->syncing_commit = TRUE;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen            mail_storage_set_index_error(&mbox->box);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen            ret = -1;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        }
        mbox->syncing_commit = FALSE;
    }

    maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx);
    index_sync_changes_deinit(&ctx->sync_changes);
    i_free(ctx);
    return ret;
}

int maildir_sync_index_commit(struct maildir_index_sync_context **_ctx)
{
    struct maildir_index_sync_context *ctx = *_ctx;

    *_ctx = NULL;
    return maildir_sync_index_finish(ctx, TRUE);
}

void maildir_sync_index_rollback(struct maildir_index_sync_context **_ctx)
{
    struct maildir_index_sync_context *ctx = *_ctx;

    *_ctx = NULL;
    (void)maildir_sync_index_finish(ctx, FALSE);
}

static int uint_cmp(const void *p1, const void *p2)
{
    const unsigned int *i1 = p1, *i2 = p2;

    if (*i1 < *i2)
        return -1;
    else if (*i1 > *i2)
        return 1;
    else
        return 0;
}

static void
maildir_sync_mail_keywords(struct maildir_index_sync_context *ctx, uint32_t seq)
{
    struct mailbox *box = &ctx->mbox->box;
    struct mail_keywords *kw;
    unsigned int i, j, old_count, new_count;
    const unsigned int *old_indexes, *new_indexes;
    bool have_indexonly_keywords;
    int diff;

    mail_index_lookup_keywords(ctx->view, seq, &ctx->idx_keywords);
    if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords)) {
        /* no changes - we should get here usually */
        return;
    }

    /* sort the keywords */
    array_sort(&ctx->idx_keywords, uint_cmp);
    array_sort(&ctx->keywords, uint_cmp);

    /* drop keywords that are in index-only. we don't want to touch them. */
    old_indexes = array_get(&ctx->idx_keywords, &old_count);
    have_indexonly_keywords = FALSE;
    for (i = old_count; i > 0; i--) {
        if (old_indexes[i-1] < MAILDIR_MAX_KEYWORDS)
            break;
        have_indexonly_keywords = TRUE;
        array_delete(&ctx->idx_keywords, i-1, 1);
    }

    if (!have_indexonly_keywords) {
        /* no index-only keywords found, so something changed.
           just replace them all. */
        kw = mail_index_keywords_create_from_indexes(box->index,
                                 &ctx->keywords);
        mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, kw);
        mail_index_keywords_unref(&kw);
        return;
    }

    /* check again if non-index-only keywords changed */
    if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords))
        return;

    /* we can't reset all the keywords or we'd drop indexonly keywords too.
       so first remove the unwanted keywords and then add back the wanted
       ones. we can get these lists easily by removing common elements
       from old and new keywords. */
    new_indexes = array_get(&ctx->keywords, &new_count);
    for (i = j = 0; i < old_count && j < new_count; ) {
        diff = (int)old_indexes[i] - (int)new_indexes[j];
        if (diff == 0) {
            array_delete(&ctx->keywords, j, 1);
            array_delete(&ctx->idx_keywords, i, 1);
            old_indexes = array_get(&ctx->idx_keywords, &old_count);
            new_indexes = array_get(&ctx->keywords, &new_count);
        } else if (diff < 0) {
            i++;
        } else {
            j++;
        }
    }

    if (array_count(&ctx->idx_keywords) > 0) {
        kw = mail_index_keywords_create_from_indexes(box->index,
                                 &ctx->idx_keywords);
        mail_index_update_keywords(ctx->trans, seq, MODIFY_REMOVE, kw);
        mail_index_keywords_unref(&kw);
    }

    if (array_count(&ctx->keywords) > 0) {
        kw = mail_index_keywords_create_from_indexes(box->index,
                                 &ctx->keywords);
        mail_index_update_keywords(ctx->trans, seq, MODIFY_ADD, kw);
        mail_index_keywords_unref(&kw);
    }
}

int maildir_sync_index(struct maildir_index_sync_context *ctx,
               bool partial)
{
    struct maildir_mailbox *mbox = ctx->mbox;
    struct mail_index_view *view = ctx->view;
    struct mail_index_view *view2;
    struct maildir_uidlist_iter_ctx *iter;
    struct mail_index_transaction *trans = ctx->trans;
    const struct mail_index_header *hdr;
    struct mail_index_header empty_hdr;
    const struct mail_index_record *rec;
    uint32_t seq, seq2, uid, prev_uid;
        enum maildir_uidlist_rec_flag uflags;
    const char *filename;
    uint32_t uid_validity, next_uid, hdr_next_uid, first_recent_uid;
    uint32_t first_uid;
    unsigned int changes = 0;
    int ret = 0;
    time_t time_before_sync;
    uint8_t expunged_guid_128[MAIL_GUID_128_SIZE];
    bool expunged, full_rescan = FALSE;

    i_assert(!mbox->syncing_commit);

    first_uid = 1;
    hdr = mail_index_get_header(view);
    uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
    if (uid_validity != hdr->uid_validity &&
        uid_validity != 0 && hdr->uid_validity != 0) {
        /* uidvalidity changed and index isn't being synced for the
           first time, reset the index so we can add all messages as
           new */
        i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)",
              mbox->box.path, hdr->uid_validity, uid_validity);
        mail_index_reset(trans);
        index_mailbox_reset_uidvalidity(&mbox->box);

        first_uid = hdr->messages_count + 1;
        memset(&empty_hdr, 0, sizeof(empty_hdr));
        empty_hdr.next_uid = 1;
        hdr = &empty_hdr;
    }
    hdr_next_uid = hdr->next_uid;

    time_before_sync = time(NULL);
    mbox->syncing_commit = TRUE;
    seq = prev_uid = 0; first_recent_uid = I_MAX(hdr->first_recent_uid, 1);
    i_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS);
    i_array_init(&ctx->idx_keywords, MAILDIR_MAX_KEYWORDS);
    iter = maildir_uidlist_iter_init(mbox->uidlist);
    while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
        maildir_filename_get_flags(ctx->keywords_sync_ctx, filename,
                       &ctx->flags, &ctx->keywords);

        i_assert(uid > prev_uid);
        prev_uid = uid;

        /* the private flags are kept only in indexes. don't use them
           at all even for newly seen mails */
        ctx->flags &= ~mbox->box.private_flags_mask;

    again:
        seq++;
        ctx->uid = uid;

        if (seq > hdr->messages_count) {
            if (uid < hdr_next_uid) {
                if (maildir_handle_uid_insertion(ctx, uflags,
                                 filename,
                                 uid) < 0)
                    ret = -1;
                seq--;
                continue;
            }

            /* Trust uidlist recent flags only for newly added
               messages. When saving/copying messages with flags
               they're stored to cur/ and uidlist treats them
               as non-recent. */
            if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) == 0) {
                if (uid >= first_recent_uid)
                    first_recent_uid = uid + 1;
            }

            hdr_next_uid = uid + 1;
            mail_index_append(trans, uid, &seq);
            mail_index_update_flags(trans, seq, MODIFY_REPLACE,
                        ctx->flags);
            if (array_count(&ctx->keywords) > 0) {
                struct mail_keywords *kw;

                kw = mail_index_keywords_create_from_indexes(
                    mbox->box.index, &ctx->keywords);
                mail_index_update_keywords(trans, seq,
                               MODIFY_REPLACE, kw);
                mail_index_keywords_unref(&kw);
            }
            continue;
        }

        rec = mail_index_lookup(view, seq);
        if (uid > rec->uid) {
            /* expunged */
            maildir_index_expunge(ctx, seq);
            goto again;
        }

        if (uid < rec->uid) {
            if (maildir_handle_uid_insertion(ctx, uflags,
                             filename, uid) < 0)
                ret = -1;
            seq--;
            continue;
        }

        index_sync_changes_read(ctx->sync_changes, rec->uid, &expunged,
                    expunged_guid_128);
        if (expunged) {
            if (!maildir_expunge_is_valid_guid(ctx, filename,
                               expunged_guid_128))
                continue;
            if (maildir_file_do(mbox, ctx->uid,
                        maildir_expunge, ctx) >= 0) {
                /* successful expunge */
                maildir_index_expunge(ctx, seq);
            }
            if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
                maildir_sync_notify(ctx->maildir_sync_ctx);
            continue;
        }

        /* the private flags are stored only in indexes, keep them */
        ctx->flags |= rec->flags & mbox->box.private_flags_mask;

        if (index_sync_changes_have(ctx->sync_changes)) {
            /* apply flag changes to maildir */
            if (maildir_file_do(mbox, ctx->uid,
                        maildir_sync_flags, ctx) < 0)
                ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
            if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
                maildir_sync_notify(ctx->maildir_sync_ctx);
        }

        if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
            /* partial syncing */
            if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) {
                /* we last saw this mail in new/, but it's
                   not there anymore. possibly expunged,
                   make sure. */
                full_rescan = TRUE;
            }
            continue;
        }

        if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
            /* we haven't been able to update maildir with this
               record's flag changes. don't sync them. */
            continue;
        }

        if (ctx->flags != (rec->flags & MAIL_FLAGS_NONRECENT)) {
            mail_index_update_flags(trans, seq, MODIFY_REPLACE,
                        ctx->flags);
        }

        maildir_sync_mail_keywords(ctx, seq);
    }
    maildir_uidlist_iter_deinit(&iter);
    mbox->syncing_commit = FALSE;

    if (!partial) {
        /* expunge the rest */
        for (seq++; seq <= hdr->messages_count; seq++)
            maildir_index_expunge(ctx, seq);
    }

    /* add \Recent flags. use updated view so it contains newly
       appended messages. */
    view2 = mail_index_transaction_open_updated_view(trans);
    if (mail_index_lookup_seq_range(view2, first_recent_uid, (uint32_t)-1,
                    &seq, &seq2) && seq2 >= first_uid) {
        if (seq < first_uid) {
            /* UIDVALIDITY changed, skip over the old messages */
            seq = first_uid;
        }
        index_mailbox_set_recent_seq(&mbox->box, view2, seq, seq2);
    }
    mail_index_view_close(&view2);

    if (ctx->uidlist_sync_ctx != NULL) {
        if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx,
                        TRUE) < 0)
            ret = -1;
    }

    if (mbox->box.v.sync_notify != NULL)
        mbox->box.v.sync_notify(&mbox->box, 0, 0);

    /* check cur/ mtime later. if we came here from saving messages they
       could still be moved to cur/ directory. */
    ctx->update_maildir_hdr_cur = TRUE;
    mbox->maildir_hdr.cur_check_time = time_before_sync;

    if (uid_validity == 0) {
        uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity :
            maildir_get_uidvalidity_next(mbox->box.list);
        maildir_uidlist_set_uid_validity(mbox->uidlist, uid_validity);
    }
    maildir_uidlist_set_next_uid(mbox->uidlist, hdr_next_uid, FALSE);

    if (uid_validity != hdr->uid_validity) {
        mail_index_update_header(trans,
            offsetof(struct mail_index_header, uid_validity),
            &uid_validity, sizeof(uid_validity), TRUE);
    }

    next_uid = maildir_uidlist_get_next_uid(mbox->uidlist);
    if (hdr_next_uid < next_uid) {
        mail_index_update_header(trans,
            offsetof(struct mail_index_header, next_uid),
            &next_uid, sizeof(next_uid), FALSE);
    }

    i_assert(hdr->first_recent_uid <= first_recent_uid);
    if (hdr->first_recent_uid < first_recent_uid) {
        mail_index_update_header(ctx->trans,
            offsetof(struct mail_index_header, first_recent_uid),
            &first_recent_uid, sizeof(first_recent_uid), FALSE);
    }
    array_free(&ctx->keywords);
    array_free(&ctx->idx_keywords);
    return ret < 0 ? -1 : (full_rescan ? 0 : 1);
}

static unsigned int maildir_list_get_ext_id(struct maildir_storage *storage,
                        struct mail_index_view *view)
{
    if (storage->maildir_list_ext_id == (uint32_t)-1) {
        storage->maildir_list_ext_id =
            mail_index_ext_register(mail_index_view_get_index(view),
                "maildir", 0,
                sizeof(struct maildir_list_index_record),
                sizeof(uint32_t));
    }
    return storage->maildir_list_ext_id;
}

int maildir_list_index_has_changed(struct mailbox *box,
                   struct mail_index_view *list_view,
                   uint32_t seq)
{
    struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
    const struct maildir_list_index_record *rec;
    const void *data;
    const char *root_dir, *new_dir, *cur_dir;
    struct stat st;
    uint32_t ext_id;
    bool expunged;

    ext_id = maildir_list_get_ext_id(mbox->storage, list_view);
    mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
    rec = data;

    if (rec == NULL || expunged ||
        rec->new_mtime == 0 || rec->cur_mtime == 0) {
        /* doesn't exist, not synced or dirty-synced */
        return 1;
    }

    root_dir = mailbox_list_get_path(box->list, box->name,
                     MAILBOX_LIST_PATH_TYPE_MAILBOX);

    /* check if new/ changed */
    new_dir = t_strconcat(root_dir, "/new", NULL);
    if (stat(new_dir, &st) < 0) {
        mail_storage_set_critical(box->storage,
                      "stat(%s) failed: %m", new_dir);
        return -1;
    }
    if ((time_t)rec->new_mtime != st.st_mtime)
        return 1;

    /* check if cur/ changed */
    cur_dir = t_strconcat(root_dir, "/cur", NULL);
    if (stat(cur_dir, &st) < 0) {
        mail_storage_set_critical(box->storage,
                      "stat(%s) failed: %m", cur_dir);
        return -1;
    }
    if ((time_t)rec->cur_mtime != st.st_mtime)
        return 1;
    return 0;
}

int maildir_list_index_update_sync(struct mailbox *box,
                   struct mail_index_transaction *trans,
                   uint32_t seq)
{
    struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
    struct mail_index_view *list_view;
    const struct maildir_index_header *mhdr = &mbox->maildir_hdr;
    const struct maildir_list_index_record *old_rec;
    struct maildir_list_index_record new_rec;
    const void *data;
    uint32_t ext_id;
    bool expunged;

    /* get the current record */
    list_view = mail_index_transaction_get_view(trans);
    ext_id = maildir_list_get_ext_id(mbox->storage, list_view);
    mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
    if (expunged)
        return 0;
    old_rec = data;

    memset(&new_rec, 0, sizeof(new_rec));
    if (mhdr->new_check_time <= mhdr->new_mtime + MAILDIR_SYNC_SECS ||
        mhdr->cur_check_time <= mhdr->cur_mtime + MAILDIR_SYNC_SECS) {
        /* dirty, we need a refresh next time */
    } else {
        new_rec.new_mtime = mhdr->new_mtime;
        new_rec.cur_mtime = mhdr->cur_mtime;
    }

    if (old_rec == NULL ||
        memcmp(old_rec, &new_rec, sizeof(old_rec)) != 0) {
        mail_index_update_ext(trans, seq,
                      mbox->storage->maildir_list_ext_id,
                      &new_rec, NULL);
    }
    return 0;
}