index/maildir/maildir-sync-index.c

	maildir-sync-index.c revision f6f021c133f680cf3d559187524fd9abcbaae9b9
2e37d45867d081db150ab78dad303b9077aea24fTimo Sirainen/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "lib.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "ioloop.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "array.h"
472369cba85d9f7c995dda60e7cd01d78b4a960aTimo Sirainen#include "hex-binary.h"
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen#include "maildir-storage.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include "index-sync-changes.h"
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen#include "maildir-uidlist.h"
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen#include "maildir-keywords.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include "maildir-filename.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include "maildir-sync.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include <stdio.h>
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include <stdlib.h>
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen#include <unistd.h>
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen#define MAILDIR_SYNC_TIME_WARN_SECS 60
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainenstruct maildir_index_sync_context {
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen        struct maildir_mailbox *mbox;
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen    struct maildir_sync_context *maildir_sync_ctx;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_index_view *view;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_index_sync_ctx *sync_ctx;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        struct maildir_keywords_sync_ctx *keywords_sync_ctx;
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen    struct mail_index_transaction *trans;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen    struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct index_sync_changes_context *sync_changes;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    enum mail_flags flags;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    ARRAY_TYPE(keyword_indexes) keywords, idx_keywords;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    uint32_t uid;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    bool update_maildir_hdr_cur;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    time_t start_time;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    unsigned int flag_change_count, expunge_count, new_msgs_count;
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen};
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainenstruct maildir_keywords_sync_ctx *
de486b59018016977015ef42e6071155b60e82e1Timo Sirainenmaildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx)
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen{
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen    return ctx->keywords_sync_ctx;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen}
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainenvoid maildir_sync_set_new_msgs_count(struct maildir_index_sync_context *ctx,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen                     unsigned int count)
220e21750948941dc6e33b8f11b552fa21d7f81eTimo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    ctx->new_msgs_count = count;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainenstatic bool
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainenmaildir_expunge_is_valid_guid(struct maildir_index_sync_context *ctx,
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen                  uint32_t uid, const char *filename,
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen                  uint8_t expunged_guid_128[MAIL_GUID_128_SIZE])
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen{
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen    uint8_t guid_128[MAIL_GUID_128_SIZE];
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen    const char *guid;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen    if (mail_guid_128_is_empty(expunged_guid_128)) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        /* no GUID associated with expunge */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        return TRUE;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    T_BEGIN {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        guid = maildir_uidlist_lookup_ext(ctx->mbox->uidlist, uid,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                          MAILDIR_UIDLIST_REC_EXT_GUID);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        if (guid == NULL)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            guid = t_strcut(filename, ':');
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        mail_generate_guid_128_hash(guid, guid_128);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    } T_END;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (memcmp(guid_128, expunged_guid_128, sizeof(guid_128)) == 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        return TRUE;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    mail_storage_set_critical(&ctx->mbox->storage->storage,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        "Mailbox %s: Expunged GUID mismatch for UID %u: %s vs %s",
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen        ctx->mbox->box.vname, ctx->uid,
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen        binary_to_hex(guid_128, sizeof(guid_128)),
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen        binary_to_hex(expunged_guid_128, MAIL_GUID_128_SIZE));
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen    return FALSE;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic int maildir_expunge(struct maildir_mailbox *mbox, const char *path,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen               struct maildir_index_sync_context *ctx)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mailbox *box = &mbox->box;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    ctx->expunge_count++;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (unlink(path) == 0) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        if (box->v.sync_notify != NULL) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            box->v.sync_notify(box, ctx->uid,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                       MAILBOX_SYNC_TYPE_EXPUNGE);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        return 1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (errno == ENOENT)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen        return 0;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen    if (errno == EISDIR)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen        return maildir_lose_unexpected_dir(box->storage, path);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen    mail_storage_set_critical(&mbox->storage->storage,
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen                  "unlink(%s) failed: %m", path);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen    return -1;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen}
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainenstatic int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path,
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen                  struct maildir_index_sync_context *ctx)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen{
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen    struct mailbox *box = &mbox->box;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    struct stat st;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    const char *dir, *fname, *newfname, *newpath;
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen    enum mail_index_sync_type sync_type;
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen    uint8_t flags8;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    ctx->flag_change_count++;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    fname = strrchr(path, '/');
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    i_assert(fname != NULL);
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen    fname++;
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen    dir = t_strdup_until(path, fname);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    i_assert(*fname != '\0');
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    /* get the current flags and keywords */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    maildir_filename_get_flags(ctx->keywords_sync_ctx,
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen                   fname, &ctx->flags, &ctx->keywords);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    /* apply changes */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    flags8 = ctx->flags;
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen    index_sync_changes_apply(ctx->sync_changes, NULL,
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen                 &flags8, &ctx->keywords, &sync_type);
6600c05e2ab38e9f662582b63c56b0c980a03748Timo Sirainen    ctx->flags = flags8;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen    /* and try renaming with the new name */
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen    newfname = maildir_filename_set_flags(ctx->keywords_sync_ctx, fname,
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen                          ctx->flags, &ctx->keywords);
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen    newpath = t_strconcat(dir, newfname, NULL);
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen    if (strcmp(path, newpath) == 0) {
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen        /* just make sure that the file still exists. avoid rename()
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen           here because it's slow on HFS. */
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen        if (stat(path, &st) < 0) {
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen            if (errno == ENOENT)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen                return 0;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen            mail_storage_set_critical(box->storage,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                "stat(%s) failed: %m", path);
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen            return -1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    } else {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        if (rename(path, newpath) < 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            if (errno == ENOENT)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                return 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            if (!ENOSPACE(errno) && errno != EACCES) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                mail_storage_set_critical(box->storage,
3fe67ec75ccae1230bb9eb9f16affc48377f6441Timo Sirainen                    "rename(%s, %s) failed: %m",
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                    path, newpath);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            }
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen            return -1;
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen        }
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen    }
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen    if (box->v.sync_notify != NULL) {
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen        box->v.sync_notify(box, ctx->uid,
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen                   index_sync_type_convert(sync_type));
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen    }
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen    return 1;
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen}
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainenstatic int maildir_handle_uid_insertion(struct maildir_index_sync_context *ctx,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                    enum maildir_uidlist_rec_flag uflags,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                    const char *filename, uint32_t uid)
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    int ret;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
27586e4785d56aeb76e1fd96af8db799688dc64aTimo Sirainen        /* partial syncing */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        return 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    /* most likely a race condition: we read the maildir, then someone else
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen       expunged messages and committed changes to index. so, this message
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen       shouldn't actually exist. */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) == 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        /* mark it racy and check in next sync */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        ctx->mbox->maildir_hdr.cur_check_time = 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        maildir_uidlist_add_flags(ctx->mbox->uidlist, filename,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                      MAILDIR_UIDLIST_REC_FLAG_RACING);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        return 0;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk    }
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk    if (ctx->uidlist_sync_ctx == NULL) {
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk        ret = maildir_uidlist_sync_init(ctx->mbox->uidlist,
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk                        MAILDIR_UIDLIST_SYNC_PARTIAL |
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk                        MAILDIR_UIDLIST_SYNC_KEEP_STATE,
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk                        &ctx->uidlist_sync_ctx);
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk        if (ret <= 0)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            return -1;
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen    }
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen    uflags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen    maildir_uidlist_sync_remove(ctx->uidlist_sync_ctx, filename);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen                    filename, uflags);
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen    i_assert(ret > 0);
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen    /* give the new UID to it immediately */
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen    maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    i_warning("Maildir %s: Expunged message reappeared, giving a new UID "
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen          "(old uid=%u, file=%s)%s", ctx->mbox->box.path,
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen          uid, filename, strncmp(filename, "msg.", 4) != 0 ? "" :
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen          " (Your MDA is saving MH files into Maildir?)");
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    return 0;
9abf5be0962538e1f6f5c73c838ff677341da0c9Timo Sirainen}
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainenint maildir_sync_index_begin(struct maildir_mailbox *mbox,
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen                 struct maildir_sync_context *maildir_sync_ctx,
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen                 struct maildir_index_sync_context **ctx_r)
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen{
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    struct mailbox *_box = &mbox->box;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    struct maildir_index_sync_context *ctx;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    struct mail_index_sync_ctx *sync_ctx;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    struct mail_index_view *view;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen    struct mail_index_transaction *trans;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    enum mail_index_sync_flags sync_flags;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    sync_flags = index_storage_get_sync_flags(&mbox->box);
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    /* don't drop recent messages if we're saving messages */
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    if (maildir_sync_ctx == NULL)
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen        sync_flags &= ~MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    if (mail_index_sync_begin(_box->index, &sync_ctx, &view,
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen                  &trans, sync_flags) < 0) {
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen        mail_storage_set_index_error(_box);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen        return -1;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    }
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    ctx = i_new(struct maildir_index_sync_context, 1);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    ctx->mbox = mbox;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    ctx->maildir_sync_ctx = maildir_sync_ctx;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    ctx->sync_ctx = sync_ctx;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    ctx->view = view;
5c597df6aa8d81de4053c6986fab7739f3b44b20Timo Sirainen    ctx->trans = trans;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    ctx->keywords_sync_ctx =
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen        maildir_keywords_sync_init(mbox->keywords, _box->index);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    ctx->sync_changes =
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen        index_sync_changes_init(ctx->sync_ctx, ctx->view, ctx->trans,
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen                    mbox->box.backend_readonly);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    ctx->start_time = time(NULL);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    *ctx_r = ctx;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    return 0;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen}
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainenstatic bool
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainenmaildir_index_header_has_changed(const struct maildir_index_header *old_hdr,
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen                 const struct maildir_index_header *new_hdr)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen{
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen#define DIR_DELAYED_REFRESH(hdr, name) \
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    ((hdr)->name ## _check_time <= \
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen        (hdr)->name ## _mtime + MAILDIR_SYNC_SECS)
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen    if (old_hdr->new_mtime != new_hdr->new_mtime ||
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs ||
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen        old_hdr->cur_mtime != new_hdr->cur_mtime ||
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen        old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs ||
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen        old_hdr->uidlist_mtime != new_hdr->uidlist_mtime ||
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen        old_hdr->uidlist_mtime_nsecs != new_hdr->uidlist_mtime_nsecs ||
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen        old_hdr->uidlist_size != new_hdr->uidlist_size)
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen        return TRUE;
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen    return DIR_DELAYED_REFRESH(old_hdr, new) !=
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen        DIR_DELAYED_REFRESH(new_hdr, new) ||
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen        DIR_DELAYED_REFRESH(old_hdr, cur) !=
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen        DIR_DELAYED_REFRESH(new_hdr, cur);
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen}
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainenstatic void
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainenmaildir_sync_index_update_ext_header(struct maildir_index_sync_context *ctx)
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen{
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    struct maildir_mailbox *mbox = ctx->mbox;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    const void *data;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    size_t data_size;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen    struct stat st;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    if (ctx->update_maildir_hdr_cur &&
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen        stat(t_strconcat(mbox->box.path, "/cur", NULL), &st) == 0) {
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen        if ((time_t)mbox->maildir_hdr.cur_check_time < st.st_mtime)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen            mbox->maildir_hdr.cur_check_time = st.st_mtime;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen        mbox->maildir_hdr.cur_mtime = st.st_mtime;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen        mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st);
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen    }
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen    mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id,
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen                  &data, &data_size);
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen    if (data_size != sizeof(mbox->maildir_hdr) ||
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        maildir_index_header_has_changed(data, &mbox->maildir_hdr)) {
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen        mail_index_update_header_ext(ctx->trans, mbox->maildir_ext_id,
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen                         0, &mbox->maildir_hdr,
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen                         sizeof(mbox->maildir_hdr));
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    }
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen}
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainenstatic int maildir_sync_index_finish(struct maildir_index_sync_context *ctx,
5494a6bc149da8f02fd25c0434a9d612ac33f659Timo Sirainen                     bool success)
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen{
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    struct maildir_mailbox *mbox = ctx->mbox;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    unsigned int time_diff;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    int ret = success ? 0 : -1;
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen    time_diff = time(NULL) - ctx->start_time;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen    if (time_diff >= MAILDIR_SYNC_TIME_WARN_SECS) {
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen        i_warning("Maildir %s: Synchronization took %u seconds "
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen              "(%u new msgs, %u flag change attempts, "
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen              "%u expunge attempts)",
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen              ctx->mbox->box.path, time_diff,
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen              ctx->new_msgs_count, ctx->flag_change_count,
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen              ctx->expunge_count);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    }
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen    if (ret < 0)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        mail_index_sync_rollback(&ctx->sync_ctx);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    else {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        maildir_sync_index_update_ext_header(ctx);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        /* Set syncing_commit=TRUE so that if any sync callbacks try
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen           to access mails which got lost (eg. expunge callback trying
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen           to open the file which was just unlinked) we don't try to
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen           start a second index sync and crash. */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        mbox->syncing_commit = TRUE;
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen        if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen            mail_storage_set_index_error(&mbox->box);
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen            ret = -1;
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen        }
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen        mbox->syncing_commit = FALSE;
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen    }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    index_sync_changes_deinit(&ctx->sync_changes);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    i_free(ctx);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    return ret;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainenint maildir_sync_index_commit(struct maildir_index_sync_context **_ctx)
4a514fb20e04df397842cde11cc9ea92abfe9728Timo Sirainen{
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen    struct maildir_index_sync_context *ctx = *_ctx;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    *_ctx = NULL;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    return maildir_sync_index_finish(ctx, TRUE);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainenvoid maildir_sync_index_rollback(struct maildir_index_sync_context **_ctx)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen{
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    struct maildir_index_sync_context *ctx = *_ctx;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    *_ctx = NULL;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    (void)maildir_sync_index_finish(ctx, FALSE);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainenstatic int uint_cmp(const void *p1, const void *p2)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen{
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    const unsigned int *i1 = p1, *i2 = p2;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    if (*i1 < *i2)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        return -1;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    else if (*i1 > *i2)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        return 1;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    else
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        return 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen}
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainenstatic void
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainenmaildir_sync_mail_keywords(struct maildir_index_sync_context *ctx, uint32_t seq)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen{
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    struct mailbox *box = &ctx->mbox->box;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    struct mail_keywords *kw;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    unsigned int i, j, old_count, new_count;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    const unsigned int *old_indexes, *new_indexes;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    bool have_indexonly_keywords;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    int diff;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    mail_index_lookup_keywords(ctx->view, seq, &ctx->idx_keywords);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords)) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        /* no changes - we should get here usually */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        return;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    /* sort the keywords */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    array_sort(&ctx->idx_keywords, uint_cmp);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    array_sort(&ctx->keywords, uint_cmp);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    /* drop keywords that are in index-only. we don't want to touch them. */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    old_indexes = array_get(&ctx->idx_keywords, &old_count);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    have_indexonly_keywords = FALSE;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    for (i = old_count; i > 0; i--) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        if (old_indexes[i-1] < MAILDIR_MAX_KEYWORDS)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            break;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        have_indexonly_keywords = TRUE;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen        array_delete(&ctx->idx_keywords, i-1, 1);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen    }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    if (!have_indexonly_keywords) {
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen        /* no index-only keywords found, so something changed.
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen           just replace them all. */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        kw = mail_index_keywords_create_from_indexes(box->index,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                                 &ctx->keywords);
494a5de15db3b2806ab31d5ecc3e1c306ae14d06Timo Sirainen        mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, kw);
08ed4ab71fd2a4e800d9025a736f0f46b771ea90Timo Sirainen        mail_index_keywords_unref(&kw);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        return;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen    }
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen    /* check again if non-index-only keywords changed */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords))
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        return;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    /* we can't reset all the keywords or we'd drop indexonly keywords too.
08ed4ab71fd2a4e800d9025a736f0f46b771ea90Timo Sirainen       so first remove the unwanted keywords and then add back the wanted
08ed4ab71fd2a4e800d9025a736f0f46b771ea90Timo Sirainen       ones. we can get these lists easily by removing common elements
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen       from old and new keywords. */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    new_indexes = array_get(&ctx->keywords, &new_count);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    for (i = j = 0; i < old_count && j < new_count; ) {
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen        diff = (int)old_indexes[i] - (int)new_indexes[j];
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen        if (diff == 0) {
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen            array_delete(&ctx->keywords, j, 1);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen            array_delete(&ctx->idx_keywords, i, 1);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen            old_indexes = array_get(&ctx->idx_keywords, &old_count);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen            new_indexes = array_get(&ctx->keywords, &new_count);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        } else if (diff < 0) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            i++;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        } else {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            j++;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen        }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    if (array_count(&ctx->idx_keywords) > 0) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        kw = mail_index_keywords_create_from_indexes(box->index,
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen                                 &ctx->idx_keywords);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        mail_index_update_keywords(ctx->trans, seq, MODIFY_REMOVE, kw);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        mail_index_keywords_unref(&kw);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen    }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    if (array_count(&ctx->keywords) > 0) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        kw = mail_index_keywords_create_from_indexes(box->index,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen                                 &ctx->keywords);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        mail_index_update_keywords(ctx->trans, seq, MODIFY_ADD, kw);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        mail_index_keywords_unref(&kw);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen}
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainenint maildir_sync_index(struct maildir_index_sync_context *ctx,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen               bool partial)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen{
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    struct maildir_mailbox *mbox = ctx->mbox;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    struct mail_index_view *view = ctx->view;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    struct mail_index_view *view2;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    struct maildir_uidlist_iter_ctx *iter;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    struct mail_index_transaction *trans = ctx->trans;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    const struct mail_index_header *hdr;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    struct mail_index_header empty_hdr;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    const struct mail_index_record *rec;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    uint32_t seq, seq2, uid, prev_uid;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        enum maildir_uidlist_rec_flag uflags;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    const char *filename;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    uint32_t uid_validity, next_uid, hdr_next_uid, first_recent_uid;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    uint32_t first_uid;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    unsigned int changes = 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    int ret = 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    time_t time_before_sync;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    uint8_t expunged_guid_128[MAIL_GUID_128_SIZE];
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    bool expunged, full_rescan = FALSE;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    i_assert(!mbox->syncing_commit);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    first_uid = 1;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    hdr = mail_index_get_header(view);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen    if (uid_validity != hdr->uid_validity &&
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        uid_validity != 0 && hdr->uid_validity != 0) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        /* uidvalidity changed and index isn't being synced for the
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen           first time, reset the index so we can add all messages as
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen           new */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)",
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen              mbox->box.path, hdr->uid_validity, uid_validity);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen        mail_index_reset(trans);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        index_mailbox_reset_uidvalidity(&mbox->box);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        first_uid = hdr->messages_count + 1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        memset(&empty_hdr, 0, sizeof(empty_hdr));
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen        empty_hdr.next_uid = 1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        hdr = &empty_hdr;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    hdr_next_uid = hdr->next_uid;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    time_before_sync = time(NULL);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    mbox->syncing_commit = TRUE;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    seq = prev_uid = 0; first_recent_uid = I_MAX(hdr->first_recent_uid, 1);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    i_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen    i_array_init(&ctx->idx_keywords, MAILDIR_MAX_KEYWORDS);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    iter = maildir_uidlist_iter_init(mbox->uidlist);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        maildir_filename_get_flags(ctx->keywords_sync_ctx, filename,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen                       &ctx->flags, &ctx->keywords);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        i_assert(uid > prev_uid);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        prev_uid = uid;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen        /* the private flags are kept only in indexes. don't use them
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen           at all even for newly seen mails */
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen        ctx->flags &= ~mbox->box.private_flags_mask;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    again:
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        seq++;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        ctx->uid = uid;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        if (seq > hdr->messages_count) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            if (uid < hdr_next_uid) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                if (maildir_handle_uid_insertion(ctx, uflags,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                                 filename,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                                 uid) < 0)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                    ret = -1;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                seq--;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                continue;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen            }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen            /* Trust uidlist recent flags only for newly added
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen               messages. When saving/copying messages with flags
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen               they're stored to cur/ and uidlist treats them
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen               as non-recent. */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen            if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) == 0) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                if (uid >= first_recent_uid)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen                    first_recent_uid = uid + 1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            hdr_next_uid = uid + 1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            mail_index_append(trans, uid, &seq);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            mail_index_update_flags(trans, seq, MODIFY_REPLACE,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen                        ctx->flags);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            if (array_count(&ctx->keywords) > 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                struct mail_keywords *kw;
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                kw = mail_index_keywords_create_from_indexes(
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                    mbox->box.index, &ctx->keywords);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                mail_index_update_keywords(trans, seq,
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen                               MODIFY_REPLACE, kw);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                mail_index_keywords_unref(&kw);
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen            }
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen            continue;
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen        }
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen        rec = mail_index_lookup(view, seq);
cb933f0a570a9cef5c975eadb818aa6b1002a269Timo Sirainen        if (uid > rec->uid) {
cb933f0a570a9cef5c975eadb818aa6b1002a269Timo Sirainen            /* already expunged (no point in showing guid in the
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen               expunge record anymore) */
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen            mail_index_expunge(ctx->trans, seq);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            goto again;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen        if (uid < rec->uid) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            if (maildir_handle_uid_insertion(ctx, uflags,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                             filename, uid) < 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen                ret = -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            seq--;
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen            continue;
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen        }
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen        index_sync_changes_read(ctx->sync_changes, ctx->uid, &expunged,
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen                    expunged_guid_128);
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen        if (expunged) {
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen            if (!maildir_expunge_is_valid_guid(ctx, ctx->uid,
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen                               filename,
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen                               expunged_guid_128))
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen                continue;
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen            if (maildir_file_do(mbox, ctx->uid,
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen                        maildir_expunge, ctx) >= 0) {
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen                /* successful expunge */
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen                mail_index_expunge(ctx->trans, seq);
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen            }
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen            if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen                maildir_sync_notify(ctx->maildir_sync_ctx);
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen            continue;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen        }
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen        /* the private flags are stored only in indexes, keep them */
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen        ctx->flags |= rec->flags & mbox->box.private_flags_mask;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen        if (index_sync_changes_have(ctx->sync_changes)) {
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen            /* apply flag changes to maildir */
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen            if (maildir_file_do(mbox, ctx->uid,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen                        maildir_sync_flags, ctx) < 0)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                maildir_sync_notify(ctx->maildir_sync_ctx);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            /* partial syncing */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                /* we last saw this mail in new/, but it's
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                   not there anymore. possibly expunged,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                   make sure. */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen                full_rescan = TRUE;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen            continue;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen        }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            /* we haven't been able to update maildir with this
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen               record's flag changes. don't sync them. */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen            continue;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        }
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen        if (ctx->flags != (rec->flags & MAIL_FLAGS_NONRECENT)) {
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen            mail_index_update_flags(trans, seq, MODIFY_REPLACE,
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen                        ctx->flags);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen        }
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen        maildir_sync_mail_keywords(ctx, seq);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen    }
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen    maildir_uidlist_iter_deinit(&iter);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen    if (!partial) {
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen        /* expunge the rest */
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen        for (seq++; seq <= hdr->messages_count; seq++)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen            mail_index_expunge(ctx->trans, seq);
b82d6d7f02734007c129fa25bc876049c8d9bddeTimo Sirainen    }
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen    /* add \Recent flags. use updated view so it contains newly
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen       appended messages. */
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen    view2 = mail_index_transaction_open_updated_view(trans);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen    if (mail_index_lookup_seq_range(view2, first_recent_uid, (uint32_t)-1,
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen                    &seq, &seq2) && seq2 >= first_uid) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen        if (seq < first_uid) {
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk            /* UIDVALIDITY changed, skip over the old messages */
c78e7a94528078728cc639b26a1c83e11b4d7e1bTimo Sirainen            seq = first_uid;
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen        }
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen        index_mailbox_set_recent_seq(&mbox->box, view2, seq, seq2);
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen    }
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen    mail_index_view_close(&view2);
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen    if (ctx->uidlist_sync_ctx != NULL) {
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen        if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx,
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen                        TRUE) < 0)
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen            ret = -1;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen    }
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen    if (mbox->box.v.sync_notify != NULL)
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen        mbox->box.v.sync_notify(&mbox->box, 0, 0);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen    /* check cur/ mtime later. if we came here from saving messages they
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen       could still be moved to cur/ directory. */
0910ea0672c0295c442eb686cc41e98656831f37Timo Sirainen    ctx->update_maildir_hdr_cur = TRUE;
0910ea0672c0295c442eb686cc41e98656831f37Timo Sirainen    mbox->maildir_hdr.cur_check_time = time_before_sync;
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen
e91543761d0b7b97a1dc28e036e44d76405545c2Timo Sirainen    if (uid_validity == 0) {
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen        uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity :
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen            maildir_get_uidvalidity_next(mbox->box.list);
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen        maildir_uidlist_set_uid_validity(mbox->uidlist, uid_validity);
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    }
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    maildir_uidlist_set_next_uid(mbox->uidlist, hdr_next_uid, FALSE);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    if (uid_validity != hdr->uid_validity) {
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen        mail_index_update_header(trans,
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen            offsetof(struct mail_index_header, uid_validity),
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen            &uid_validity, sizeof(uid_validity), TRUE);
93688bfedcfb2b9c02750b8d4d409123a386de5cTimo Sirainen    }
93688bfedcfb2b9c02750b8d4d409123a386de5cTimo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    next_uid = maildir_uidlist_get_next_uid(mbox->uidlist);
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    if (hdr_next_uid < next_uid) {
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen        mail_index_update_header(trans,
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen            offsetof(struct mail_index_header, next_uid),
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen            &next_uid, sizeof(next_uid), FALSE);
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    }
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    i_assert(hdr->first_recent_uid <= first_recent_uid);
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen    if (hdr->first_recent_uid < first_recent_uid) {
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen        mail_index_update_header(ctx->trans,
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen            offsetof(struct mail_index_header, first_recent_uid),
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen            &first_recent_uid, sizeof(first_recent_uid), FALSE);
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    }
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    array_free(&ctx->keywords);
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen    array_free(&ctx->idx_keywords);
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen    mbox->syncing_commit = FALSE;
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen    return ret < 0 ? -1 : (full_rescan ? 0 : 1);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen}
fe813f74aaccb12f38e1bd9cd338c6a37fa646e5Timo Sirainen
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volkstatic unsigned int maildir_list_get_ext_id(struct maildir_storage *storage,
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen                        struct mail_index_view *view)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen{
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk    if (storage->maildir_list_ext_id == (uint32_t)-1) {
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk        storage->maildir_list_ext_id =
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk            mail_index_ext_register(mail_index_view_get_index(view),
c78e7a94528078728cc639b26a1c83e11b4d7e1bTimo Sirainen                "maildir", 0,
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen                sizeof(struct maildir_list_index_record),
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen                sizeof(uint32_t));
17cf149e5f47183bfcc1503649dfb92a14f9dcd9Timo Sirainen    }
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen    return storage->maildir_list_ext_id;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen}
17cf149e5f47183bfcc1503649dfb92a14f9dcd9Timo Sirainen
17cf149e5f47183bfcc1503649dfb92a14f9dcd9Timo Sirainenint maildir_list_index_has_changed(struct mailbox *box,
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen                   struct mail_index_view *list_view,
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen                   uint32_t seq)
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen{
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen    struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
17cf149e5f47183bfcc1503649dfb92a14f9dcd9Timo Sirainen    const struct maildir_list_index_record *rec;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen    const void *data;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen    const char *root_dir, *new_dir, *cur_dir;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen    struct stat st;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen    uint32_t ext_id;
fea7b8b3fc182e415b1875d79587c0aa1adb09d7Timo Sirainen    bool expunged;
fea7b8b3fc182e415b1875d79587c0aa1adb09d7Timo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    ext_id = maildir_list_get_ext_id(mbox->storage, list_view);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    rec = data;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    if (rec == NULL || expunged ||
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        rec->new_mtime == 0 || rec->cur_mtime == 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        /* doesn't exist, not synced or dirty-synced */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        return 1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    root_dir = mailbox_list_get_path(box->list, box->name,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen                     MAILBOX_LIST_PATH_TYPE_MAILBOX);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    /* check if new/ changed */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    new_dir = t_strconcat(root_dir, "/new", NULL);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    if (stat(new_dir, &st) < 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        mail_storage_set_critical(box->storage,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen                      "stat(%s) failed: %m", new_dir);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        return -1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    if ((time_t)rec->new_mtime != st.st_mtime)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        return 1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    /* check if cur/ changed */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    cur_dir = t_strconcat(root_dir, "/cur", NULL);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    if (stat(cur_dir, &st) < 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        mail_storage_set_critical(box->storage,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen                      "stat(%s) failed: %m", cur_dir);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        return -1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    if ((time_t)rec->cur_mtime != st.st_mtime)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        return 1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    return 0;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainenint maildir_list_index_update_sync(struct mailbox *box,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen                   struct mail_index_transaction *trans,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen                   uint32_t seq)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen{
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    struct mail_index_view *list_view;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    const struct maildir_index_header *mhdr = &mbox->maildir_hdr;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    const struct maildir_list_index_record *old_rec;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    struct maildir_list_index_record new_rec;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    const void *data;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    uint32_t ext_id;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    bool expunged;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    /* get the current record */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    list_view = mail_index_transaction_get_view(trans);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    ext_id = maildir_list_get_ext_id(mbox->storage, list_view);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    if (expunged)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        return 0;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    old_rec = data;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    memset(&new_rec, 0, sizeof(new_rec));
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    if (mhdr->new_check_time <= mhdr->new_mtime + MAILDIR_SYNC_SECS ||
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        mhdr->cur_check_time <= mhdr->cur_mtime + MAILDIR_SYNC_SECS) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        /* dirty, we need a refresh next time */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    } else {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        new_rec.new_mtime = mhdr->new_mtime;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        new_rec.cur_mtime = mhdr->cur_mtime;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    }
be110f0a3888a24282f06820ec0b5068d2b8f906Timo Sirainen
be110f0a3888a24282f06820ec0b5068d2b8f906Timo Sirainen    if (old_rec == NULL ||
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        memcmp(old_rec, &new_rec, sizeof(old_rec)) != 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen        mail_index_update_ext(trans, seq,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen                      mbox->storage->maildir_list_ext_id,
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen                      &new_rec, NULL);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    return 0;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen}
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen