index/pop3c/pop3c-sync.c

bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "lib.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "ioloop.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "istream.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "bsearch-insert-pos.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "str.h"
6e2856a5beeeb0edf5d852dde63d99fb9af11151Martti Rannanjärvi#include "sort.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "strnum.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "index-mail.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "pop3c-client.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "pop3c-storage.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen#include "pop3c-sync.h"
436adac819e7cbeef04af08dcc6a4f3ecd4a1d9eMartti Rannanjärvi#include "mailbox-recent-flags.h"
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainenstruct pop3c_sync_msg {
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    uint32_t seq;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    const char *uidl;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen};
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo SirainenARRAY_DEFINE_TYPE(pop3c_sync_msg, struct pop3c_sync_msg);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainenint pop3c_sync_get_uidls(struct pop3c_mailbox *mbox)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen{
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    ARRAY_TYPE(const_string) uidls;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    struct istream *input;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    const char *error, *cline;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    char *line, *p;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    unsigned int seq, line_seq;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (mbox->msg_uidls != NULL)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return 0;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if ((pop3c_client_get_capabilities(mbox->client) &
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen         POP3C_CAPABILITY_UIDL) == 0) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        mail_storage_set_error(mbox->box.storage,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                       MAIL_ERROR_NOTPOSSIBLE,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                       "UIDLs not supported by server");
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (pop3c_client_cmd_stream(mbox->client, "UIDL\r\n",
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                    &input, &error) < 0) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi        mailbox_set_critical(&mbox->box, "UIDL failed: %s", error);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    mbox->uidl_pool = pool_alloconly_create("POP3 UIDLs", 1024*32);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    p_array_init(&uidls, mbox->uidl_pool, 64); seq = 0;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    while ((line = i_stream_read_next_line(input)) != NULL) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        seq++;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        p = strchr(line, ' ');
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (p == NULL) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi            mailbox_set_critical(&mbox->box,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                "Invalid UIDL line: %s", line);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            break;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        *p++ = '\0';
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (str_to_uint(line, &line_seq) < 0 || line_seq != seq) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi            mailbox_set_critical(&mbox->box,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                "Unexpected UIDL seq: %s != %u", line, seq);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            break;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        cline = p_strdup(mbox->uidl_pool, p);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        array_append(&uidls, &cline, 1);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    i_stream_destroy(&input);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (line != NULL) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        pool_unref(&mbox->uidl_pool);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (seq == 0) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        /* make msg_uidls non-NULL */
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen        array_append_zero(&uidls);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    mbox->msg_uidls = array_idx(&uidls, 0);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    mbox->msg_count = seq;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    return 0;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen}
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainenint pop3c_sync_get_sizes(struct pop3c_mailbox *mbox)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen{
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    struct istream *input;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    const char *error;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    char *line, *p;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    unsigned int seq, line_seq;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    i_assert(mbox->msg_sizes == NULL);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (mbox->msg_uidls == NULL) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (pop3c_sync_get_uidls(mbox) < 0)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (mbox->msg_count == 0) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        mbox->msg_sizes = i_new(uoff_t, 1);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return 0;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (pop3c_client_cmd_stream(mbox->client, "LIST\r\n",
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                    &input, &error) < 0) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi        mailbox_set_critical(&mbox->box, "LIST failed: %s", error);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    mbox->msg_sizes = i_new(uoff_t, mbox->msg_count); seq = 0;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    while ((line = i_stream_read_next_line(input)) != NULL) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (++seq > mbox->msg_count) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi            mailbox_set_critical(&mbox->box,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                "Too much data in LIST: %s", line);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            break;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        p = strchr(line, ' ');
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (p == NULL) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi            mailbox_set_critical(&mbox->box,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                "Invalid LIST line: %s", line);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            break;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        *p++ = '\0';
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (str_to_uint(line, &line_seq) < 0 || line_seq != seq) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi            mailbox_set_critical(&mbox->box,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                "Unexpected LIST seq: %s != %u", line, seq);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            break;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (str_to_uoff(p, &mbox->msg_sizes[seq-1]) < 0) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi            mailbox_set_critical(&mbox->box,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                "Invalid LIST size: %s", p);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            break;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    i_stream_destroy(&input);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (line != NULL) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        i_free_and_null(mbox->msg_sizes);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    return 0;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen}
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainenstatic void
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainenpop3c_get_local_msgs(pool_t pool, ARRAY_TYPE(pop3c_sync_msg) *local_msgs,
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen             uint32_t messages_count,
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen             struct mail_cache_view *cache_view,
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen             unsigned int cache_idx)
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen{
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    string_t *str = t_str_new(128);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    struct pop3c_sync_msg msg;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    uint32_t seq;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch    i_zero(&msg);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    for (seq = 1; seq <= messages_count; seq++) {
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        str_truncate(str, 0);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        if (mail_cache_lookup_field(cache_view, str, seq,
d546cddd14aa72823a1104e371527ae0a8a99e00Timo Sirainen                        cache_idx) > 0)
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            msg.uidl = p_strdup(pool, str_c(str));
d546cddd14aa72823a1104e371527ae0a8a99e00Timo Sirainen        msg.seq = seq;
d546cddd14aa72823a1104e371527ae0a8a99e00Timo Sirainen        array_idx_set(local_msgs, seq-1, &msg);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    }
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen}
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainenstatic void
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainenpop3c_get_remote_msgs(ARRAY_TYPE(pop3c_sync_msg) *remote_msgs,
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen              struct pop3c_mailbox *mbox)
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen{
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    struct pop3c_sync_msg *msg;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    uint32_t seq;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    for (seq = 1; seq <= mbox->msg_count; seq++) {
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        msg = array_append_space(remote_msgs);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        msg->seq = seq;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        msg->uidl = mbox->msg_uidls[seq-1];
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    }
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen}
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainenstatic int pop3c_sync_msg_uidl_cmp(const struct pop3c_sync_msg *msg1,
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen                   const struct pop3c_sync_msg *msg2)
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen{
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    return null_strcmp(msg1->uidl, msg2->uidl);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen}
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainenstatic void
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainenpop3c_sync_messages(struct pop3c_mailbox *mbox,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            struct mail_index_view *sync_view,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            struct mail_index_transaction *sync_trans,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            struct mail_cache_view *cache_view)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen{
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    struct index_mailbox_context *ibox =
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        INDEX_STORAGE_CONTEXT(&mbox->box);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    const struct mail_index_header *hdr;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    struct mail_cache_transaction_ctx *cache_trans;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    ARRAY_TYPE(pop3c_sync_msg) local_msgs, remote_msgs;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    const struct pop3c_sync_msg *lmsg, *rmsg;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    uint32_t seq1, seq2, next_uid;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    unsigned int lidx, ridx, lcount, rcount;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    unsigned int cache_idx = ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    pool_t pool;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    i_assert(mbox->msg_uids == NULL);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    /* set our uidvalidity */
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    hdr = mail_index_get_header(sync_view);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (hdr->uid_validity == 0) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        uint32_t uid_validity = ioloop_time;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        mail_index_update_header(sync_trans,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            offsetof(struct mail_index_header, uid_validity),
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            &uid_validity, sizeof(uid_validity), TRUE);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    pool = pool_alloconly_create(MEMPOOL_GROWING"pop3c sync", 10240);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    p_array_init(&local_msgs, pool, hdr->messages_count);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    pop3c_get_local_msgs(pool, &local_msgs, hdr->messages_count,
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen                 cache_view, cache_idx);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    p_array_init(&remote_msgs, pool, mbox->msg_count);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    pop3c_get_remote_msgs(&remote_msgs, mbox);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    /* sort the messages by UIDLs, because some servers reorder messages */
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    array_sort(&local_msgs, pop3c_sync_msg_uidl_cmp);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    array_sort(&remote_msgs, pop3c_sync_msg_uidl_cmp);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
c06f45d06438bd063bd3d915682d9db3b6adf725Timo Sirainen    /* skip over existing messages with matching UIDLs and expunge the ones
e7d0bea63a08b08c47c4b5c187d2cb7127859657Timo Sirainen       that no longer exist in remote. */
e7d0bea63a08b08c47c4b5c187d2cb7127859657Timo Sirainen    mbox->msg_uids = mbox->msg_count == 0 ?
e7d0bea63a08b08c47c4b5c187d2cb7127859657Timo Sirainen        i_new(uint32_t, 1) : /* avoid malloc(0) assert */
e7d0bea63a08b08c47c4b5c187d2cb7127859657Timo Sirainen        i_new(uint32_t, mbox->msg_count);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    cache_trans = mail_cache_get_transaction(cache_view, sync_trans);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    lmsg = array_get(&local_msgs, &lcount);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    rmsg = array_get(&remote_msgs, &rcount);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    next_uid = hdr->next_uid;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    lidx = ridx = 0;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    while (lidx < lcount || ridx < rcount) {
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        uint32_t lseq = lidx < lcount ? lmsg[lidx].seq : 0;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        uint32_t rseq = ridx < rcount ? rmsg[ridx].seq : 0;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        int ret;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        if (lidx >= lcount)
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            ret = 1;
d546cddd14aa72823a1104e371527ae0a8a99e00Timo Sirainen        else if (ridx >= rcount || lmsg[lidx].uidl == NULL)
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            ret = -1;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        else
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            ret = strcmp(lmsg[lidx].uidl, rmsg[ridx].uidl);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        if (ret < 0) {
d546cddd14aa72823a1104e371527ae0a8a99e00Timo Sirainen            /* message expunged in remote, or we didn't have a
d546cddd14aa72823a1104e371527ae0a8a99e00Timo Sirainen               local message's UIDL in cache. */
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            mail_index_expunge(sync_trans, lseq);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            lidx++;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        } else if (ret > 0) {
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            /* new message in remote */
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            i_assert(mbox->msg_uids[rseq-1] == 0);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            mbox->msg_uids[rseq-1] = next_uid;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            mail_index_append(sync_trans, next_uid++, &lseq);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            mail_cache_add(cache_trans, lseq, cache_idx,
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen                       rmsg[ridx].uidl,
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen                       strlen(rmsg[ridx].uidl)+1);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            ridx++;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen        } else {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            /* UIDL matched */
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            i_assert(mbox->msg_uids[rseq-1] == 0);
c06f45d06438bd063bd3d915682d9db3b6adf725Timo Sirainen            mail_index_lookup_uid(sync_view, lseq,
c06f45d06438bd063bd3d915682d9db3b6adf725Timo Sirainen                          &mbox->msg_uids[rseq-1]);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            lidx++;
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen            ridx++;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    /* mark the newly seen messages as recent */
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (mail_index_lookup_seq_range(sync_view, hdr->first_recent_uid,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                    hdr->next_uid, &seq1, &seq2))
817d027593510c3ba70ad542ce0011f5f6916d1eTimo Sirainen        mailbox_recent_flags_set_seqs(&mbox->box, sync_view, seq1, seq2);
2676c973a61edf2a9dd5d0196e0993ecb04bb2a2Timo Sirainen    pool_unref(&pool);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen}
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainenint pop3c_sync(struct pop3c_mailbox *mbox)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen{
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        struct mail_index_sync_ctx *index_sync_ctx;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    struct mail_index_view *sync_view, *trans_view;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    struct mail_index_transaction *sync_trans;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    struct mail_index_sync_rec sync_rec;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    struct mail_cache_view *cache_view = NULL;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    enum mail_index_sync_flags sync_flags;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    unsigned int idx;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    string_t *str;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    const char *reply;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    int ret;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    bool deletions = FALSE;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (pop3c_sync_get_uidls(mbox) < 0)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    sync_flags = index_storage_get_sync_flags(&mbox->box) |
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    ret = mail_index_sync_begin(mbox->box.index, &index_sync_ctx,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                    &sync_view, &sync_trans, sync_flags);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (ret <= 0) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (ret < 0)
1f1ee8db68d9ae1604350801cd8dc33ebe29fe8aTimo Sirainen            mailbox_set_index_error(&mbox->box);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return ret;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (mbox->msg_uids == NULL) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        trans_view = mail_index_transaction_open_updated_view(sync_trans);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        cache_view = mail_cache_view_open(mbox->box.cache, trans_view);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        pop3c_sync_messages(mbox, sync_view, sync_trans, cache_view);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    /* mark expunges messages as deleted in this pop3 session,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen       if those exist */
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    str = t_str_new(32);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    while (mail_index_sync_next(index_sync_ctx, &sync_rec)) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (sync_rec.type != MAIL_INDEX_SYNC_TYPE_EXPUNGE)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            continue;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (!bsearch_insert_pos(&sync_rec.uid1, mbox->msg_uids,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                    mbox->msg_count, sizeof(uint32_t),
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                    uint32_cmp, &idx)) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            /* no such messages in this session */
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            continue;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        for (; idx < mbox->msg_count; idx++) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            i_assert(mbox->msg_uids[idx] >= sync_rec.uid1);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            if (mbox->msg_uids[idx] > sync_rec.uid2)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                break;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            str_truncate(str, 0);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            str_printfa(str, "DELE %u\r\n", idx+1);
60d1fdf2c17fd0c7020234590dbd73da81c3ce8fTimo Sirainen            pop3c_client_cmd_line_async_nocb(mbox->client, str_c(str));
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            deletions = TRUE;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (mail_index_sync_commit(&index_sync_ctx) < 0) {
1f1ee8db68d9ae1604350801cd8dc33ebe29fe8aTimo Sirainen        mailbox_set_index_error(&mbox->box);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (cache_view != NULL) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        mail_cache_view_close(&cache_view);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        mail_index_view_close(&trans_view);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    if (deletions) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        if (pop3c_client_cmd_line(mbox->client, "QUIT\r\n",
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                      &reply) < 0) {
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            mail_storage_set_error(mbox->box.storage,
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen                           MAIL_ERROR_TEMP, reply);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen            return -1;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen        }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    return 0;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen}
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainenstruct mailbox_sync_context *
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainenpop3c_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen{
95d62f8d6d281cc488dc4f488d4388701e559012Josef 'Jeff' Sipek    struct pop3c_mailbox *mbox = POP3C_MAILBOX(box);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    int ret = 0;
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
e098ac1cd4ff0afb33b22cc7fd288014f4fdb0bcTimo Sirainen    if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
e098ac1cd4ff0afb33b22cc7fd288014f4fdb0bcTimo Sirainen        mbox->msg_uidls == NULL) {
e098ac1cd4ff0afb33b22cc7fd288014f4fdb0bcTimo Sirainen        /* FIXME: reconnect */
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    }
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen
e098ac1cd4ff0afb33b22cc7fd288014f4fdb0bcTimo Sirainen    ret = pop3c_sync(mbox);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen    return index_mailbox_sync_init(box, flags, ret < 0);
01435c38e7d671d5a892c4b802cfb204881cd454Timo Sirainen}