plugins/pop3-migration/pop3-migration-plugin.c

	pop3-migration-plugin.c revision 8c3872c26b18421d62c52cbfe0b81b1d79239f89
5a580c3a38ced62d4bcc95b8ac7c4f2935b5d294Timo Sirainen/* Copyright (c) 2007-2013 Dovecot authors, see the included COPYING file */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "lib.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "array.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "istream.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "istream-header-filter.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "sha1.h"
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen#include "message-size.h"
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen#include "message-header-parser.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-namespace.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-search-build.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-storage-private.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "pop3-migration-plugin.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#define POP3_MIGRATION_CONTEXT(obj) \
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT(obj, pop3_migration_storage_module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#define POP3_MIGRATION_MAIL_CONTEXT(obj) \
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT(obj, pop3_migration_mail_module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_uidl_map {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    uint32_t pop3_seq;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    uint32_t imap_uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* UIDL */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *pop3_uidl;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* LIST size */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    uoff_t size;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* sha1(TOP 0) - set only when needed */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned char hdr_sha1[SHA1_RESULTLEN];
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int hdr_sha1_set:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct imap_msg_map {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    uint32_t uid, pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    uoff_t psize;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *pop3_uidl;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* sha1(header) - set only when needed */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned char hdr_sha1[SHA1_RESULTLEN];
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int hdr_sha1_set:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_migration_mail_storage {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    union mail_storage_module_context module_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *pop3_box_vname;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen    ARRAY(struct pop3_uidl_map) pop3_uidl_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int all_mailboxes:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int pop3_all_hdr_sha1_set:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_migration_mailbox {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    union mailbox_module_context module_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen    ARRAY(struct imap_msg_map) imap_msg_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int first_unfound_idx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int uidl_synced:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int uidl_sync_failed:1;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    unsigned int uidl_ordered:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic const char *hdr_hash_skip_headers[] = {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "Content-Length",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "Status",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-IMAP",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-IMAPbase",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-Keywords",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-Message-Flag",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-Status",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-UID",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-UIDL"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
bd63b5b860658b01b1f46f26d406e1e4a9dc019aTimo Sirainenconst char *pop3_migration_plugin_version = DOVECOT_ABI_VERSION;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  &mail_storage_module_register);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  &mail_module_register);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_msg_map_uid_cmp(const struct imap_msg_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                const struct imap_msg_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (map1->uid < map2->uid)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (map1->uid > map2->uid)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return 1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                      const struct pop3_uidl_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (map1->pop3_seq < map2->pop3_seq)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (map1->pop3_seq > map2->pop3_seq)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return 1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                 const struct pop3_uidl_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                const struct imap_msg_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic void
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenpop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED,
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen                struct message_header_line *hdr,
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen                bool *matched ATTR_UNUSED, bool *have_eoh)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    if (hdr != NULL && hdr->eoh)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        *have_eoh = TRUE;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic int
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenget_hdr_sha1_stream(struct mail *mail, struct istream *input, uoff_t hdr_size,
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen            unsigned char sha1_r[SHA1_RESULTLEN], bool *have_eoh_r)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen{
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    struct istream *input2;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    const unsigned char *data, *p;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    size_t size, idx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct sha1_ctxt sha1_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    *have_eoh_r = FALSE;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    input2 = i_stream_create_limit(input, hdr_size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* hide headers that might change or be different in IMAP vs. POP3 */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    input = i_stream_create_header_filter(input2,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                hdr_hash_skip_headers,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                N_ELEMENTS(hdr_hash_skip_headers),
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen                pop3_header_filter_callback, have_eoh_r);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    i_stream_unref(&input2);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    sha1_init(&sha1_ctx);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (i_stream_read_data(input, &data, &size, 0) > 0) {
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen        /* if there are NULs in header, replace them with 0x80
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen           character. This is done by at least Dovecot IMAP and also
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen           POP3 with outlook-no-nuls workaround. */
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen        while ((p = memchr(data, '\0', size)) != NULL) {
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen            idx = p - data;
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen            sha1_loop(&sha1_ctx, data, idx);
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen            sha1_loop(&sha1_ctx, "\x80", 1);
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen            i_assert(size > idx);
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen            data += idx + 1;
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen            size -= idx + 1;
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        sha1_loop(&sha1_ctx, data, size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        i_stream_skip(input, size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (input->stream_errno != 0) {
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        i_error("pop3_migration: Failed to read header for msg %u: %s",
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen            mail->seq, i_stream_get_error(input));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        i_stream_unref(&input);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    sha1_result(&sha1_ctx, sha1_r);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    i_stream_unref(&input);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic int
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenget_hdr_sha1(struct mail *mail, unsigned char sha1_r[SHA1_RESULTLEN])
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen{
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    struct istream *input;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    struct message_size hdr_size;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    bool have_eoh;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) {
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        i_error("pop3_migration: Failed to get header for msg %u: %s",
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen            mail->seq, mailbox_get_last_error(mail->box, NULL));
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        return -1;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    }
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    if (get_hdr_sha1_stream(mail, input, hdr_size.physical_size,
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen                sha1_r, &have_eoh) < 0)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        return -1;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    if (have_eoh)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        return 0;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    /* The empty "end of headers" line is missing. Either this means that
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       the headers ended unexpectedly (which is ok) or that the remote
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       server is buggy. Some servers have problems with
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       1) header line continuations that contain only whitespace and
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       2) headers that have no ":". The header gets truncated when such
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       line is reached.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       At least Oracle IMS IMAP FETCH BODY[HEADER] handles 1) by not
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       returning the whitespace line and 2) by returning the line but
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       truncating the rest. POP3 TOP instead returns the entire header.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       This causes the IMAP and POP3 hashes not to match.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       So we'll try to avoid this by falling back to full FETCH BODY[]
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       (and/or RETR) and we'll parse the header ourself from it. This
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       should work around any similar bugs in all IMAP/POP3 servers. */
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    if (mail_get_stream(mail, &hdr_size, NULL, &input) < 0) {
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        i_error("pop3_migration: Failed to get body for msg %u: %s",
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen            mail->seq, mailbox_get_last_error(mail->box, NULL));
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        return -1;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    }
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    return get_hdr_sha1_stream(mail, input, hdr_size.physical_size,
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen                   sha1_r, &have_eoh);
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic struct mailbox *pop3_mailbox_alloc(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_namespace *ns;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ns = mail_namespace_find(storage->user->namespaces,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                 mstorage->pop3_box_vname);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    i_assert(ns != NULL);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    return mailbox_alloc(ns->list, mstorage->pop3_box_vname,
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen                 MAILBOX_FLAG_READONLY | MAILBOX_FLAG_POP3_SESSION);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int pop3_map_read(struct mail_storage *storage, struct mailbox *pop3_box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_uidl_map *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *uidl;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    uoff_t size;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (array_is_created(&mstorage->pop3_uidl_map)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* already read these, just reset the imap_uids */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            map->imap_uid = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    i_array_init(&mstorage->pop3_uidl_map, 128);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mailbox_sync(pop3_box, 0) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        i_error("pop3_migration: Couldn't sync mailbox %s: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    t = mailbox_transaction_begin(pop3_box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_build_add_all(search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ctx = mailbox_search_init(t, search_args, NULL,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  MAIL_FETCH_VIRTUAL_SIZE, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (mailbox_search_next(ctx, &mail)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (mail_get_virtual_size(mail, &size) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            i_error("pop3_migration: Failed to get size for msg %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mail->seq,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mail->seq,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (*uidl == '\0') {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            i_warning("pop3_migration: UIDL for msg %u is empty",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  mail->seq);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map = array_append_space(&mstorage->pop3_uidl_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->pop3_seq = mail->seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->pop3_uidl = p_strdup(storage->pool, uidl);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->size = size;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mailbox_search_deinit(&ctx) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenpop3_map_read_hdr_hashes(struct mail_storage *storage, struct mailbox *pop3_box,
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen             unsigned first_seq)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_uidl_map *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mstorage->pop3_all_hdr_sha1_set)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mstorage->all_mailboxes) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* we may be matching against multiple mailboxes.
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen           read all the hashes only once. */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        first_seq = 1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    t = mailbox_transaction_begin(pop3_box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_build_add_seqset(search_args, first_seq,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                     array_count(&mstorage->pop3_uidl_map)+1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ctx = mailbox_search_init(t, search_args, NULL,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  MAIL_FETCH_STREAM_HEADER, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (mailbox_search_next(ctx, &mail)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map = array_idx_modifiable(&mstorage->pop3_uidl_map,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                       mail->seq-1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (get_hdr_sha1(mail, map->hdr_sha1) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        else
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            map->hdr_sha1_set = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mailbox_search_deinit(&ctx) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (ret == 0 && first_seq == 1)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        mstorage->pop3_all_hdr_sha1_set = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_map_read(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mailbox_status status;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    uoff_t psize;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mailbox_get_open_status(box, STATUS_MESSAGES, &status);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    i_assert(!array_is_created(&mbox->imap_msg_map));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    t = mailbox_transaction_begin(box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_build_add_all(search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ctx = mailbox_search_init(t, search_args, NULL,
c8c4bbf6b1415e9d0845bc8f1cd6d19b76ab0392Timo Sirainen                  MAIL_FETCH_PHYSICAL_SIZE, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (mailbox_search_next(ctx, &mail)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (mail_get_physical_size(mail, &psize) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mail->uid,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mailbox_get_last_error(box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map = array_append_space(&mbox->imap_msg_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->uid = mail->uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->psize = psize;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mailbox_search_deinit(&ctx) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_map_read_hdr_hashes(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    t = mailbox_transaction_begin(box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_build_add_seqset(search_args, mbox->first_unfound_idx+1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                     array_count(&mbox->imap_msg_map)+1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ctx = mailbox_search_init(t, search_args, NULL,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  MAIL_FETCH_STREAM_HEADER, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (mailbox_search_next(ctx, &mail)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map = array_idx_modifiable(&mbox->imap_msg_map, mail->seq-1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (get_hdr_sha1(mail, map->hdr_sha1) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        else
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            map->hdr_sha1_set = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mailbox_search_deinit(&ctx) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic bool pop3_uidl_assign_by_size(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_uidl_map *pop3_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map *imap_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int i, pop3_count, imap_count, count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    count = I_MIN(pop3_count, imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* see if we can match the messages using sizes */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    for (i = 0; i < count; i++) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (pop3_map[i].size != imap_map[i].psize)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            /* two messages with same size, don't trust them */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        pop3_map[i].imap_uid = imap_map[i].uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        imap_map[i].pop3_seq = pop3_map[i].pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mbox->first_unfound_idx = i;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return i == count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenpop3_uidl_assign_by_hdr_hash(struct mailbox *box, struct mailbox *pop3_box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_uidl_map *pop3_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map *imap_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int pop3_idx, imap_idx, pop3_count, imap_count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int first_seq, missing_uids_count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    first_seq = mbox->first_unfound_idx+1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    if (pop3_map_read_hdr_hashes(box->storage, pop3_box, first_seq) < 0 ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        imap_map_read_hdr_hashes(box) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_hdr_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    array_sort(&mbox->imap_msg_map, imap_msg_map_hdr_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    pop3_idx = imap_idx = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (pop3_idx < pop3_count && imap_idx < imap_count) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (!pop3_map[pop3_idx].hdr_sha1_set ||
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            pop3_map[pop3_idx].imap_uid != 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            pop3_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (!imap_map[imap_idx].hdr_sha1_set ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            imap_map[imap_idx].pop3_uidl != NULL) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            imap_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        ret = memcmp(pop3_map[pop3_idx].hdr_sha1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                 imap_map[imap_idx].hdr_sha1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                 sizeof(pop3_map[pop3_idx].hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (ret < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            pop3_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        else if (ret > 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            imap_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        else {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            imap_map[imap_idx].pop3_uidl =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                pop3_map[pop3_idx].pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            imap_map[imap_idx].pop3_seq =
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen                pop3_map[pop3_idx].pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    missing_uids_count = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    for (pop3_idx = 0; pop3_idx < pop3_count; pop3_idx++) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        if (pop3_map[pop3_idx].imap_uid == 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            missing_uids_count++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (missing_uids_count > 0 && !mstorage->all_mailboxes) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        i_warning("pop3_migration: %u POP3 messages have no "
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen              "matching IMAP messages", missing_uids_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_migration_uidl_sync(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        POP3_MIGRATION_CONTEXT(box->storage);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    struct mailbox *pop3_box;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    const struct pop3_uidl_map *pop3_map;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    unsigned int i, count;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    uint32_t prev_uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mbox->uidl_synced)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    pop3_box = pop3_mailbox_alloc(box->storage);
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen    /* the POP3 server isn't connected to yet. handle all IMAP traffic
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen       first before connecting, so POP3 server won't disconnect us due to
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen       idling. */
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    if (imap_map_read(box) < 0 ||
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        pop3_map_read(box->storage, pop3_box) < 0) {
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (!pop3_uidl_assign_by_size(box)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* everything wasn't assigned, figure out the rest with
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen           header hashes */
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        if (pop3_uidl_assign_by_hdr_hash(box, pop3_box) < 0) {
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen            mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            return -1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    /* see if the POP3 UIDL order is the same as IMAP UID order */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    mbox->uidl_ordered = TRUE;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    pop3_map = array_get(&mstorage->pop3_uidl_map, &count);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    prev_uid = 0;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    for (i = 0; i < count; i++) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        if (pop3_map[i].imap_uid == 0)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            continue;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        if (prev_uid > pop3_map[i].imap_uid) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            mbox->uidl_ordered = FALSE;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        prev_uid = pop3_map[i].imap_uid;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mbox->uidl_synced = TRUE;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenpop3_migration_get_special(struct mail *_mail, enum mail_fetch_field field,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen               const char **value_r)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_private *mail = (struct mail_private *)_mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    union mail_module_context *mmail = POP3_MIGRATION_MAIL_CONTEXT(mail);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(_mail->box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map map_key, *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    if (field == MAIL_FETCH_UIDL_BACKEND ||
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        field == MAIL_FETCH_POP3_ORDER) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (mbox->uidl_sync_failed ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            pop3_migration_uidl_sync(_mail->box) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            mbox->uidl_sync_failed = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            mail_storage_set_error(_mail->box->storage,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                           MAIL_ERROR_TEMP,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                           "POP3 UIDLs couldn't be synced");
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        memset(&map_key, 0, sizeof(map_key));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map_key.uid = _mail->uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map = array_bsearch(&mbox->imap_msg_map, &map_key,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                    imap_msg_map_uid_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (map != NULL && map->pop3_uidl != NULL) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            if (field == MAIL_FETCH_UIDL_BACKEND)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen                *value_r = map->pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            else
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen                *value_r = t_strdup_printf("%u", map->pop3_seq);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        /* not found from POP3 server, fallback to default */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return mmail->super.get_special(_mail, field, value_r);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_allocated(struct mail *_mail)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(_mail->box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_private *mail = (struct mail_private *)_mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_vfuncs *v = mail->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    union mail_module_context *mmail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_namespace *ns;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mstorage == NULL ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        (!mstorage->all_mailboxes && !_mail->box->inbox_user)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* assigns UIDLs only for INBOX */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ns = mail_namespace_find(_mail->box->storage->user->namespaces,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                 mstorage->pop3_box_vname);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (ns == mailbox_get_namespace(_mail->box)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* we're accessing the pop3-migration namespace itself */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mmail = p_new(mail->pool, union mail_module_context, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mmail->super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail->vlast = &mmail->super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    v->get_special = pop3_migration_get_special;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT_SET_SELF(mail, pop3_migration_mail_module, mmail);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mailbox_allocated(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mailbox_vfuncs *v = box->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mbox = p_new(box->pool, struct pop3_migration_mailbox, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mbox->module_ctx.super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    box->vlast = &mbox->module_ctx.super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT_SET(box, pop3_migration_storage_module, mbox);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_storage_destroy(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (array_is_created(&mstorage->pop3_uidl_map))
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        array_free(&mstorage->pop3_uidl_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c6afd726060aae56b6622c6c52aec10231c4bf1cTimo Sirainen    mstorage->module_ctx.super.destroy(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_storage_created(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_storage_vfuncs *v = storage->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *pop3_box_vname;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    pop3_box_vname = mail_user_plugin_getenv(storage->user,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                         "pop3_migration_mailbox");
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (pop3_box_vname == NULL)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mstorage = p_new(storage->pool, struct pop3_migration_mail_storage, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mstorage->module_ctx.super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    storage->vlast = &mstorage->module_ctx.super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    v->destroy = pop3_migration_mail_storage_destroy;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mstorage->pop3_box_vname = p_strdup(storage->pool, pop3_box_vname);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    mstorage->all_mailboxes =
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        mail_user_plugin_getenv(storage->user,
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen                    "pop3_migration_all_mailboxes") != NULL;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic struct mail_storage_hooks pop3_migration_mail_storage_hooks = {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    .mail_allocated = pop3_migration_mail_allocated,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    .mailbox_allocated = pop3_migration_mailbox_allocated,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    .mail_storage_created = pop3_migration_mail_storage_created
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid pop3_migration_plugin_init(struct module *module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_storage_hooks_add(module, &pop3_migration_mail_storage_hooks);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid pop3_migration_plugin_deinit(void)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_storage_hooks_remove(&pop3_migration_mail_storage_hooks);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}