plugins/pop3-migration/pop3-migration-plugin.c

	pop3-migration-plugin.c revision 0dffa25d211be541ee3c953b23566a1a990789df
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2007-2016 Dovecot authors, see the included COPYING file */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "lib.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "array.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "istream.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "istream-header-filter.h"
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen#include "str.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "sha1.h"
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen#include "message-size.h"
06fda713b84e857dbc3e80f401a54085c9b0ed16Timo Sirainen#include "message-header-hash.h"
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen#include "message-header-parser.h"
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen#include "mail-cache.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-namespace.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-search-build.h"
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen#include "index-storage.h"
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen#include "index-mail.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "pop3-migration-plugin.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#define POP3_MIGRATION_CONTEXT(obj) \
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT(obj, pop3_migration_storage_module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#define POP3_MIGRATION_MAIL_CONTEXT(obj) \
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT(obj, pop3_migration_mail_module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenstruct msg_map_common {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    /* sha1(header) - set only when needed */
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    unsigned char hdr_sha1[SHA1_RESULTLEN];
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool hdr_sha1_set:1;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen};
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_uidl_map {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    struct msg_map_common common;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    uint32_t pop3_seq;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    uint32_t imap_uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* UIDL */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *pop3_uidl;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* LIST size */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    uoff_t size;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct imap_msg_map {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    struct msg_map_common common;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    uint32_t uid, pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    uoff_t psize;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *pop3_uidl;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_migration_mail_storage {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    union mail_storage_module_context module_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *pop3_box_vname;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen    ARRAY(struct pop3_uidl_map) pop3_uidl_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool all_mailboxes:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool pop3_all_hdr_sha1_set:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool ignore_missing_uidls:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool skip_size_check:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_migration_mailbox {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    union mailbox_module_context module_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen    ARRAY(struct imap_msg_map) imap_msg_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int first_unfound_idx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    struct mail_cache_field cache_field;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool cache_field_registered:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool uidl_synced:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool uidl_sync_failed:1;
0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen    bool uidl_ordered:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
35ef661bd85c64834e3e34eeeb3c393b81108760Timo Sirainen/* NOTE: these headers must be sorted */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic const char *hdr_hash_skip_headers[] = {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "Content-Length",
35ef661bd85c64834e3e34eeeb3c393b81108760Timo Sirainen    "Return-Path", /* Yahoo IMAP has Return-Path, Yahoo POP3 doesn't */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "Status",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-IMAP",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-IMAPbase",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-Keywords",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-Message-Flag",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-Status",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    "X-UID",
c39c3d8089fbdd8eb34646c25167aa4551064cf4Timo Sirainen    "X-UIDL",
c39c3d8089fbdd8eb34646c25167aa4551064cf4Timo Sirainen    "X-Yahoo-Newman-Property"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
bd63b5b860658b01b1f46f26d406e1e4a9dc019aTimo Sirainenconst char *pop3_migration_plugin_version = DOVECOT_ABI_VERSION;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  &mail_storage_module_register);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  &mail_module_register);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_msg_map_uid_cmp(const struct imap_msg_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                const struct imap_msg_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (map1->uid < map2->uid)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (map1->uid > map2->uid)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return 1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                      const struct pop3_uidl_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (map1->pop3_seq < map2->pop3_seq)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (map1->pop3_seq > map2->pop3_seq)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return 1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                 const struct pop3_uidl_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    return memcmp(map1->common.hdr_sha1, map2->common.hdr_sha1,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen              sizeof(map1->common.hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                const struct imap_msg_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    return memcmp(map1->common.hdr_sha1, map2->common.hdr_sha1,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen              sizeof(map1->common.hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainenstruct pop3_hdr_context {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    bool have_eoh;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    bool stop;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen};
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainenstatic bool header_name_is_valid(const char *name)
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen{
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen    unsigned int i;
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen    for (i = 0; name[i] != '\0'; i++) {
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen        if ((uint8_t)name[i] <= 0x20 || name[i] >= 0x7f)
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen            return FALSE;
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen    }
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen    return TRUE;
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen}
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic void
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenpop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED,
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen                struct message_header_line *hdr,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen                bool *matched, struct pop3_hdr_context *ctx)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    if (hdr == NULL)
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen        return;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    if (hdr->eoh) {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen        ctx->have_eoh = TRUE;
44ef49403ac7bddac84a1e322d170ed53cd37c95Timo Sirainen        if (ctx->stop)
44ef49403ac7bddac84a1e322d170ed53cd37c95Timo Sirainen            *matched = TRUE;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    } else {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen        if (strspn(hdr->name, "\r") == hdr->name_len) {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen            /* CR+CR+LF - some servers stop the header processing
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen             here while others don't. To make sure they can be
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen             matched correctly we want to stop here entirely. */
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen            ctx->stop = TRUE;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen        }
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen        if (ctx->stop)
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen            *matched = TRUE;
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen        else if (!header_name_is_valid(hdr->name)) {
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen            /* Yahoo IMAP drops headers with invalid names, while
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen               Yahoo POP3 preserves them. Drop them all. */
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen            *matched = TRUE;
5fa253bd316540ec280ca76b39d62a9e32da228bTimo Sirainen        }
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    }
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainenint pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen                uoff_t hdr_size,
9625595c47c665f5aee57ebfcb1fcbe9ad1bf3a0Martti Rannanjärvi                unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN],
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen                bool *have_eoh_r)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen{
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    struct istream *input2;
2f19f8ff906a0017b906763e0f7675d49ab0e58fTimo Sirainen    const unsigned char *data;
06fda713b84e857dbc3e80f401a54085c9b0ed16Timo Sirainen    size_t size;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct sha1_ctxt sha1_ctx;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    struct pop3_hdr_context hdr_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    memset(&hdr_ctx, 0, sizeof(hdr_ctx));
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    input2 = i_stream_create_limit(input, hdr_size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* hide headers that might change or be different in IMAP vs. POP3 */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    input = i_stream_create_header_filter(input2,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                hdr_hash_skip_headers,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                N_ELEMENTS(hdr_hash_skip_headers),
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen                pop3_header_filter_callback, &hdr_ctx);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    i_stream_unref(&input2);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    sha1_init(&sha1_ctx);
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody    while (i_stream_read_more(input, &data, &size) > 0) {
06fda713b84e857dbc3e80f401a54085c9b0ed16Timo Sirainen        message_header_hash_more(&hash_method_sha1, &sha1_ctx, 2,
06fda713b84e857dbc3e80f401a54085c9b0ed16Timo Sirainen                     data, size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        i_stream_skip(input, size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (input->stream_errno != 0) {
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        i_error("pop3_migration: Failed to read header for msg %u: %s",
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen            mail_seq, i_stream_get_error(input));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        i_stream_unref(&input);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    sha1_result(&sha1_ctx, sha1_r);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    i_stream_unref(&input);
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    *have_eoh_r = hdr_ctx.have_eoh;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainenstatic unsigned int get_cache_idx(struct mail *mail)
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen{
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(mail->box);
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    if (mbox->cache_field_registered)
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen        return mbox->cache_field.idx;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    mbox->cache_field.name = "pop3-migration.hdr";
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    mbox->cache_field.type = MAIL_CACHE_FIELD_FIXED_SIZE;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    mbox->cache_field.field_size = SHA1_RESULTLEN;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    mail_cache_register_fields(mail->box->cache, &mbox->cache_field, 1);
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    mbox->cache_field_registered = TRUE;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    return mbox->cache_field.idx;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen}
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic int
9625595c47c665f5aee57ebfcb1fcbe9ad1bf3a0Martti Rannanjärviget_hdr_sha1(struct mail *mail, unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN])
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen{
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    struct istream *input;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    struct message_size hdr_size;
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen    const char *errstr;
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen    enum mail_error error;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    bool have_eoh;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) {
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        errstr = mailbox_get_last_error(mail->box, &error);
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        i_error("pop3_migration: Failed to get header for msg %u: %s",
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen            mail->seq, errstr);
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        return error == MAIL_ERROR_EXPUNGED ? 0 : -1;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    }
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    if (pop3_migration_get_hdr_sha1(mail->seq, input,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen                    hdr_size.physical_size,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen                    sha1_r, &have_eoh) < 0)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        return -1;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    if (have_eoh) {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        struct index_mail *imail = (struct index_mail *)mail;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        index_mail_cache_add_idx(imail, get_cache_idx(mail),
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                     sha1_r, SHA1_RESULTLEN);
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        return 1;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    }
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    /* The empty "end of headers" line is missing. Either this means that
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       the headers ended unexpectedly (which is ok) or that the remote
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       server is buggy. Some servers have problems with
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       1) header line continuations that contain only whitespace and
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       2) headers that have no ":". The header gets truncated when such
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       line is reached.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       At least Oracle IMS IMAP FETCH BODY[HEADER] handles 1) by not
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       returning the whitespace line and 2) by returning the line but
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       truncating the rest. POP3 TOP instead returns the entire header.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       This causes the IMAP and POP3 hashes not to match.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen       If there's LF+CR+CR+LF in the middle of headers, Courier IMAP's
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen       FETCH BODY[HEADER] stops after that, but Courier POP3's TOP doesn't.
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       So we'll try to avoid this by falling back to full FETCH BODY[]
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       (and/or RETR) and we'll parse the header ourself from it. This
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen       should work around any similar bugs in all IMAP/POP3 servers. */
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    if (mail_get_stream(mail, &hdr_size, NULL, &input) < 0) {
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        errstr = mailbox_get_last_error(mail->box, &error);
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen        i_error("pop3_migration: Failed to get body for msg %u: %s",
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen            mail->seq, errstr);
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        return error == MAIL_ERROR_EXPUNGED ? 0 : -1;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen    }
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen    return pop3_migration_get_hdr_sha1(mail->seq, input,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen                       hdr_size.physical_size,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen                       sha1_r, &have_eoh);
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenstatic bool
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainenget_cached_hdr_sha1(struct mail *mail, buffer_t *cache_buf,
9625595c47c665f5aee57ebfcb1fcbe9ad1bf3a0Martti Rannanjärvi            unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN])
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen{
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    struct index_mail *imail = (struct index_mail *)mail;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    buffer_set_used_size(cache_buf, 0);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    if (index_mail_cache_lookup_field(imail, cache_buf,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                      get_cache_idx(mail)) > 0 &&
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen        cache_buf->used == SHA1_RESULTLEN) {
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen        memcpy(sha1_r, cache_buf->data, cache_buf->used);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        return TRUE;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen    }
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    return FALSE;
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen}
57e1fdc2f8f2bf1c6fcd9523f93459404c2359c8Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic struct mailbox *pop3_mailbox_alloc(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_namespace *ns;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ns = mail_namespace_find(storage->user->namespaces,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                 mstorage->pop3_box_vname);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    i_assert(ns != NULL);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    return mailbox_alloc(ns->list, mstorage->pop3_box_vname,
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen                 MAILBOX_FLAG_READONLY | MAILBOX_FLAG_POP3_SESSION);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int pop3_map_read(struct mail_storage *storage, struct mailbox *pop3_box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_uidl_map *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *uidl;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen    uoff_t size = (uoff_t)-1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (array_is_created(&mstorage->pop3_uidl_map)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* already read these, just reset the imap_uids */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            map->imap_uid = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    i_array_init(&mstorage->pop3_uidl_map, 128);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mailbox_sync(pop3_box, 0) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        i_error("pop3_migration: Couldn't sync mailbox %s: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    t = mailbox_transaction_begin(pop3_box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_build_add_all(search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ctx = mailbox_search_init(t, search_args, NULL,
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen                  mstorage->skip_size_check ? 0 :
b3070aca2aa0dbdcc7f30e6e2bb1f0455127a343Timo Sirainen                  MAIL_FETCH_PHYSICAL_SIZE, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (mailbox_search_next(ctx, &mail)) {
7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen        /* get the size with LIST instead of RETR */
7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen        mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen        if (mstorage->skip_size_check)
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen            ;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen        else if (mail_get_physical_size(mail, &size) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            i_error("pop3_migration: Failed to get size for msg %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mail->seq,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mail->seq,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (*uidl == '\0') {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            i_warning("pop3_migration: UIDL for msg %u is empty",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  mail->seq);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map = array_append_space(&mstorage->pop3_uidl_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->pop3_seq = mail->seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->pop3_uidl = p_strdup(storage->pool, uidl);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->size = size;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    if (mailbox_search_deinit(&ctx) < 0) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen        i_error("pop3_migration: Failed to search all POP3 mails: %s",
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen            mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        ret = -1;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenstatic void
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenpop3_map_read_cached_hdr_hashes(struct mailbox_transaction_context *t,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                struct mail_search_args *search_args,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                struct array *msg_map)
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen{
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    struct mail_search_context *ctx;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    struct mail *mail;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    struct msg_map_common *map;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    buffer_t *cache_buf;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    cache_buf = buffer_create_dynamic(pool_datastack_create(), SHA1_RESULTLEN);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    while (mailbox_search_next(ctx, &mail)) {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        map = array_idx_modifiable_i(msg_map, mail->seq-1);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        if (get_cached_hdr_sha1(mail, cache_buf, map->hdr_sha1))
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen            map->hdr_sha1_set = TRUE;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    }
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    if (mailbox_search_deinit(&ctx) < 0) {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        i_warning("pop3_migration: Failed to search all cached POP3 header hashes: %s - ignoring",
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen              mailbox_get_last_error(t->box, NULL));
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    }
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen}
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenstatic void map_remove_found_seqs(struct mail_search_arg *search_arg,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                  struct array *msg_map, uint32_t seq1)
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen{
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    const struct msg_map_common *map;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    uint32_t seq, count = array_count_i(msg_map);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    i_assert(search_arg->type == SEARCH_SEQSET);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    for (seq = seq1; seq <= count; seq++) {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        map = array_idx_i(msg_map, seq-1);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        if (map->hdr_sha1_set)
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen            seq_range_array_remove(&search_arg->value.seqset, seq);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    }
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen}
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenmap_read_hdr_hashes(struct mailbox *box, struct array *msg_map, uint32_t seq1)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail *mail;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    struct msg_map_common *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    t = mailbox_transaction_begin(box, 0);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    /* get all the cached hashes */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    search_args = mail_search_build_init();
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    mail_search_build_add_seqset(search_args, seq1, array_count_i(msg_map));
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    pop3_map_read_cached_hdr_hashes(t, search_args, msg_map);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    /* read all the non-cached hashes. doing this in two passes allows
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen       us to set wanted_fields=MAIL_FETCH_STREAM_HEADER, which allows
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen       prefetching to work without downloading all the headers even
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen       for mails that already are cached. */
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    map_remove_found_seqs(search_args->args, msg_map, seq1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ctx = mailbox_search_init(t, search_args, NULL,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                  MAIL_FETCH_STREAM_HEADER, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (mailbox_search_next(ctx, &mail)) {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        map = array_idx_modifiable_i(msg_map, mail->seq-1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        if ((ret = get_hdr_sha1(mail, map->hdr_sha1)) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
cfb351c1d28fab5a461f6506471c6b6eb26c0bdaTimo Sirainen            break;
cfb351c1d28fab5a461f6506471c6b6eb26c0bdaTimo Sirainen        }
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        if (ret > 0)
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen            map->hdr_sha1_set = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    if (mailbox_search_deinit(&ctx) < 0) {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        i_error("pop3_migration: Failed to search all mail headers: %s",
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen            mailbox_get_last_error(box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        ret = -1;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    (void)mailbox_transaction_commit(&t);
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen    return ret < 0 ? -1 : 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenstatic int
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainenpop3_map_read_hdr_hashes(struct mail_storage *storage, struct mailbox *pop3_box,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen             unsigned first_seq)
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen{
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    struct pop3_migration_mail_storage *mstorage =
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        POP3_MIGRATION_CONTEXT(storage);
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    if (mstorage->pop3_all_hdr_sha1_set)
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        return 0;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    if (mstorage->all_mailboxes) {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        /* we may be matching against multiple mailboxes.
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen           read all the hashes only once. */
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        first_seq = 1;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    }
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    if (map_read_hdr_hashes(pop3_box, &mstorage->pop3_uidl_map.arr,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                first_seq) < 0)
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        return -1;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    if (first_seq == 1)
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        mstorage->pop3_all_hdr_sha1_set = TRUE;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    return 0;
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen}
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_map_read(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen        POP3_MIGRATION_CONTEXT(box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mailbox_status status;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map *map;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen    uoff_t psize = (uoff_t)-1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mailbox_get_open_status(box, STATUS_MESSAGES, &status);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    i_assert(!array_is_created(&mbox->imap_msg_map));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    t = mailbox_transaction_begin(box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_build_add_all(search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ctx = mailbox_search_init(t, search_args, NULL,
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen                  mstorage->skip_size_check ? 0 :
c8c4bbf6b1415e9d0845bc8f1cd6d19b76ab0392Timo Sirainen                  MAIL_FETCH_PHYSICAL_SIZE, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (mailbox_search_next(ctx, &mail)) {
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen        if (mstorage->skip_size_check)
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen            ;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen        else if (mail_get_physical_size(mail, &psize) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mail->uid,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                mailbox_get_last_error(box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map = array_append_space(&mbox->imap_msg_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->uid = mail->uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map->psize = psize;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    if (mailbox_search_deinit(&ctx) < 0) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen        i_error("pop3_migration: Failed to search all IMAP mails: %s",
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen            mailbox_get_last_error(box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        ret = -1;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_map_read_hdr_hashes(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen    return map_read_hdr_hashes(box, &mbox->imap_msg_map.arr,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                   mbox->first_unfound_idx+1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic bool pop3_uidl_assign_by_size(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_uidl_map *pop3_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map *imap_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int i, pop3_count, imap_count, count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
414d4ee117b45834e6e02f32cda9c9472dd89e1dTimo Sirainen    if (mstorage->skip_size_check)
414d4ee117b45834e6e02f32cda9c9472dd89e1dTimo Sirainen        return FALSE;
414d4ee117b45834e6e02f32cda9c9472dd89e1dTimo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    count = I_MIN(pop3_count, imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    /* see if we can match the messages using sizes */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    for (i = 0; i < count; i++) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (pop3_map[i].size != imap_map[i].psize)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            /* two messages with same size, don't trust them */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        pop3_map[i].imap_uid = imap_map[i].uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        imap_map[i].pop3_seq = pop3_map[i].pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mbox->first_unfound_idx = i;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    if (box->storage->user->mail_debug)
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen        i_debug("pop3_migration: %u/%u mails matched by size", i, count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return i == count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenpop3_uidl_assign_by_hdr_hash(struct mailbox *box, struct mailbox *pop3_box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_uidl_map *pop3_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map *imap_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int pop3_idx, imap_idx, pop3_count, imap_count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    unsigned int first_seq, missing_uids_count;
7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen    uint32_t first_missing_idx = (uint32_t)-1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    int ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    first_seq = mbox->first_unfound_idx+1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    if (pop3_map_read_hdr_hashes(box->storage, pop3_box, first_seq) < 0 ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        imap_map_read_hdr_hashes(box) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_hdr_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    array_sort(&mbox->imap_msg_map, imap_msg_map_hdr_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    pop3_idx = imap_idx = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    while (pop3_idx < pop3_count && imap_idx < imap_count) {
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        if (!pop3_map[pop3_idx].common.hdr_sha1_set ||
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            pop3_map[pop3_idx].imap_uid != 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            pop3_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        if (!imap_map[imap_idx].common.hdr_sha1_set ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            imap_map[imap_idx].pop3_uidl != NULL) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            imap_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen        ret = memcmp(pop3_map[pop3_idx].common.hdr_sha1,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                 imap_map[imap_idx].common.hdr_sha1,
9abc6ac61e70b809f7e1c352c7a3ad1081994d2eTimo Sirainen                 sizeof(pop3_map[pop3_idx].common.hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (ret < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            pop3_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        else if (ret > 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            imap_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        else {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            imap_map[imap_idx].pop3_uidl =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                pop3_map[pop3_idx].pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            imap_map[imap_idx].pop3_seq =
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen                pop3_map[pop3_idx].pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    missing_uids_count = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    for (pop3_idx = 0; pop3_idx < pop3_count; pop3_idx++) {
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        if (pop3_map[pop3_idx].imap_uid != 0) {
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen            /* matched */
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        } else if (!pop3_map[pop3_idx].common.hdr_sha1_set) {
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen            /* we treated this mail as expunged - ignore */
a58963a8bdac0438a31abc03711e8870b4dfa4f4Timo Sirainen        } else {
7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen            if (first_missing_idx == (uint32_t)-1)
12e32734b3fe762c647c1ae5552185517a988cf8Timo Sirainen                first_missing_idx = pop3_idx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            missing_uids_count++;
7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (missing_uids_count > 0 && !mstorage->all_mailboxes) {
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen        string_t *str = t_str_new(128);
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen        str_printfa(str, "pop3_migration: %u POP3 messages have no "
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen                "matching IMAP messages (first POP3 msg %u UIDL %s)",
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen                missing_uids_count,
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen                pop3_map[first_missing_idx].pop3_seq,
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen                pop3_map[first_missing_idx].pop3_uidl);
e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen        if (imap_count + missing_uids_count == pop3_count) {
e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen            str_append(str, " - all IMAP messages were found "
e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen                "(POP3 contains more than IMAP INBOX - you may want to set pop3_migration_all_mailboxes=yes)");
e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen        }
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen        if (!mstorage->ignore_missing_uidls) {
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen            i_error("%s - set pop3_migration_ignore_missing_uidls=yes to continue anyway",
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen                str_c(str));
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen            return -1;
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen        }
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen        i_warning("%s", str_c(str));
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    } else if (box->storage->user->mail_debug) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen        i_debug("pop3_migration: %u mails matched by headers", pop3_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_migration_uidl_sync(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        POP3_MIGRATION_CONTEXT(box->storage);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    struct mailbox *pop3_box;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    const struct pop3_uidl_map *pop3_map;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    unsigned int i, count;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    uint32_t prev_uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    pop3_box = pop3_mailbox_alloc(box->storage);
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen    /* the POP3 server isn't connected to yet. handle all IMAP traffic
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen       first before connecting, so POP3 server won't disconnect us due to
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen       idling. */
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    if (imap_map_read(box) < 0 ||
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        pop3_map_read(box->storage, pop3_box) < 0) {
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return -1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (!pop3_uidl_assign_by_size(box)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* everything wasn't assigned, figure out the rest with
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen           header hashes */
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        if (pop3_uidl_assign_by_hdr_hash(box, pop3_box) < 0) {
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen            mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            return -1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen        }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    /* see if the POP3 UIDL order is the same as IMAP UID order */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    mbox->uidl_ordered = TRUE;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    pop3_map = array_get(&mstorage->pop3_uidl_map, &count);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    prev_uid = 0;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    for (i = 0; i < count; i++) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        if (pop3_map[i].imap_uid == 0)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            continue;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        if (prev_uid > pop3_map[i].imap_uid) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            mbox->uidl_ordered = FALSE;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        prev_uid = pop3_map[i].imap_uid;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mbox->uidl_synced = TRUE;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainenstatic int pop3_migration_uidl_sync_if_needed(struct mailbox *box)
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen{
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    if (mbox->uidl_synced)
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen        return 0;
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    if (mbox->uidl_sync_failed ||
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen        pop3_migration_uidl_sync(box) < 0) {
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen        mbox->uidl_sync_failed = TRUE;
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen        mail_storage_set_error(box->storage, MAIL_ERROR_TEMP,
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen                       "POP3 UIDLs couldn't be synced");
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen        return -1;
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    }
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    return 0;
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen}
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenpop3_migration_get_special(struct mail *_mail, enum mail_fetch_field field,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen               const char **value_r)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_private *mail = (struct mail_private *)_mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    union mail_module_context *mmail = POP3_MIGRATION_MAIL_CONTEXT(mail);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(_mail->box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct imap_msg_map map_key, *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen    if (field == MAIL_FETCH_UIDL_BACKEND ||
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        field == MAIL_FETCH_POP3_ORDER) {
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen        if (pop3_migration_uidl_sync_if_needed(_mail->box) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            return -1;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        memset(&map_key, 0, sizeof(map_key));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map_key.uid = _mail->uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        map = array_bsearch(&mbox->imap_msg_map, &map_key,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                    imap_msg_map_uid_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        if (map != NULL && map->pop3_uidl != NULL) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            if (field == MAIL_FETCH_UIDL_BACKEND)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen                *value_r = map->pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen            else
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen                *value_r = t_strdup_printf("%u", map->pop3_seq);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen            return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen        /* not found from POP3 server, fallback to default */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    return mmail->super.get_special(_mail, field, value_r);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_allocated(struct mail *_mail)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(_mail->box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_private *mail = (struct mail_private *)_mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_vfuncs *v = mail->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    union mail_module_context *mmail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_namespace *ns;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (mstorage == NULL ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        (!mstorage->all_mailboxes && !_mail->box->inbox_user)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* assigns UIDLs only for INBOX */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    ns = mail_namespace_find(_mail->box->storage->user->namespaces,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                 mstorage->pop3_box_vname);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (ns == mailbox_get_namespace(_mail->box)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        /* we're accessing the pop3-migration namespace itself */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mmail = p_new(mail->pool, union mail_module_context, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mmail->super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail->vlast = &mmail->super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    v->get_special = pop3_migration_get_special;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT_SET_SELF(mail, pop3_migration_mail_module, mmail);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainenstatic struct mail_search_context *
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainenpop3_migration_mailbox_search_init(struct mailbox_transaction_context *t,
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen                   struct mail_search_args *args,
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen                   const enum mail_sort_type *sort_program,
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen                   enum mail_fetch_field wanted_fields,
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen                   struct mailbox_header_lookup_ctx *wanted_headers)
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen{
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(t->box);
d430698b6d9d47ff9136ee137cc58a1c657baeddTimo Sirainen    struct pop3_migration_mail_storage *mstorage =
d430698b6d9d47ff9136ee137cc58a1c657baeddTimo Sirainen        POP3_MIGRATION_CONTEXT(t->box->storage);
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    if ((wanted_fields & (MAIL_FETCH_UIDL_BACKEND |
d430698b6d9d47ff9136ee137cc58a1c657baeddTimo Sirainen                  MAIL_FETCH_POP3_ORDER)) != 0 &&
d430698b6d9d47ff9136ee137cc58a1c657baeddTimo Sirainen        (mstorage->all_mailboxes || t->box->inbox_user)) {
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen        /* Start POP3 UIDL syncing before the search, so we'll do it
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen           before we start sending any FETCH BODY[]s to IMAP. It
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen           shouldn't matter much, except this works around a bug in
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen           Yahoo IMAP where it sometimes breaks its state when doing
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen           a FETCH BODY[] followed by FETCH BODY[HEADER].. */
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen        (void)pop3_migration_uidl_sync_if_needed(t->box);
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    }
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    return mbox->module_ctx.super.search_init(t, args, sort_program,
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen                          wanted_fields, wanted_headers);
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen}
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mailbox_allocated(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mailbox_vfuncs *v = box->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mailbox *mbox;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mbox = p_new(box->pool, struct pop3_migration_mailbox, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mbox->module_ctx.super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    box->vlast = &mbox->module_ctx.super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen    v->search_init = pop3_migration_mailbox_search_init;
5407a86a03963261bf5ba8793b8c97879ad2e224Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT_SET(box, pop3_migration_storage_module, mbox);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_storage_destroy(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    if (array_is_created(&mstorage->pop3_uidl_map))
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        array_free(&mstorage->pop3_uidl_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c6afd726060aae56b6622c6c52aec10231c4bf1cTimo Sirainen    mstorage->module_ctx.super.destroy(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_storage_created(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct pop3_migration_mail_storage *mstorage;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    struct mail_storage_vfuncs *v = storage->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    const char *pop3_box_vname;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    pop3_box_vname = mail_user_plugin_getenv(storage->user,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen                         "pop3_migration_mailbox");
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    if (pop3_box_vname == NULL) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen        if (storage->user->mail_debug)
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen            i_debug("pop3_migration: No pop3_migration_mailbox setting - disabled");
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen        return;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen    }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mstorage = p_new(storage->pool, struct pop3_migration_mail_storage, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mstorage->module_ctx.super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    storage->vlast = &mstorage->module_ctx.super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    v->destroy = pop3_migration_mail_storage_destroy;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mstorage->pop3_box_vname = p_strdup(storage->pool, pop3_box_vname);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen    mstorage->all_mailboxes =
71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang        mail_user_plugin_getenv_bool(storage->user,
71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang                    "pop3_migration_all_mailboxes");
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen    mstorage->ignore_missing_uidls =
71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang        mail_user_plugin_getenv_bool(storage->user,
71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang            "pop3_migration_ignore_missing_uidls");
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen    mstorage->skip_size_check =
71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang        mail_user_plugin_getenv_bool(storage->user,
71748cca1bacd74451fd228db5536828bdfeb190Baofeng Wang            "pop3_migration_skip_size_check");
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic struct mail_storage_hooks pop3_migration_mail_storage_hooks = {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    .mail_allocated = pop3_migration_mail_allocated,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    .mailbox_allocated = pop3_migration_mailbox_allocated,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    .mail_storage_created = pop3_migration_mail_storage_created
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid pop3_migration_plugin_init(struct module *module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_storage_hooks_add(module, &pop3_migration_mail_storage_hooks);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid pop3_migration_plugin_deinit(void)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen    mail_storage_hooks_remove(&pop3_migration_mail_storage_hooks);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}