pop3-migration-plugin.c revision e9d659ad49a3cf2190606a62289c86347608bffa
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2007-2016 Dovecot authors, see the included COPYING file */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "lib.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "array.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "istream.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "istream-header-filter.h"
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen#include "str.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "sha1.h"
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen#include "message-size.h"
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen#include "message-header-parser.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-namespace.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-search-build.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "mail-storage-private.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#include "pop3-migration-plugin.h"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#define POP3_MIGRATION_CONTEXT(obj) \
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen MODULE_CONTEXT(obj, pop3_migration_storage_module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen#define POP3_MIGRATION_MAIL_CONTEXT(obj) \
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen MODULE_CONTEXT(obj, pop3_migration_mail_module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_uidl_map {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uint32_t pop3_seq;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen uint32_t imap_uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* UIDL */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *pop3_uidl;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* LIST size */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uoff_t size;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* sha1(TOP 0) - set only when needed */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned char hdr_sha1[SHA1_RESULTLEN];
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int hdr_sha1_set:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct imap_msg_map {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen uint32_t uid, pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen uoff_t psize;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *pop3_uidl;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* sha1(header) - set only when needed */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned char hdr_sha1[SHA1_RESULTLEN];
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int hdr_sha1_set:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_migration_mail_storage {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen union mail_storage_module_context module_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *pop3_box_vname;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct pop3_uidl_map) pop3_uidl_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int all_mailboxes:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int pop3_all_hdr_sha1_set:1;
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen unsigned int ignore_missing_uidls:1;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen unsigned int skip_size_check:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstruct pop3_migration_mailbox {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen union mailbox_module_context module_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(struct imap_msg_map) imap_msg_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int first_unfound_idx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int uidl_synced:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int uidl_sync_failed:1;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen unsigned int uidl_ordered:1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
35ef661bd85c64834e3e34eeeb3c393b81108760Timo Sirainen/* NOTE: these headers must be sorted */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic const char *hdr_hash_skip_headers[] = {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "Content-Length",
35ef661bd85c64834e3e34eeeb3c393b81108760Timo Sirainen "Return-Path", /* Yahoo IMAP has Return-Path, Yahoo POP3 doesn't */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "Status",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "X-IMAP",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "X-IMAPbase",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "X-Keywords",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "X-Message-Flag",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "X-Status",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "X-UID",
c39c3d8089fbdd8eb34646c25167aa4551064cf4Timo Sirainen "X-UIDL",
c39c3d8089fbdd8eb34646c25167aa4551064cf4Timo Sirainen "X-Yahoo-Newman-Property"
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
bd63b5b860658b01b1f46f26d406e1e4a9dc019aTimo Sirainenconst char *pop3_migration_plugin_version = DOVECOT_ABI_VERSION;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen &mail_storage_module_register);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen &mail_module_register);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_msg_map_uid_cmp(const struct imap_msg_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const struct imap_msg_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map1->uid < map2->uid)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map1->uid > map2->uid)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const struct pop3_uidl_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map1->pop3_seq < map2->pop3_seq)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map1->pop3_seq > map2->pop3_seq)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const struct pop3_uidl_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const struct imap_msg_map *map2)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainenstruct pop3_hdr_context {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen bool have_eoh;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen bool stop;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen};
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic void
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenpop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED,
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen struct message_header_line *hdr,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen bool *matched, struct pop3_hdr_context *ctx)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen if (hdr == NULL)
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen return;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen if (hdr->eoh) {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen ctx->have_eoh = TRUE;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen if (ctx->stop) {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen /* matched is handled differently for eoh by
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen istream-header-filter. a design bug I guess.. */
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen *matched = FALSE;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen }
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen } else {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen if (strspn(hdr->name, "\r") == hdr->name_len) {
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen /* CR+CR+LF - some servers stop the header processing
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen here while others don't. To make sure they can be
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen matched correctly we want to stop here entirely. */
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen ctx->stop = TRUE;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen }
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen if (ctx->stop)
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen *matched = TRUE;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen }
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainenint pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen uoff_t hdr_size,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen unsigned char sha1_r[SHA1_RESULTLEN],
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen bool *have_eoh_r)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen{
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen struct istream *input2;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen const unsigned char *data, *p;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen size_t size, idx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct sha1_ctxt sha1_ctx;
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen struct pop3_hdr_context hdr_ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen memset(&hdr_ctx, 0, sizeof(hdr_ctx));
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen input2 = i_stream_create_limit(input, hdr_size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* hide headers that might change or be different in IMAP vs. POP3 */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen input = i_stream_create_header_filter(input2,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen hdr_hash_skip_headers,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen N_ELEMENTS(hdr_hash_skip_headers),
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen pop3_header_filter_callback, &hdr_ctx);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen i_stream_unref(&input2);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen sha1_init(&sha1_ctx);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen while (i_stream_read_data(input, &data, &size, 0) > 0) {
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen /* if there are NULs in header, replace them with 0x80
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen character. This is done by at least Dovecot IMAP and also
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen POP3 with outlook-no-nuls workaround. */
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen while ((p = memchr(data, '\0', size)) != NULL) {
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen idx = p - data;
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen sha1_loop(&sha1_ctx, data, idx);
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen sha1_loop(&sha1_ctx, "\x80", 1);
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen i_assert(size > idx);
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen data += idx + 1;
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen size -= idx + 1;
8c3872c26b18421d62c52cbfe0b81b1d79239f89Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen sha1_loop(&sha1_ctx, data, size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_stream_skip(input, size);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (input->stream_errno != 0) {
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen i_error("pop3_migration: Failed to read header for msg %u: %s",
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen mail_seq, i_stream_get_error(input));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_stream_unref(&input);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen sha1_result(&sha1_ctx, sha1_r);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_stream_unref(&input);
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen *have_eoh_r = hdr_ctx.have_eoh;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenstatic int
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainenget_hdr_sha1(struct mail *mail, unsigned char sha1_r[SHA1_RESULTLEN])
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen{
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen struct istream *input;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen struct message_size hdr_size;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen bool have_eoh;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) {
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen i_error("pop3_migration: Failed to get header for msg %u: %s",
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen mail->seq, mailbox_get_last_error(mail->box, NULL));
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen return -1;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen }
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen if (pop3_migration_get_hdr_sha1(mail->seq, input,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen hdr_size.physical_size,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen sha1_r, &have_eoh) < 0)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen return -1;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen if (have_eoh)
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen return 0;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen /* The empty "end of headers" line is missing. Either this means that
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen the headers ended unexpectedly (which is ok) or that the remote
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen server is buggy. Some servers have problems with
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen 1) header line continuations that contain only whitespace and
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen 2) headers that have no ":". The header gets truncated when such
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen line is reached.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen At least Oracle IMS IMAP FETCH BODY[HEADER] handles 1) by not
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen returning the whitespace line and 2) by returning the line but
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen truncating the rest. POP3 TOP instead returns the entire header.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen This causes the IMAP and POP3 hashes not to match.
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen If there's LF+CR+CR+LF in the middle of headers, Courier IMAP's
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen FETCH BODY[HEADER] stops after that, but Courier POP3's TOP doesn't.
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen So we'll try to avoid this by falling back to full FETCH BODY[]
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen (and/or RETR) and we'll parse the header ourself from it. This
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen should work around any similar bugs in all IMAP/POP3 servers. */
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen if (mail_get_stream(mail, &hdr_size, NULL, &input) < 0) {
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen i_error("pop3_migration: Failed to get body for msg %u: %s",
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen mail->seq, mailbox_get_last_error(mail->box, NULL));
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen return -1;
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen }
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen return pop3_migration_get_hdr_sha1(mail->seq, input,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen hdr_size.physical_size,
e18e90938ffd9e31c796c405404be0b7dcd5c807Timo Sirainen sha1_r, &have_eoh);
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen}
31fd39a3a3d544b1a8afb9aef07f180d0d40fda2Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic struct mailbox *pop3_mailbox_alloc(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_namespace *ns;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ns = mail_namespace_find(storage->user->namespaces,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mstorage->pop3_box_vname);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen i_assert(ns != NULL);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen return mailbox_alloc(ns->list, mstorage->pop3_box_vname,
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen MAILBOX_FLAG_READONLY | MAILBOX_FLAG_POP3_SESSION);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int pop3_map_read(struct mail_storage *storage, struct mailbox *pop3_box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_uidl_map *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *uidl;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen uoff_t size = (uoff_t)-1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (array_is_created(&mstorage->pop3_uidl_map)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* already read these, just reset the imap_uids */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen map->imap_uid = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_array_init(&mstorage->pop3_uidl_map, 128);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (mailbox_sync(pop3_box, 0) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_error("pop3_migration: Couldn't sync mailbox %s: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen t = mailbox_transaction_begin(pop3_box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_search_build_add_all(search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen mstorage->skip_size_check ? 0 :
b3070aca2aa0dbdcc7f30e6e2bb1f0455127a343Timo Sirainen MAIL_FETCH_PHYSICAL_SIZE, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen while (mailbox_search_next(ctx, &mail)) {
7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen /* get the size with LIST instead of RETR */
7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
7e8bfb5b0af9606f131fc440e61f3752da335ac9Timo Sirainen
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen if (mstorage->skip_size_check)
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen ;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen else if (mail_get_physical_size(mail, &size) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_error("pop3_migration: Failed to get size for msg %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail->seq,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail->seq,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (*uidl == '\0') {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_warning("pop3_migration: UIDL for msg %u is empty",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail->seq);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map = array_append_space(&mstorage->pop3_uidl_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->pop3_seq = mail->seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->pop3_uidl = p_strdup(storage->pool, uidl);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->size = size;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (mailbox_search_deinit(&ctx) < 0) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen i_error("pop3_migration: Failed to search all POP3 mails: %s",
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenpop3_map_read_hdr_hashes(struct mail_storage *storage, struct mailbox *pop3_box,
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen unsigned first_seq)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_uidl_map *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (mstorage->pop3_all_hdr_sha1_set)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (mstorage->all_mailboxes) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* we may be matching against multiple mailboxes.
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen read all the hashes only once. */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen first_seq = 1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen t = mailbox_transaction_begin(pop3_box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_search_build_add_seqset(search_args, first_seq,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_count(&mstorage->pop3_uidl_map)+1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen MAIL_FETCH_STREAM_HEADER, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen while (mailbox_search_next(ctx, &mail)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map = array_idx_modifiable(&mstorage->pop3_uidl_map,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail->seq-1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (get_hdr_sha1(mail, map->hdr_sha1) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen else
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->hdr_sha1_set = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (mailbox_search_deinit(&ctx) < 0) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen i_error("pop3_migration: Failed to search all POP3 mail hashes: %s",
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen mailbox_get_last_error(pop3_box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (ret == 0 && first_seq == 1)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mstorage->pop3_all_hdr_sha1_set = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_map_read(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen struct pop3_migration_mail_storage *mstorage =
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen POP3_MIGRATION_CONTEXT(box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mailbox_status status;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct imap_msg_map *map;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen uoff_t psize = (uoff_t)-1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mailbox_get_open_status(box, STATUS_MESSAGES, &status);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_assert(!array_is_created(&mbox->imap_msg_map));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen t = mailbox_transaction_begin(box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_search_build_add_all(search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen mstorage->skip_size_check ? 0 :
c8c4bbf6b1415e9d0845bc8f1cd6d19b76ab0392Timo Sirainen MAIL_FETCH_PHYSICAL_SIZE, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen while (mailbox_search_next(ctx, &mail)) {
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen if (mstorage->skip_size_check)
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen ;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen else if (mail_get_physical_size(mail, &psize) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail->uid,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mailbox_get_last_error(box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen break;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map = array_append_space(&mbox->imap_msg_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->uid = mail->uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->psize = psize;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (mailbox_search_deinit(&ctx) < 0) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen i_error("pop3_migration: Failed to search all IMAP mails: %s",
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen mailbox_get_last_error(box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int imap_map_read_hdr_hashes(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mailbox_transaction_context *t;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_search_args *search_args;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_search_context *ctx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail *mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct imap_msg_map *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen int ret = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen t = mailbox_transaction_begin(box, 0);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen search_args = mail_search_build_init();
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_search_build_add_seqset(search_args, mbox->first_unfound_idx+1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_count(&mbox->imap_msg_map)+1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen MAIL_FETCH_STREAM_HEADER, NULL);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_search_args_unref(&search_args);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen while (mailbox_search_next(ctx, &mail)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map = array_idx_modifiable(&mbox->imap_msg_map, mail->seq-1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (get_hdr_sha1(mail, map->hdr_sha1) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen else
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map->hdr_sha1_set = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (mailbox_search_deinit(&ctx) < 0) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen i_error("pop3_migration: Failed to search all IMAP mail hashes: %s",
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen mailbox_get_last_error(box, NULL));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = -1;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (void)mailbox_transaction_commit(&t);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic bool pop3_uidl_assign_by_size(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen POP3_MIGRATION_CONTEXT(box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_uidl_map *pop3_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct imap_msg_map *imap_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int i, pop3_count, imap_count, count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen count = I_MIN(pop3_count, imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* see if we can match the messages using sizes */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen for (i = 0; i < count; i++) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (pop3_map[i].size != imap_map[i].psize)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* two messages with same size, don't trust them */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen pop3_map[i].imap_uid = imap_map[i].uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen imap_map[i].pop3_seq = pop3_map[i].pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mbox->first_unfound_idx = i;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (box->storage->user->mail_debug)
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen i_debug("pop3_migration: %u/%u mails matched by size", i, count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return i == count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenstatic int
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainenpop3_uidl_assign_by_hdr_hash(struct mailbox *box, struct mailbox *pop3_box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen POP3_MIGRATION_CONTEXT(box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_uidl_map *pop3_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct imap_msg_map *imap_map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int pop3_idx, imap_idx, pop3_count, imap_count;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen unsigned int first_seq, missing_uids_count;
7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen uint32_t first_missing_idx = (uint32_t)-1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen int ret;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen first_seq = mbox->first_unfound_idx+1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen if (pop3_map_read_hdr_hashes(box->storage, pop3_box, first_seq) < 0 ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_map_read_hdr_hashes(box) < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_hdr_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_sort(&mbox->imap_msg_map, imap_msg_map_hdr_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_idx = imap_idx = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen while (pop3_idx < pop3_count && imap_idx < imap_count) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!pop3_map[pop3_idx].hdr_sha1_set ||
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen pop3_map[pop3_idx].imap_uid != 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!imap_map[imap_idx].hdr_sha1_set ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_map[imap_idx].pop3_uidl != NULL) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen continue;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ret = memcmp(pop3_map[pop3_idx].hdr_sha1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_map[imap_idx].hdr_sha1,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen sizeof(pop3_map[pop3_idx].hdr_sha1));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (ret < 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen else if (ret > 0)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_idx++;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen else {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_map[imap_idx].pop3_uidl =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_map[pop3_idx].pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen imap_map[imap_idx].pop3_seq =
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen pop3_map[pop3_idx].pop3_seq;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen missing_uids_count = 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen for (pop3_idx = 0; pop3_idx < pop3_count; pop3_idx++) {
7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen if (pop3_map[pop3_idx].imap_uid == 0) {
7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen if (first_missing_idx == (uint32_t)-1)
12e32734b3fe762c647c1ae5552185517a988cf8Timo Sirainen first_missing_idx = pop3_idx;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen missing_uids_count++;
7faf475a2faea9ac291898a6593870b01fbc30d4Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (missing_uids_count > 0 && !mstorage->all_mailboxes) {
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen string_t *str = t_str_new(128);
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen str_printfa(str, "pop3_migration: %u POP3 messages have no "
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen "matching IMAP messages (first POP3 msg %u UIDL %s)",
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen missing_uids_count,
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen pop3_map[first_missing_idx].pop3_seq,
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen pop3_map[first_missing_idx].pop3_uidl);
e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen if (imap_count + missing_uids_count == pop3_count) {
e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen str_append(str, " - all IMAP messages were found "
e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen "(POP3 contains more than IMAP INBOX - you may want to set pop3_migration_all_mailboxes=yes)");
e9d659ad49a3cf2190606a62289c86347608bffaTimo Sirainen }
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen if (!mstorage->ignore_missing_uidls) {
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen i_error("%s - set pop3_migration_ignore_missing_uidls=yes to continue anyway",
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen str_c(str));
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen return -1;
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen }
af99ca825f4b7674ec6dd0269bbca665775205aaTimo Sirainen i_warning("%s", str_c(str));
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen } else if (box->storage->user->mail_debug) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen i_debug("pop3_migration: %u mails matched by headers", pop3_count);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int pop3_migration_uidl_sync(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen struct pop3_migration_mail_storage *mstorage =
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen POP3_MIGRATION_CONTEXT(box->storage);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen struct mailbox *pop3_box;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen const struct pop3_uidl_map *pop3_map;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen unsigned int i, count;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen uint32_t prev_uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (mbox->uidl_synced)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen pop3_box = pop3_mailbox_alloc(box->storage);
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen /* the POP3 server isn't connected to yet. handle all IMAP traffic
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen first before connecting, so POP3 server won't disconnect us due to
0e7d5ff38f28d8c85e197a031bbb66b322ff89e6Timo Sirainen idling. */
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen if (imap_map_read(box) < 0 ||
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen pop3_map_read(box->storage, pop3_box) < 0) {
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (!pop3_uidl_assign_by_size(box)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* everything wasn't assigned, figure out the rest with
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen header hashes */
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen if (pop3_uidl_assign_by_hdr_hash(box, pop3_box) < 0) {
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* see if the POP3 UIDL order is the same as IMAP UID order */
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen mbox->uidl_ordered = TRUE;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen pop3_map = array_get(&mstorage->pop3_uidl_map, &count);
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen prev_uid = 0;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen for (i = 0; i < count; i++) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (pop3_map[i].imap_uid == 0)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen continue;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (prev_uid > pop3_map[i].imap_uid) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen mbox->uidl_ordered = FALSE;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen break;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen prev_uid = pop3_map[i].imap_uid;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mbox->uidl_synced = TRUE;
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen mailbox_free(&pop3_box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic int
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenpop3_migration_get_special(struct mail *_mail, enum mail_fetch_field field,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char **value_r)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen union mail_module_context *mmail = POP3_MIGRATION_MAIL_CONTEXT(mail);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(_mail->box);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct imap_msg_map map_key, *map;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (field == MAIL_FETCH_UIDL_BACKEND ||
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen field == MAIL_FETCH_POP3_ORDER) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (mbox->uidl_sync_failed ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_migration_uidl_sync(_mail->box) < 0) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mbox->uidl_sync_failed = TRUE;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_storage_set_error(_mail->box->storage,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen MAIL_ERROR_TEMP,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "POP3 UIDLs couldn't be synced");
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return -1;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen memset(&map_key, 0, sizeof(map_key));
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map_key.uid = _mail->uid;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen map = array_bsearch(&mbox->imap_msg_map, &map_key,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen imap_msg_map_uid_cmp);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (map != NULL && map->pop3_uidl != NULL) {
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen if (field == MAIL_FETCH_UIDL_BACKEND)
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen *value_r = map->pop3_uidl;
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen else
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen *value_r = t_strdup_printf("%u", map->pop3_seq);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return 0;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
b10c3f9ed997748fdbb03b9daadc8c31eed02120Timo Sirainen /* not found from POP3 server, fallback to default */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return mmail->super.get_special(_mail, field, value_r);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_allocated(struct mail *_mail)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen POP3_MIGRATION_CONTEXT(_mail->box->storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_vfuncs *v = mail->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen union mail_module_context *mmail;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_namespace *ns;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (mstorage == NULL ||
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen (!mstorage->all_mailboxes && !_mail->box->inbox_user)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* assigns UIDLs only for INBOX */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen ns = mail_namespace_find(_mail->box->storage->user->namespaces,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mstorage->pop3_box_vname);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (ns == mailbox_get_namespace(_mail->box)) {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen /* we're accessing the pop3-migration namespace itself */
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mmail = p_new(mail->pool, union mail_module_context, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mmail->super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail->vlast = &mmail->super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen v->get_special = pop3_migration_get_special;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen MODULE_CONTEXT_SET_SELF(mail, pop3_migration_mail_module, mmail);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mailbox_allocated(struct mailbox *box)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mailbox_vfuncs *v = box->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mailbox *mbox;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mbox = p_new(box->pool, struct pop3_migration_mailbox, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mbox->module_ctx.super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen box->vlast = &mbox->module_ctx.super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen MODULE_CONTEXT_SET(box, pop3_migration_storage_module, mbox);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_storage_destroy(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mail_storage *mstorage =
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen POP3_MIGRATION_CONTEXT(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen if (array_is_created(&mstorage->pop3_uidl_map))
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen array_free(&mstorage->pop3_uidl_map);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c6afd726060aae56b6622c6c52aec10231c4bf1cTimo Sirainen mstorage->module_ctx.super.destroy(storage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic void pop3_migration_mail_storage_created(struct mail_storage *storage)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct pop3_migration_mail_storage *mstorage;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen struct mail_storage_vfuncs *v = storage->vlast;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen const char *pop3_box_vname;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen pop3_box_vname = mail_user_plugin_getenv(storage->user,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen "pop3_migration_mailbox");
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (pop3_box_vname == NULL) {
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen if (storage->user->mail_debug)
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen i_debug("pop3_migration: No pop3_migration_mailbox setting - disabled");
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen return;
ba57ea2c696f9e9aae909f073069848876a641f4Timo Sirainen }
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mstorage = p_new(storage->pool, struct pop3_migration_mail_storage, 1);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mstorage->module_ctx.super = *v;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen storage->vlast = &mstorage->module_ctx.super;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen v->destroy = pop3_migration_mail_storage_destroy;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mstorage->pop3_box_vname = p_strdup(storage->pool, pop3_box_vname);
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen mstorage->all_mailboxes =
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen mail_user_plugin_getenv(storage->user,
1c3dc4c08ced3948f52c3c6c171ed77310b2cbfdTimo Sirainen "pop3_migration_all_mailboxes") != NULL;
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen mstorage->ignore_missing_uidls =
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen mail_user_plugin_getenv(storage->user,
2200adee458ca662d32b5ec0e01d8c5cba0cc0a8Timo Sirainen "pop3_migration_ignore_missing_uidls") != NULL;
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen mstorage->skip_size_check =
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen mail_user_plugin_getenv(storage->user,
b215322367dbd94df3e2e4bb643b53460e6adc51Timo Sirainen "pop3_migration_skip_size_check") != NULL;
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenstatic struct mail_storage_hooks pop3_migration_mail_storage_hooks = {
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen .mail_allocated = pop3_migration_mail_allocated,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen .mailbox_allocated = pop3_migration_mailbox_allocated,
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen .mail_storage_created = pop3_migration_mail_storage_created
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen};
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid pop3_migration_plugin_init(struct module *module)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_storage_hooks_add(module, &pop3_migration_mail_storage_hooks);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainenvoid pop3_migration_plugin_deinit(void)
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen{
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen mail_storage_hooks_remove(&pop3_migration_mail_storage_hooks);
c8593b070319d0ff83f8d6c4b5ed5abf2d578a06Timo Sirainen}