pop3-migration-plugin.c revision b215322367dbd94df3e2e4bb643b53460e6adc51
45312f52ff3a3d4c137447be4c7556500c2f8bf2Timo Sirainen/* Copyright (c) 2007-2016 Dovecot authors, see the included COPYING file */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen MODULE_CONTEXT(obj, pop3_migration_storage_module)
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen MODULE_CONTEXT(obj, pop3_migration_mail_module)
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen /* LIST size */
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen /* sha1(TOP 0) - set only when needed */
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen /* sha1(header) - set only when needed */
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen/* NOTE: these headers must be sorted */
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainenstatic const char *hdr_hash_skip_headers[] = {
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen "Content-Length",
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen "Return-Path", /* Yahoo IMAP has Return-Path, Yahoo POP3 doesn't */
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen "X-IMAPbase",
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen "X-Keywords",
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen "X-Message-Flag",
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainenconst char *pop3_migration_plugin_version = DOVECOT_ABI_VERSION;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module,
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic int imap_msg_map_uid_cmp(const struct imap_msg_map *map1,
4d2211dac61c615c5bdfd501ea54d46c89d41b0fTimo Sirainenstatic int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
f70a4526fa8eaf5faafca01800faf590a4eaedb0Timo Sirainenstatic int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainenstatic int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1,
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenpop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED,
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen /* matched is handled differently for eoh by
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen istream-header-filter. a design bug I guess.. */
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen if (strspn(hdr->name, "\r") == hdr->name_len) {
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen /* CR+CR+LF - some servers stop the header processing
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen here while others don't. To make sure they can be
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen matched correctly we want to stop here entirely. */
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainenint pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input,
dcd50ecbfe796bd76f2d63483c534cc0e4e94164Timo Sirainen const unsigned char *data, *p;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen input2 = i_stream_create_limit(input, hdr_size);
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen /* hide headers that might change or be different in IMAP vs. POP3 */
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen while (i_stream_read_data(input, &data, &size, 0) > 0) {
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen /* if there are NULs in header, replace them with 0x80
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen character. This is done by at least Dovecot IMAP and also
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen POP3 with outlook-no-nuls workaround. */
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen while ((p = memchr(data, '\0', size)) != NULL) {
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainen i_error("pop3_migration: Failed to read header for msg %u: %s",
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainenget_hdr_sha1(struct mail *mail, unsigned char sha1_r[SHA1_RESULTLEN])
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) {
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen i_error("pop3_migration: Failed to get header for msg %u: %s",
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen mail->seq, mailbox_get_last_error(mail->box, NULL));
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen if (pop3_migration_get_hdr_sha1(mail->seq, input,
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen /* The empty "end of headers" line is missing. Either this means that
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen the headers ended unexpectedly (which is ok) or that the remote
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen server is buggy. Some servers have problems with
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen 1) header line continuations that contain only whitespace and
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen 2) headers that have no ":". The header gets truncated when such
b87436ebb957a9eb182be72ba00e2c8eae59a2e4Timo Sirainen line is reached.
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen At least Oracle IMS IMAP FETCH BODY[HEADER] handles 1) by not
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen returning the whitespace line and 2) by returning the line but
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen truncating the rest. POP3 TOP instead returns the entire header.
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen This causes the IMAP and POP3 hashes not to match.
4981827cb5e32cf767b7b0e3070137e6b36f42afTimo Sirainen If there's LF+CR+CR+LF in the middle of headers, Courier IMAP's
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen FETCH BODY[HEADER] stops after that, but Courier POP3's TOP doesn't.
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen So we'll try to avoid this by falling back to full FETCH BODY[]
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainen (and/or RETR) and we'll parse the header ourself from it. This
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen should work around any similar bugs in all IMAP/POP3 servers. */
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen if (mail_get_stream(mail, &hdr_size, NULL, &input) < 0) {
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainen i_error("pop3_migration: Failed to get body for msg %u: %s",
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainen mail->seq, mailbox_get_last_error(mail->box, NULL));
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainen return pop3_migration_get_hdr_sha1(mail->seq, input,
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainenstatic struct mailbox *pop3_mailbox_alloc(struct mail_storage *storage)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen struct pop3_migration_mail_storage *mstorage =
cf942dce0253075911a96cff323b5f30eb654ae0Timo Sirainen ns = mail_namespace_find(storage->user->namespaces,
3091db1d26fed038841a3a81dd5ff507d0b375ceTimo Sirainen return mailbox_alloc(ns->list, mstorage->pop3_box_vname,
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen MAILBOX_FLAG_READONLY | MAILBOX_FLAG_POP3_SESSION);
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainenstatic int pop3_map_read(struct mail_storage *storage, struct mailbox *pop3_box)
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen struct pop3_migration_mail_storage *mstorage =
3091db1d26fed038841a3a81dd5ff507d0b375ceTimo Sirainen if (array_is_created(&mstorage->pop3_uidl_map)) {
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen /* already read these, just reset the imap_uids */
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen i_error("pop3_migration: Couldn't sync mailbox %s: %s",
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
3091db1d26fed038841a3a81dd5ff507d0b375ceTimo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
3091db1d26fed038841a3a81dd5ff507d0b375ceTimo Sirainen /* get the size with LIST instead of RETR */
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen else if (mail_get_physical_size(mail, &size) < 0) {
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen i_error("pop3_migration: Failed to get size for msg %u: %s",
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen i_warning("pop3_migration: UIDL for msg %u is empty",
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen map = array_append_space(&mstorage->pop3_uidl_map);
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen map->pop3_uidl = p_strdup(storage->pool, uidl);
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen i_error("pop3_migration: Failed to search all POP3 mails: %s",
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainenpop3_map_read_hdr_hashes(struct mail_storage *storage, struct mailbox *pop3_box,
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen struct pop3_migration_mail_storage *mstorage =
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen /* we may be matching against multiple mailboxes.
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen read all the hashes only once. */
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen mail_search_build_add_seqset(search_args, first_seq,
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen map = array_idx_modifiable(&mstorage->pop3_uidl_map,
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen i_error("pop3_migration: Failed to search all POP3 mail hashes: %s",
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen struct pop3_migration_mail_storage *mstorage =
75bf6f49f6b8ee403f26a609d5c0c726a3262c54Timo Sirainen mailbox_get_open_status(box, STATUS_MESSAGES, &status);
75bf6f49f6b8ee403f26a609d5c0c726a3262c54Timo Sirainen i_assert(!array_is_created(&mbox->imap_msg_map));
75bf6f49f6b8ee403f26a609d5c0c726a3262c54Timo Sirainen p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
370c4117f854357c9ea86495fb3b4e8769b6db07Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen else if (mail_get_physical_size(mail, &psize) < 0) {
75bf6f49f6b8ee403f26a609d5c0c726a3262c54Timo Sirainen i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
75bf6f49f6b8ee403f26a609d5c0c726a3262c54Timo Sirainen map = array_append_space(&mbox->imap_msg_map);
6e22333f2cc2562d9bebd93803f70e7ac987f66aTimo Sirainen i_error("pop3_migration: Failed to search all IMAP mails: %s",
6e22333f2cc2562d9bebd93803f70e7ac987f66aTimo Sirainenstatic int imap_map_read_hdr_hashes(struct mailbox *box)
6e22333f2cc2562d9bebd93803f70e7ac987f66aTimo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
6e22333f2cc2562d9bebd93803f70e7ac987f66aTimo Sirainen mail_search_build_add_seqset(search_args, mbox->first_unfound_idx+1,
6e22333f2cc2562d9bebd93803f70e7ac987f66aTimo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
6e22333f2cc2562d9bebd93803f70e7ac987f66aTimo Sirainen map = array_idx_modifiable(&mbox->imap_msg_map, mail->seq-1);
75bf6f49f6b8ee403f26a609d5c0c726a3262c54Timo Sirainen i_error("pop3_migration: Failed to search all IMAP mail hashes: %s",
3ccfcf0856958cb9208a9fc51c3bdf13c58ad52aTimo Sirainenstatic bool pop3_uidl_assign_by_size(struct mailbox *box)
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
bd7b1a9000b12349e2a99bb43b3ce8b96a18e92bTimo Sirainen struct pop3_migration_mail_storage *mstorage =
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen unsigned int i, pop3_count, imap_count, count;
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen /* see if we can match the messages using sizes */
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen for (i = 0; i < count; i++) {
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen /* two messages with same size, don't trust them */
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen i_debug("pop3_migration: %u/%u mails matched by size", i, count);
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainenpop3_uidl_assign_by_hdr_hash(struct mailbox *box, struct mailbox *pop3_box)
d23c747de9d33966483fbdd41f08ad7766da7c5cTimo Sirainen struct pop3_migration_mail_storage *mstorage =
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen unsigned int pop3_idx, imap_idx, pop3_count, imap_count;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen if (pop3_map_read_hdr_hashes(box->storage, pop3_box, first_seq) < 0 ||
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_hdr_cmp);
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen array_sort(&mbox->imap_msg_map, imap_msg_map_hdr_cmp);
29e82a14501731bef8c41a1b27fc415d634fa01dTimo Sirainen pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
a443e5aaf632257bfd1e7aa9b3c42c09512bbe43Timo Sirainen imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
82995cc154a929f37aa486a72a6485e9f8d34a30Timo Sirainen while (pop3_idx < pop3_count && imap_idx < imap_count) {
e726bf74fcc8d24f4c9d0d83217b3db4314d9d1fTimo Sirainen else if (ret > 0)
missing_uids_count = 0;
unsigned int i, count;
prev_uid = 0;
for (i = 0; i < count; i++) {
const char **value_r)
const char *pop3_box_vname;
void pop3_migration_plugin_deinit(void)