pop3-migration-plugin.c revision 0e7d5ff38f28d8c85e197a031bbb66b322ff89e6
e59faf65ce864fe95dc00f5d52b8323cdbd0608aTimo Sirainen/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen MODULE_CONTEXT(obj, pop3_migration_storage_module)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen MODULE_CONTEXT(obj, pop3_migration_mail_module)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* LIST size */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* sha1(TOP 0) - set only when needed */
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen /* sha1(header) - set only when needed */
17fe695b985e9d6e9dc39c05b24e6b3c3b7e1ba1Timo Sirainen ARRAY_DEFINE(pop3_uidl_map, struct pop3_uidl_map);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen ARRAY_DEFINE(imap_msg_map, struct imap_msg_map);
f46363f428d8f2784146d36692b21936a48a7006Timo Sirainenstatic const char *hdr_hash_skip_headers[] = {
b437874782ad048daa155e0ac863c2326c3f5e43Timo Sirainen "Content-Length",
b437874782ad048daa155e0ac863c2326c3f5e43Timo Sirainen "X-IMAPbase",
b437874782ad048daa155e0ac863c2326c3f5e43Timo Sirainen "X-Keywords",
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen "X-Message-Flag",
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenconst char *pop3_migration_plugin_version = DOVECOT_VERSION;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenstatic int imap_msg_map_uid_cmp(const struct imap_msg_map *map1,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
0c22bef8f5b35c645de8affd8746307fc53bd222Timo Sirainen return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainenstatic int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1,
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1));
8451c4b5afc1ff5366438b2766f75b592c33e1ecTimo Sirainenstatic int get_hdr_sha1(struct mail *mail, unsigned char sha1[SHA1_RESULTLEN])
8451c4b5afc1ff5366438b2766f75b592c33e1ecTimo Sirainen const unsigned char *data;
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen i_error("pop3_migration: Failed to get header for msg %u: %s",
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail->seq, mailbox_get_last_error(mail->box, NULL));
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen input2 = i_stream_create_limit(input, hdr_size.physical_size);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* hide headers that might change or be different in IMAP vs. POP3 */
cd2ed64888b42b481cde6bb9548c8520516fa3e9Timo Sirainen while (i_stream_read_data(input, &data, &size, 0) > 0) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen i_error("pop3_migration: Failed to read header for msg %u: %m",
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int pop3_mailbox_open(struct mail_storage *storage)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct pop3_migration_mail_storage *mstorage =
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen ns = mail_namespace_find(storage->user->namespaces,
78fa3c578c14ee8a612f86cf73b6181c7f16463fTimo Sirainen i_error("pop3_migration: Namespace not found for mailbox %s",
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen mstorage->pop3_box = mailbox_alloc(ns->list, mstorage->pop3_box_vname,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int pop3_map_read(struct mail_storage *storage)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct pop3_migration_mail_storage *mstorage =
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct mailbox *pop3_box = mstorage->pop3_box;
578ef2538ccf42e2a48234c24a8b709397101d88Timo Sirainen if (array_is_created(&mstorage->pop3_uidl_map)) {
578ef2538ccf42e2a48234c24a8b709397101d88Timo Sirainen /* already read these, just reset the imap_uids */
578ef2538ccf42e2a48234c24a8b709397101d88Timo Sirainen array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen i_error("pop3_migration: Couldn't sync mailbox %s: %s",
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
6d2b3ce2c6ef62334985ece4f0ab8b154e0e9560Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen i_error("pop3_migration: Failed to get size for msg %u: %s",
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
deb06d37292d9112d74bdf80cfebb92ab5151679Timo Sirainen i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
deb06d37292d9112d74bdf80cfebb92ab5151679Timo Sirainen i_warning("pop3_migration: UIDL for msg %u is empty",
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen map = array_append_space(&mstorage->pop3_uidl_map);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen map->pop3_uidl = p_strdup(storage->pool, uidl);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int pop3_map_read_hdr_hashes(struct mail_storage *storage,
deb06d37292d9112d74bdf80cfebb92ab5151679Timo Sirainen struct pop3_migration_mail_storage *mstorage =
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* we may be matching against multiple mailboxes.
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen read all the hashes only once. */
3f603ef00e35fca21605afa0ad8d76e94fee2b96Timo Sirainen t = mailbox_transaction_begin(mstorage->pop3_box, 0);
3f603ef00e35fca21605afa0ad8d76e94fee2b96Timo Sirainen mail_search_build_add_seqset(search_args, first_seq,
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen map = array_idx_modifiable(&mstorage->pop3_uidl_map,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
8d7eb4104707c60ca7e9d0228b37c5133476907bTimo Sirainen mailbox_get_open_status(box, STATUS_MESSAGES, &status);
8d7eb4104707c60ca7e9d0228b37c5133476907bTimo Sirainen i_assert(!array_is_created(&mbox->imap_msg_map));
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen if (mail_get_physical_size(mail, &psize) < 0) {
8d7eb4104707c60ca7e9d0228b37c5133476907bTimo Sirainen i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map = array_append_space(&mbox->imap_msg_map);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int imap_map_read_hdr_hashes(struct mailbox *box)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen mail_search_build_add_seqset(search_args, mbox->first_unfound_idx+1,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen ctx = mailbox_search_init(t, search_args, NULL,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen map = array_idx_modifiable(&mbox->imap_msg_map, mail->seq-1);
9261dbf0675204898c6557591c7aa376e23a52b2Timo Sirainenstatic bool pop3_uidl_assign_by_size(struct mailbox *box)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen struct pop3_migration_mail_storage *mstorage =
788f275469ad9ed530e440d6690d0e4381a323b2Timo Sirainen unsigned int i, pop3_count, imap_count, count;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* see if we can match the messages using sizes */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen for (i = 0; i < count; i++) {
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen /* two messages with same size, don't trust them */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
9240d99920783c56405dda74a1f6c7ff1ebed8e6Timo Sirainenstatic int pop3_uidl_assign_by_hdr_hash(struct mailbox *box)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct pop3_migration_mail_storage *mstorage =
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen unsigned int pop3_idx, imap_idx, pop3_count, imap_count;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (pop3_map_read_hdr_hashes(box->storage, first_seq) < 0 ||
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_hdr_cmp);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen array_sort(&mbox->imap_msg_map, imap_msg_map_hdr_cmp);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
9240d99920783c56405dda74a1f6c7ff1ebed8e6Timo Sirainen while (pop3_idx < pop3_count && imap_idx < imap_count) {
885e1b36da370a674c0fd3b85db53740d7dcbd9bTimo Sirainen else if (ret > 0)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen for (pop3_idx = 0; pop3_idx < pop3_count; pop3_idx++) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (missing_uids_count > 0 && !mstorage->all_mailboxes) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen i_warning("pop3_migration: %u POP3 messages have no "
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen "matching IMAP messages", missing_uids_count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int pop3_migration_uidl_sync(struct mailbox *box)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct pop3_migration_mail_storage *mstorage =
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen unsigned int i, count;
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen /* the POP3 server isn't connected to yet. handle all IMAP traffic
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen first before connecting, so POP3 server won't disconnect us due to
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* everything wasn't assigned, figure out the rest with
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen header hashes */
b2d562f9c7fd13f9a16e9b3bcee904630b80b1feTimo Sirainen /* see if the POP3 UIDL order is the same as IMAP UID order */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen pop3_map = array_get(&mstorage->pop3_uidl_map, &count);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen for (i = 0; i < count; i++) {
bd20ef9d5c639faf470912ab94e6e6627d3eaebaTimo Sirainenpop3_migration_get_special(struct mail *_mail, enum mail_fetch_field field,
275385a2ecc58e41dc7df3ce3cd943caaa58c4d1Timo Sirainen const char **value_r)
bd20ef9d5c639faf470912ab94e6e6627d3eaebaTimo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
275385a2ecc58e41dc7df3ce3cd943caaa58c4d1Timo Sirainen union mail_module_context *mmail = POP3_MIGRATION_MAIL_CONTEXT(mail);
275385a2ecc58e41dc7df3ce3cd943caaa58c4d1Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(_mail->box);
275385a2ecc58e41dc7df3ce3cd943caaa58c4d1Timo Sirainen "POP3 UIDLs couldn't be synced");
275385a2ecc58e41dc7df3ce3cd943caaa58c4d1Timo Sirainen map = array_bsearch(&mbox->imap_msg_map, &map_key,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen *value_r = t_strdup_printf("%u", map->pop3_seq);
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen /* not found from POP3 server, fallback to default */
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen return mmail->super.get_special(_mail, field, value_r);
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainenstatic void pop3_migration_mail_allocated(struct mail *_mail)
1bea4b9c24fbe2b457950c09cf072292a6701cffTimo Sirainen struct pop3_migration_mail_storage *mstorage =
1bea4b9c24fbe2b457950c09cf072292a6701cffTimo Sirainen struct mail_private *mail = (struct mail_private *)_mail;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen (!mstorage->all_mailboxes && !_mail->box->inbox_user)) {
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen /* assigns UIDLs only for INBOX */
db693bf6fcae96d834567f1782257517b7207655Timo Sirainen ns = mail_namespace_find(_mail->box->storage->user->namespaces,
db693bf6fcae96d834567f1782257517b7207655Timo Sirainen if (ns == mailbox_get_namespace(_mail->box)) {
db693bf6fcae96d834567f1782257517b7207655Timo Sirainen /* we're accessing the pop3-migration namespace itself */
2ac0a22865272cb4311a1bd09eb69b475625b3ebTimo Sirainen mmail = p_new(mail->pool, union mail_module_context, 1);
de70a6f77fc3b350eeee4e2a0d29dd07ddde431bTimo Sirainen MODULE_CONTEXT_SET_SELF(mail, pop3_migration_mail_module, mmail);
de70a6f77fc3b350eeee4e2a0d29dd07ddde431bTimo Sirainenstatic void pop3_migration_mailbox_allocated(struct mailbox *box)
885e1b36da370a674c0fd3b85db53740d7dcbd9bTimo Sirainen mbox = p_new(box->pool, struct pop3_migration_mailbox, 1);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen MODULE_CONTEXT_SET(box, pop3_migration_storage_module, mbox);
de70a6f77fc3b350eeee4e2a0d29dd07ddde431bTimo Sirainenstatic void pop3_migration_mail_storage_destroy(struct mail_storage *storage)
de70a6f77fc3b350eeee4e2a0d29dd07ddde431bTimo Sirainen struct pop3_migration_mail_storage *mstorage =
c668292359474a4aa8c608b30a858337fa3fc813Timo Sirainen if (array_is_created(&mstorage->pop3_uidl_map))
de70a6f77fc3b350eeee4e2a0d29dd07ddde431bTimo Sirainen if (mstorage->module_ctx.super.destroy != NULL)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic void pop3_migration_mail_storage_created(struct mail_storage *storage)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen struct mail_storage_vfuncs *v = storage->vlast;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen pop3_box_vname = mail_user_plugin_getenv(storage->user,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen "pop3_migration_mailbox");
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen mstorage = p_new(storage->pool, struct pop3_migration_mail_storage, 1);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen v->destroy = pop3_migration_mail_storage_destroy;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen mstorage->pop3_box_vname = p_strdup(storage->pool, pop3_box_vname);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenstatic struct mail_storage_hooks pop3_migration_mail_storage_hooks = {
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen .mail_allocated = pop3_migration_mail_allocated,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen .mailbox_allocated = pop3_migration_mailbox_allocated,
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen .mail_storage_created = pop3_migration_mail_storage_created
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenvoid pop3_migration_plugin_init(struct module *module)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen mail_storage_hooks_add(module, &pop3_migration_mail_storage_hooks);