pop3-migration-plugin.c revision 65e19f9777ebcc705d8adc33a84735c31baee66a
e59faf65ce864fe95dc00f5d52b8323cdbd0608aTimo Sirainen/* Copyright (c) 2007-2017 Dovecot authors, see the included COPYING file */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "lib.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "array.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "istream.h"
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen#include "istream-header-filter.h"
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen#include "str.h"
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen#include "sha1.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "message-size.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "message-header-hash.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "message-header-parser.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "mail-cache.h"
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen#include "mail-namespace.h"
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen#include "mail-search-build.h"
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen#include "index-storage.h"
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen#include "index-mail.h"
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen#include "pop3-migration-plugin.h"
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
daa7e7459749ae8f82cd3eed9c44522d81c609a3Timo Sirainen#define POP3_MIGRATION_CONTEXT(obj) \
bbadd5331f534017cf62d5183003b3d9fdad079eTimo Sirainen MODULE_CONTEXT(obj, pop3_migration_storage_module)
373492be949e159fda651807b3acda2c5c077027Timo Sirainen#define POP3_MIGRATION_MAIL_CONTEXT(obj) \
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen MODULE_CONTEXT(obj, pop3_migration_mail_module)
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainenstruct msg_map_common {
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen /* sha1(header) - set only when needed */
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen unsigned char hdr_sha1[SHA1_RESULTLEN];
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen bool hdr_sha1_set:1;
373492be949e159fda651807b3acda2c5c077027Timo Sirainen};
bbadd5331f534017cf62d5183003b3d9fdad079eTimo Sirainen
bbadd5331f534017cf62d5183003b3d9fdad079eTimo Sirainenstruct pop3_uidl_map {
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen struct msg_map_common common;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen uint32_t pop3_seq;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen uint32_t imap_uid;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen /* UIDL */
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen const char *pop3_uidl;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen /* LIST size */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen uoff_t size;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen};
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
635df5b4cbcd7b24c825e01d9dd66d3a4274c4c7Timo Sirainenstruct imap_msg_map {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen struct msg_map_common common;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen uint32_t uid, pop3_seq;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen uoff_t psize;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen const char *pop3_uidl;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen};
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenstruct pop3_migration_mail_storage {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen union mail_storage_module_context module_ctx;
99430beb12dfbc6c9c160f08e2102aeab38a589cTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen const char *pop3_box_vname;
93a7d1ee4b518b5c85f9721dc6539e4dab6aae00Timo Sirainen ARRAY(struct pop3_uidl_map) pop3_uidl_map;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen bool all_mailboxes:1;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen bool pop3_all_hdr_sha1_set:1;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen bool ignore_missing_uidls:1;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen bool ignore_extra_uidls:1;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen bool skip_size_check:1;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen bool skip_uidl_cache:1;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen};
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainenstruct pop3_migration_mailbox {
635df5b4cbcd7b24c825e01d9dd66d3a4274c4c7Timo Sirainen union mailbox_module_context module_ctx;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen ARRAY(struct imap_msg_map) imap_msg_map;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen unsigned int first_unfound_idx;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen struct mail_cache_field cache_field;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen bool cache_field_registered:1;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen bool uidl_synced:1;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen bool uidl_sync_failed:1;
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen};
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen/* NOTE: these headers must be sorted */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenstatic const char *hdr_hash_skip_headers[] = {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "Content-Length",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "Return-Path", /* Yahoo IMAP has Return-Path, Yahoo POP3 doesn't */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "Status",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "X-IMAP",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "X-IMAPbase",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "X-Keywords",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "X-Message-Flag",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "X-Status",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "X-UID",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "X-UIDL",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen "X-Yahoo-Newman-Property"
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen};
ba8498efbf886ca8b69fdb20c0ba2f5dba9416e3Timo Sirainenconst char *pop3_migration_plugin_version = DOVECOT_ABI_VERSION;
93a7d1ee4b518b5c85f9721dc6539e4dab6aae00Timo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module,
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen &mail_storage_module_register);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module,
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen &mail_module_register);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenstatic int imap_msg_map_uid_cmp(const struct imap_msg_map *map1,
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen const struct imap_msg_map *map2)
9f0f2de10e4ea0c99052bf4b2bef8179f2536228Timo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (map1->uid < map2->uid)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return -1;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (map1->uid > map2->uid)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return 1;
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen return 0;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenstatic int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
380dbb60ae291cbe39d1f710284562ca9167150bTimo Sirainen const struct pop3_uidl_map *map2)
380dbb60ae291cbe39d1f710284562ca9167150bTimo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (map1->pop3_seq < map2->pop3_seq)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return -1;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (map1->pop3_seq > map2->pop3_seq)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return 1;
1b823b2b7790a1e1b7974fcf11a4c48a28e70f37Timo Sirainen return 0;
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen}
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainenstatic int pop3_uidl_map_uidl_cmp(const struct pop3_uidl_map *map1,
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen const struct pop3_uidl_map *map2)
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen{
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen return strcmp(map1->pop3_uidl, map2->pop3_uidl);
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen}
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenstatic int imap_msg_map_uidl_cmp(const struct imap_msg_map *map1,
1b823b2b7790a1e1b7974fcf11a4c48a28e70f37Timo Sirainen const struct imap_msg_map *map2)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen return null_strcmp(map1->pop3_uidl, map2->pop3_uidl);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenstatic int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen const struct pop3_uidl_map *map2)
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen{
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen return memcmp(map1->common.hdr_sha1, map2->common.hdr_sha1,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen sizeof(map1->common.hdr_sha1));
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenstatic int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen const struct imap_msg_map *map2)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen return memcmp(map1->common.hdr_sha1, map2->common.hdr_sha1,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen sizeof(map1->common.hdr_sha1));
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenstruct pop3_hdr_context {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen bool have_eoh;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen bool stop;
1b823b2b7790a1e1b7974fcf11a4c48a28e70f37Timo Sirainen};
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainenstatic bool header_name_is_valid(const char *name)
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen{
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen unsigned int i;
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen for (i = 0; name[i] != '\0'; i++) {
1b823b2b7790a1e1b7974fcf11a4c48a28e70f37Timo Sirainen if ((uint8_t)name[i] <= 0x20 || name[i] >= 0x7f)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return FALSE;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen return TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainenstatic bool header_value_want_skip(const struct message_header_line *hdr)
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen{
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen for (size_t i = 0; i < hdr->value_len; i++) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (hdr->value[i] != ' ' && hdr->value[i] != '\t')
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return FALSE;
57d2429fae575e96ca276355af675deb66b76d00Timo Sirainen }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen /* "header: \r\n \r\n" - Zimbra's BODY[HEADER] strips this line away. */
401b0787fff2dc986a5321ddb32acb1947ff66b0Timo Sirainen return TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
57d2429fae575e96ca276355af675deb66b76d00Timo Sirainenstatic void
401b0787fff2dc986a5321ddb32acb1947ff66b0Timo Sirainenpop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED,
401b0787fff2dc986a5321ddb32acb1947ff66b0Timo Sirainen struct message_header_line *hdr,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen bool *matched, struct pop3_hdr_context *ctx)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (hdr == NULL)
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen return;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (hdr->eoh) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen ctx->have_eoh = TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (ctx->stop)
61618d4c58080570f689614fec204ae14e90cef2Timo Sirainen *matched = TRUE;
50e20db49f29917fe9adcf1b56b11badf28bd0e4Timo Sirainen } else {
50e20db49f29917fe9adcf1b56b11badf28bd0e4Timo Sirainen if (strspn(hdr->name, "\r") == hdr->name_len) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen /* CR+CR+LF - some servers stop the header processing
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen here while others don't. To make sure they can be
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen matched correctly we want to stop here entirely. */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen ctx->stop = TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen } else if (hdr->continued && header_value_want_skip(hdr)) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen *matched = TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (ctx->stop)
50e20db49f29917fe9adcf1b56b11badf28bd0e4Timo Sirainen *matched = TRUE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen else if (!header_name_is_valid(hdr->name)) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen /* Yahoo IMAP drops headers with invalid names, while
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen Yahoo POP3 preserves them. Drop them all. */
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen *matched = TRUE;
c6f894e1522f7b0b6068c228900914073c145175Timo Sirainen }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen }
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenint pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN],
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen bool *have_eoh_r)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen const unsigned char *data;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen size_t size;
401b0787fff2dc986a5321ddb32acb1947ff66b0Timo Sirainen struct message_header_hash_context hash_ctx;
401b0787fff2dc986a5321ddb32acb1947ff66b0Timo Sirainen struct sha1_ctxt sha1_ctx;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen struct pop3_hdr_context hdr_ctx;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen i_zero(&hdr_ctx);
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen /* hide headers that might change or be different in IMAP vs. POP3 */
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen input = i_stream_create_header_filter(input, HEADER_FILTER_HIDE_BODY |
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen hdr_hash_skip_headers,
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen N_ELEMENTS(hdr_hash_skip_headers),
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen pop3_header_filter_callback, &hdr_ctx);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen sha1_init(&sha1_ctx);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen i_zero(&hash_ctx);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen while (i_stream_read_more(input, &data, &size) > 0) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen message_header_hash_more(&hash_ctx, &hash_method_sha1, &sha1_ctx,
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen MESSAGE_HEADER_HASH_MAX_VERSION,
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen data, size);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen i_stream_skip(input, size);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen if (input->stream_errno != 0) {
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen i_error("pop3_migration: Failed to read header for msg %u: %s",
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen mail_seq, i_stream_get_error(input));
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen i_stream_unref(&input);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen return -1;
c6f894e1522f7b0b6068c228900914073c145175Timo Sirainen }
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen sha1_result(&sha1_ctx, sha1_r);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen i_stream_unref(&input);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen *have_eoh_r = hdr_ctx.have_eoh;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen return 0;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen}
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
401b0787fff2dc986a5321ddb32acb1947ff66b0Timo Sirainenstatic unsigned int get_cache_idx(struct mail *mail)
401b0787fff2dc986a5321ddb32acb1947ff66b0Timo Sirainen{
401b0787fff2dc986a5321ddb32acb1947ff66b0Timo Sirainen struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(mail->box);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen if (mbox->cache_field_registered)
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen return mbox->cache_field.idx;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen mbox->cache_field.name = "pop3-migration.hdr";
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen mbox->cache_field.type = MAIL_CACHE_FIELD_FIXED_SIZE;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen mbox->cache_field.field_size = SHA1_RESULTLEN;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen mail_cache_register_fields(mail->box->cache, &mbox->cache_field, 1);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen mbox->cache_field_registered = TRUE;
383d0e8c24451468d6bea17e4b55d74de744abe6Timo Sirainen return mbox->cache_field.idx;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen}
383d0e8c24451468d6bea17e4b55d74de744abe6Timo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainenstatic int
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenget_hdr_sha1(struct mail *mail, unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN])
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen struct istream *input;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen const char *errstr;
2024157e8de36edd31f5fd72f5ea7364a0955fa7Timo Sirainen enum mail_error error;
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen bool have_eoh;
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen int ret;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen if (mail_get_hdr_stream(mail, NULL, &input) < 0) {
15f526e5ac611b4532568d131fcd0abf664abe41Timo Sirainen errstr = mailbox_get_last_internal_error(mail->box, &error);
15f526e5ac611b4532568d131fcd0abf664abe41Timo Sirainen i_error("pop3_migration: Failed to get header for msg %u: %s",
15f526e5ac611b4532568d131fcd0abf664abe41Timo Sirainen mail->seq, errstr);
15f526e5ac611b4532568d131fcd0abf664abe41Timo Sirainen return error == MAIL_ERROR_EXPUNGED ? 0 : -1;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen }
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen if (pop3_migration_get_hdr_sha1(mail->seq, input, sha1_r, &have_eoh) < 0)
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen return -1;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen if (have_eoh) {
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen struct index_mail *imail = (struct index_mail *)mail;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen
0f62889d833767acf9c2ad010c3269806b4cfae3Timo Sirainen index_mail_cache_add_idx(imail, get_cache_idx(mail),
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen sha1_r, SHA1_RESULTLEN);
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen return 1;
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen }
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen /* The empty "end of headers" line is missing. Either this means that
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen the headers ended unexpectedly (which is ok) or that the remote
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen server is buggy. Some servers have problems with
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen 1) header line continuations that contain only whitespace and
7289c5600711b45f30fe289ab5b0293b51d87041Timo Sirainen 2) headers that have no ":". The header gets truncated when such
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen line is reached.
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen At least Oracle IMS IMAP FETCH BODY[HEADER] handles 1) by not
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen returning the whitespace line and 2) by returning the line but
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen truncating the rest. POP3 TOP instead returns the entire header.
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen This causes the IMAP and POP3 hashes not to match.
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen If there's LF+CR+CR+LF in the middle of headers, Courier IMAP's
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen FETCH BODY[HEADER] stops after that, but Courier POP3's TOP doesn't.
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen So we'll try to avoid this by falling back to full FETCH BODY[]
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen (and/or RETR) and we'll parse the header ourself from it. This
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen should work around any similar bugs in all IMAP/POP3 servers. */
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen if (mail_get_stream_because(mail, NULL, NULL, "pop3-migration", &input) < 0) {
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen errstr = mailbox_get_last_internal_error(mail->box, &error);
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen i_error("pop3_migration: Failed to get body for msg %u: %s",
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen mail->seq, errstr);
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen return error == MAIL_ERROR_EXPUNGED ? 0 : -1;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen }
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen ret = pop3_migration_get_hdr_sha1(mail->seq, input, sha1_r, &have_eoh);
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen if (ret == 0) {
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen if (!have_eoh)
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen i_warning("pop3_migration: Truncated email with UID %u stored as truncated", mail->uid);
272aca0a772140d3a45a425a3fd67854ae2ccec2Timo Sirainen struct index_mail *imail = (struct index_mail *)mail;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen index_mail_cache_add_idx(imail, get_cache_idx(mail),
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen sha1_r, SHA1_RESULTLEN);
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen return 1;
9dd1c256910f1fb42823116a641e7edb3ad11970Timo Sirainen } else {
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen return -1;
d477acb83e14a776ece4ca94dcd1869e75d0c6eeTimo Sirainen }
1d22eaac93de41319918a1fc6de42bb302e25c1aTimo Sirainen}
1d22eaac93de41319918a1fc6de42bb302e25c1aTimo Sirainen
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainenstatic bool
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainenget_cached_hdr_sha1(struct mail *mail, buffer_t *cache_buf,
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN])
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct index_mail *imail = (struct index_mail *)mail;
2024157e8de36edd31f5fd72f5ea7364a0955fa7Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen buffer_set_used_size(cache_buf, 0);
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen if (index_mail_cache_lookup_field(imail, cache_buf,
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen get_cache_idx(mail)) > 0 &&
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen cache_buf->used == SHA1_RESULTLEN) {
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen memcpy(sha1_r, cache_buf->data, cache_buf->used);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return TRUE;
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen }
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen return FALSE;
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen}
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainen
9ed2951bd0bb1878a27437d7c00611b2baadd614Timo Sirainenstatic struct mailbox *pop3_mailbox_alloc(struct mail_storage *storage)
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct pop3_migration_mail_storage *mstorage =
b215a8a123623782554a83f3025ef4e771bd8f01Timo Sirainen POP3_MIGRATION_CONTEXT(storage);
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen struct mail_namespace *ns;
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen struct mailbox *box;
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen ns = mail_namespace_find(storage->user->namespaces,
5d60e31c7b701b606067a20bc88dcc8a6de7bbd6Timo Sirainen mstorage->pop3_box_vname);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen i_assert(ns != NULL);
box = mailbox_alloc(ns->list, mstorage->pop3_box_vname,
MAILBOX_FLAG_READONLY | MAILBOX_FLAG_POP3_SESSION);
mailbox_set_reason(box, "pop3_migration");
return box;
}
static int pop3_map_read(struct mail_storage *storage, struct mailbox *pop3_box)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct pop3_uidl_map *map;
const char *uidl;
uoff_t size = (uoff_t)-1;
int ret = 0;
if (array_is_created(&mstorage->pop3_uidl_map)) {
/* already read these, just reset the imap_uids */
array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
map->imap_uid = 0;
return 0;
}
i_array_init(&mstorage->pop3_uidl_map, 128);
if (mailbox_sync(pop3_box, 0) < 0) {
i_error("pop3_migration: Couldn't sync mailbox %s: %s",
pop3_box->vname, mailbox_get_last_internal_error(pop3_box, NULL));
return -1;
}
t = mailbox_transaction_begin(pop3_box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(t, search_args, NULL,
mstorage->skip_size_check ? 0 :
MAIL_FETCH_PHYSICAL_SIZE, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
/* get the size with LIST instead of RETR */
mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
if (mstorage->skip_size_check)
;
else if (mail_get_physical_size(mail, &size) < 0) {
i_error("pop3_migration: Failed to get size for msg %u: %s",
mail->seq,
mailbox_get_last_internal_error(pop3_box, NULL));
ret = -1;
break;
}
mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
mail->seq,
mailbox_get_last_internal_error(pop3_box, NULL));
ret = -1;
break;
}
if (*uidl == '\0') {
i_warning("pop3_migration: UIDL for msg %u is empty",
mail->seq);
continue;
}
map = array_append_space(&mstorage->pop3_uidl_map);
map->pop3_seq = mail->seq;
map->pop3_uidl = p_strdup(storage->pool, uidl);
map->size = size;
}
if (mailbox_search_deinit(&ctx) < 0) {
i_error("pop3_migration: Failed to search all POP3 mails: %s",
mailbox_get_last_internal_error(pop3_box, NULL));
ret = -1;
}
(void)mailbox_transaction_commit(&t);
return ret;
}
static void
pop3_map_read_cached_hdr_hashes(struct mailbox_transaction_context *t,
struct mail_search_args *search_args,
struct array *msg_map)
{
struct mail_search_context *ctx;
struct mail *mail;
struct msg_map_common *map;
buffer_t *cache_buf;
ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
cache_buf = buffer_create_dynamic(pool_datastack_create(), SHA1_RESULTLEN);
while (mailbox_search_next(ctx, &mail)) {
map = array_idx_modifiable_i(msg_map, mail->seq-1);
if (get_cached_hdr_sha1(mail, cache_buf, map->hdr_sha1))
map->hdr_sha1_set = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0) {
i_warning("pop3_migration: Failed to search all cached POP3 header hashes: %s - ignoring",
mailbox_get_last_internal_error(t->box, NULL));
}
}
static void map_remove_found_seqs(struct mail_search_arg *search_arg,
struct array *msg_map, uint32_t seq1)
{
const struct msg_map_common *map;
uint32_t seq, count = array_count_i(msg_map);
i_assert(search_arg->type == SEARCH_SEQSET);
for (seq = seq1; seq <= count; seq++) {
map = array_idx_i(msg_map, seq-1);
if (map->hdr_sha1_set)
seq_range_array_remove(&search_arg->value.seqset, seq);
}
}
static int
map_read_hdr_hashes(struct mailbox *box, struct array *msg_map, uint32_t seq1)
{
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct msg_map_common *map;
int ret = 0;
t = mailbox_transaction_begin(box, 0);
/* get all the cached hashes */
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq1, array_count_i(msg_map));
pop3_map_read_cached_hdr_hashes(t, search_args, msg_map);
/* read all the non-cached hashes. doing this in two passes allows
us to set wanted_fields=MAIL_FETCH_STREAM_HEADER, which allows
prefetching to work without downloading all the headers even
for mails that already are cached. */
map_remove_found_seqs(search_args->args, msg_map, seq1);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_STREAM_HEADER, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
map = array_idx_modifiable_i(msg_map, mail->seq-1);
if ((ret = get_hdr_sha1(mail, map->hdr_sha1)) < 0) {
ret = -1;
break;
}
if (ret > 0)
map->hdr_sha1_set = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0) {
i_error("pop3_migration: Failed to search all mail headers: %s",
mailbox_get_last_internal_error(box, NULL));
ret = -1;
}
(void)mailbox_transaction_commit(&t);
return ret < 0 ? -1 : 0;
}
static int
pop3_map_read_hdr_hashes(struct mail_storage *storage, struct mailbox *pop3_box,
unsigned first_seq)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
if (mstorage->pop3_all_hdr_sha1_set)
return 0;
if (mstorage->all_mailboxes) {
/* we may be matching against multiple mailboxes.
read all the hashes only once. */
first_seq = 1;
}
if (map_read_hdr_hashes(pop3_box, &mstorage->pop3_uidl_map.arr,
first_seq) < 0)
return -1;
if (first_seq == 1)
mstorage->pop3_all_hdr_sha1_set = TRUE;
return 0;
}
static int imap_map_read(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
const unsigned int uidl_cache_idx =
ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
struct mailbox_status status;
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct imap_msg_map *map;
uoff_t psize = (uoff_t)-1;
string_t *uidl;
int ret = 0;
mailbox_get_open_status(box, STATUS_MESSAGES, &status);
i_assert(!array_is_created(&mbox->imap_msg_map));
p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
t = mailbox_transaction_begin(box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(t, search_args, NULL,
mstorage->skip_size_check ? 0 :
MAIL_FETCH_PHYSICAL_SIZE, NULL);
mail_search_args_unref(&search_args);
uidl = t_str_new(64);
while (mailbox_search_next(ctx, &mail)) {
if (mstorage->skip_size_check)
;
else if (mail_get_physical_size(mail, &psize) < 0) {
i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
mail->uid,
mailbox_get_last_internal_error(box, NULL));
ret = -1;
break;
}
if (!mstorage->skip_uidl_cache) {
str_truncate(uidl, 0);
(void)mail_cache_lookup_field(mail->transaction->cache_view,
uidl, mail->seq, uidl_cache_idx);
}
map = array_append_space(&mbox->imap_msg_map);
map->uid = mail->uid;
map->psize = psize;
map->pop3_uidl = p_strdup_empty(box->pool, str_c(uidl));
}
if (mailbox_search_deinit(&ctx) < 0) {
i_error("pop3_migration: Failed to search all IMAP mails: %s",
mailbox_get_last_internal_error(box, NULL));
ret = -1;
}
(void)mailbox_transaction_commit(&t);
return ret;
}
static int imap_map_read_hdr_hashes(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
return map_read_hdr_hashes(box, &mbox->imap_msg_map.arr,
mbox->first_unfound_idx+1);
}
static void pop3_uidl_assign_cached(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct pop3_uidl_map *pop3_map;
struct imap_msg_map *imap_map;
unsigned int imap_idx, pop3_idx, pop3_count, imap_count;
int ret;
if (mstorage->skip_uidl_cache)
return;
array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_uidl_cmp);
array_sort(&mbox->imap_msg_map, imap_msg_map_uidl_cmp);
pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
/* see if we can match the messages using sizes */
for (imap_idx = pop3_idx = 0; imap_idx < imap_count; imap_idx++) {
if (imap_map[imap_idx].pop3_uidl == NULL)
continue;
ret = 1;
for (; pop3_idx < pop3_count; pop3_idx++) {
ret = strcmp(imap_map[imap_idx].pop3_uidl,
pop3_map[pop3_idx].pop3_uidl);
if (ret >= 0)
break;
}
if (ret == 0) {
imap_map[imap_idx].pop3_seq =
pop3_map[pop3_idx].pop3_seq;
pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
}
}
}
static bool pop3_uidl_assign_by_size(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct pop3_uidl_map *pop3_map;
struct imap_msg_map *imap_map;
unsigned int i, pop3_count, imap_count, count;
unsigned int size_match = 0, uidl_match = 0;
pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
count = I_MIN(pop3_count, imap_count);
/* see if we can match the messages using sizes */
for (i = 0; i < count; i++) {
if (imap_map[i].pop3_uidl != NULL) {
/* some of the UIDLs were already found cached. */
if (strcmp(pop3_map[i].pop3_uidl, imap_map[i].pop3_uidl) == 0) {
uidl_match++;
continue;
}
/* mismatch - can't trust the sizes */
break;
}
if (pop3_map[i].size != imap_map[i].psize ||
mstorage->skip_size_check)
break;
if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
/* two messages with same size, don't trust them */
break;
}
size_match++;
pop3_map[i].imap_uid = imap_map[i].uid;
imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
imap_map[i].pop3_seq = pop3_map[i].pop3_seq;
}
mbox->first_unfound_idx = i;
if (box->storage->user->mail_debug) {
i_debug("pop3_migration: cached uidls=%u, size matches=%u, total=%u",
uidl_match, size_match, count);
}
return i == count;
}
static int
pop3_uidl_assign_by_hdr_hash(struct mailbox *box, struct mailbox *pop3_box)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_uidl_map *pop3_map;
struct imap_msg_map *imap_map;
unsigned int pop3_idx, imap_idx, pop3_count, imap_count;
unsigned int first_seq, missing_uids_count;
uint32_t first_missing_idx = 0, first_missing_seq = (uint32_t)-1;
int ret;
first_seq = mbox->first_unfound_idx+1;
if (pop3_map_read_hdr_hashes(box->storage, pop3_box, first_seq) < 0 ||
imap_map_read_hdr_hashes(box) < 0)
return -1;
array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_hdr_cmp);
array_sort(&mbox->imap_msg_map, imap_msg_map_hdr_cmp);
pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
pop3_idx = imap_idx = 0;
while (pop3_idx < pop3_count && imap_idx < imap_count) {
if (!pop3_map[pop3_idx].common.hdr_sha1_set ||
pop3_map[pop3_idx].imap_uid != 0) {
pop3_idx++;
continue;
}
if (!imap_map[imap_idx].common.hdr_sha1_set ||
imap_map[imap_idx].pop3_uidl != NULL) {
imap_idx++;
continue;
}
ret = memcmp(pop3_map[pop3_idx].common.hdr_sha1,
imap_map[imap_idx].common.hdr_sha1,
sizeof(pop3_map[pop3_idx].common.hdr_sha1));
if (ret < 0)
pop3_idx++;
else if (ret > 0)
imap_idx++;
else {
pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
imap_map[imap_idx].pop3_uidl =
pop3_map[pop3_idx].pop3_uidl;
imap_map[imap_idx].pop3_seq =
pop3_map[pop3_idx].pop3_seq;
}
}
missing_uids_count = 0;
for (pop3_idx = 0; pop3_idx < pop3_count; pop3_idx++) {
if (pop3_map[pop3_idx].imap_uid != 0) {
/* matched */
} else if (!pop3_map[pop3_idx].common.hdr_sha1_set) {
/* we treated this mail as expunged - ignore */
} else {
uint32_t seq = pop3_map[pop3_idx].pop3_seq;
if (first_missing_seq > seq) {
first_missing_seq = seq;
first_missing_idx = pop3_idx;
}
missing_uids_count++;
}
}
if (missing_uids_count > 0 && !mstorage->all_mailboxes) {
string_t *str = t_str_new(128);
bool all_imap_mails_found = FALSE;
str_printfa(str, "pop3_migration: %u POP3 messages have no "
"matching IMAP messages (first POP3 msg %u UIDL %s)",
missing_uids_count, first_missing_seq,
pop3_map[first_missing_idx].pop3_uidl);
if (imap_count + missing_uids_count == pop3_count) {
str_append(str, " - all IMAP messages were found "
"(POP3 contains more than IMAP INBOX - you may want to set pop3_migration_all_mailboxes=yes)");
all_imap_mails_found = TRUE;
}
if (all_imap_mails_found && mstorage->ignore_extra_uidls) {
/* pop3 had more mails than imap. maybe it was just
that a new mail was just delivered. */
} else if (!mstorage->ignore_missing_uidls) {
str_append(str, " - set pop3_migration_ignore_missing_uidls=yes");
if (all_imap_mails_found)
str_append(str, " or pop3_migration_ignore_extra_uidls=yes");
i_error("%s to continue anyway", str_c(str));
return -1;
}
i_warning("%s", str_c(str));
} else if (box->storage->user->mail_debug) {
i_debug("pop3_migration: %u mails matched by headers", pop3_count);
}
array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);
return 0;
}
static void imap_uidls_add_to_cache(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct mailbox_transaction_context *t;
struct mail *mail;
struct index_mail *imail;
struct imap_msg_map *imap_map;
unsigned int i, count;
unsigned int field_idx;
t = mailbox_transaction_begin(box, 0);
mail = mail_alloc(t, 0, NULL);
imail = (struct index_mail *)mail;
field_idx = imail->ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
imap_map = array_get_modifiable(&mbox->imap_msg_map, &count);
for (i = 0; i < count; i++) {
if (imap_map[i].pop3_uidl == NULL)
continue;
if (!mail_set_uid(mail, imap_map[i].uid))
i_unreached();
if (mail_cache_field_can_add(t->cache_trans, mail->seq, field_idx)) {
index_mail_cache_add_idx(imail, field_idx,
imap_map[i].pop3_uidl, strlen(imap_map[i].pop3_uidl)+1);
}
}
mail_free(&mail);
(void)mailbox_transaction_commit(&t);
}
static int pop3_migration_uidl_sync(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct mailbox *pop3_box;
pop3_box = pop3_mailbox_alloc(box->storage);
/* the POP3 server isn't connected to yet. handle all IMAP traffic
first before connecting, so POP3 server won't disconnect us due to
idling. */
if (imap_map_read(box) < 0 ||
pop3_map_read(box->storage, pop3_box) < 0) {
mailbox_free(&pop3_box);
return -1;
}
pop3_uidl_assign_cached(box);
array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);
if (!pop3_uidl_assign_by_size(box)) {
/* everything wasn't assigned, figure out the rest with
header hashes */
if (pop3_uidl_assign_by_hdr_hash(box, pop3_box) < 0) {
mailbox_free(&pop3_box);
return -1;
}
}
if (!mstorage->skip_uidl_cache)
imap_uidls_add_to_cache(box);
mbox->uidl_synced = TRUE;
mailbox_free(&pop3_box);
return 0;
}
static int pop3_migration_uidl_sync_if_needed(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
if (mbox->uidl_synced)
return 0;
if (mbox->uidl_sync_failed ||
pop3_migration_uidl_sync(box) < 0) {
mbox->uidl_sync_failed = TRUE;
mail_storage_set_error(box->storage, MAIL_ERROR_TEMP,
"POP3 UIDLs couldn't be synced");
return -1;
}
return 0;
}
static int
pop3_migration_get_special(struct mail *_mail, enum mail_fetch_field field,
const char **value_r)
{
struct mail_private *mail = (struct mail_private *)_mail;
union mail_module_context *mmail = POP3_MIGRATION_MAIL_CONTEXT(mail);
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(_mail->box);
struct imap_msg_map map_key, *map;
if (field == MAIL_FETCH_UIDL_BACKEND ||
field == MAIL_FETCH_POP3_ORDER) {
if (pop3_migration_uidl_sync_if_needed(_mail->box) < 0)
return -1;
i_zero(&map_key);
map_key.uid = _mail->uid;
map = array_bsearch(&mbox->imap_msg_map, &map_key,
imap_msg_map_uid_cmp);
if (map != NULL && map->pop3_uidl != NULL) {
if (field == MAIL_FETCH_UIDL_BACKEND)
*value_r = map->pop3_uidl;
else
*value_r = t_strdup_printf("%u", map->pop3_seq);
return 0;
}
/* not found from POP3 server, fallback to default */
}
return mmail->super.get_special(_mail, field, value_r);
}
static void pop3_migration_mail_allocated(struct mail *_mail)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(_mail->box->storage);
struct mail_private *mail = (struct mail_private *)_mail;
struct mail_vfuncs *v = mail->vlast;
union mail_module_context *mmail;
struct mail_namespace *ns;
if (mstorage == NULL ||
(!mstorage->all_mailboxes && !_mail->box->inbox_user)) {
/* assigns UIDLs only for INBOX */
return;
}
ns = mail_namespace_find(_mail->box->storage->user->namespaces,
mstorage->pop3_box_vname);
if (ns == mailbox_get_namespace(_mail->box)) {
/* we're accessing the pop3-migration namespace itself */
return;
}
mmail = p_new(mail->pool, union mail_module_context, 1);
mmail->super = *v;
mail->vlast = &mmail->super;
v->get_special = pop3_migration_get_special;
MODULE_CONTEXT_SET_SELF(mail, pop3_migration_mail_module, mmail);
}
static struct mail_search_context *
pop3_migration_mailbox_search_init(struct mailbox_transaction_context *t,
struct mail_search_args *args,
const enum mail_sort_type *sort_program,
enum mail_fetch_field wanted_fields,
struct mailbox_header_lookup_ctx *wanted_headers)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(t->box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(t->box->storage);
if ((wanted_fields & (MAIL_FETCH_UIDL_BACKEND |
MAIL_FETCH_POP3_ORDER)) != 0 &&
(mstorage->all_mailboxes || t->box->inbox_user)) {
/* Start POP3 UIDL syncing before the search, so we'll do it
before we start sending any FETCH BODY[]s to IMAP. It
shouldn't matter much, except this works around a bug in
Yahoo IMAP where it sometimes breaks its state when doing
a FETCH BODY[] followed by FETCH BODY[HEADER].. */
(void)pop3_migration_uidl_sync_if_needed(t->box);
}
return mbox->module_ctx.super.search_init(t, args, sort_program,
wanted_fields, wanted_headers);
}
static void pop3_migration_mailbox_allocated(struct mailbox *box)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct mailbox_vfuncs *v = box->vlast;
struct pop3_migration_mailbox *mbox;
if (mstorage == NULL)
return;
mbox = p_new(box->pool, struct pop3_migration_mailbox, 1);
mbox->module_ctx.super = *v;
box->vlast = &mbox->module_ctx.super;
v->search_init = pop3_migration_mailbox_search_init;
MODULE_CONTEXT_SET(box, pop3_migration_storage_module, mbox);
}
static void pop3_migration_mail_storage_destroy(struct mail_storage *storage)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
if (array_is_created(&mstorage->pop3_uidl_map))
array_free(&mstorage->pop3_uidl_map);
mstorage->module_ctx.super.destroy(storage);
}
static void pop3_migration_mail_storage_created(struct mail_storage *storage)
{
struct pop3_migration_mail_storage *mstorage;
struct mail_storage_vfuncs *v = storage->vlast;
const char *pop3_box_vname;
pop3_box_vname = mail_user_plugin_getenv(storage->user,
"pop3_migration_mailbox");
if (pop3_box_vname == NULL) {
if (storage->user->mail_debug)
i_debug("pop3_migration: No pop3_migration_mailbox setting - disabled");
return;
}
mstorage = p_new(storage->pool, struct pop3_migration_mail_storage, 1);
mstorage->module_ctx.super = *v;
storage->vlast = &mstorage->module_ctx.super;
v->destroy = pop3_migration_mail_storage_destroy;
mstorage->pop3_box_vname = p_strdup(storage->pool, pop3_box_vname);
mstorage->all_mailboxes =
mail_user_plugin_getenv_bool(storage->user,
"pop3_migration_all_mailboxes");
mstorage->ignore_missing_uidls =
mail_user_plugin_getenv_bool(storage->user,
"pop3_migration_ignore_missing_uidls");
mstorage->ignore_extra_uidls =
mail_user_plugin_getenv_bool(storage->user,
"pop3_migration_ignore_extra_uidls");
mstorage->skip_size_check =
mail_user_plugin_getenv_bool(storage->user,
"pop3_migration_skip_size_check");
mstorage->skip_uidl_cache =
mail_user_plugin_getenv_bool(storage->user,
"pop3_migration_skip_uidl_cache");
MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage);
}
static struct mail_storage_hooks pop3_migration_mail_storage_hooks = {
.mail_allocated = pop3_migration_mail_allocated,
.mailbox_allocated = pop3_migration_mailbox_allocated,
.mail_storage_created = pop3_migration_mail_storage_created
};
void pop3_migration_plugin_init(struct module *module)
{
mail_storage_hooks_add(module, &pop3_migration_mail_storage_hooks);
}
void pop3_migration_plugin_deinit(void)
{
mail_storage_hooks_remove(&pop3_migration_mail_storage_hooks);
}