pop3-migration-plugin.c revision 061046c9aa2eec5c6c2f148ec95a4e51db3d8fd2
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2007-2016 Dovecot authors, see the included COPYING file */
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch#include "lib.h"
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch#include "array.h"
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch#include "istream.h"
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch#include "istream-header-filter.h"
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch#include "str.h"
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch#include "sha1.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "message-size.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "message-header-hash.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "message-header-parser.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "mail-cache.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "mail-namespace.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "mail-search-build.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "index-storage.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "index-mail.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#include "pop3-migration-plugin.h"
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#define POP3_MIGRATION_CONTEXT(obj) \
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch MODULE_CONTEXT(obj, pop3_migration_storage_module)
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch#define POP3_MIGRATION_MAIL_CONTEXT(obj) \
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch MODULE_CONTEXT(obj, pop3_migration_mail_module)
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Boschstruct msg_map_common {
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch /* sha1(header) - set only when needed */
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch unsigned char hdr_sha1[SHA1_RESULTLEN];
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch bool hdr_sha1_set:1;
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch};
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Boschstruct pop3_uidl_map {
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch struct msg_map_common common;
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch uint32_t pop3_seq;
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch uint32_t imap_uid;
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch /* UIDL */
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch const char *pop3_uidl;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch /* LIST size */
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch uoff_t size;
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch};
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Boschstruct imap_msg_map {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch struct msg_map_common common;
45324f1eafa565dbc65e4dd335de9507dead55e6Timo Sirainen
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch uint32_t uid, pop3_seq;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch uoff_t psize;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch const char *pop3_uidl;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch};
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Boschstruct pop3_migration_mail_storage {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch union mail_storage_module_context module_ctx;
833bed942977673526c72e79bccc09314fc57104Phil Carmody
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch const char *pop3_box_vname;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch ARRAY(struct pop3_uidl_map) pop3_uidl_map;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch bool all_mailboxes:1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch bool pop3_all_hdr_sha1_set:1;
e9228a3918aa0243eff4aae1ff5462bd3198417fTimo Sirainen bool ignore_missing_uidls:1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch bool skip_size_check:1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch};
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
1e9296de32c9ddda40f33c06556cd568ddadf71fTimo Sirainenstruct pop3_migration_mailbox {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch union mailbox_module_context module_ctx;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch ARRAY(struct imap_msg_map) imap_msg_map;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch unsigned int first_unfound_idx;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch struct mail_cache_field cache_field;
1e9296de32c9ddda40f33c06556cd568ddadf71fTimo Sirainen
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch bool cache_field_registered:1;
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch bool uidl_synced:1;
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch bool uidl_sync_failed:1;
833bed942977673526c72e79bccc09314fc57104Phil Carmody bool uidl_ordered:1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch};
833bed942977673526c72e79bccc09314fc57104Phil Carmody
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch/* NOTE: these headers must be sorted */
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Boschstatic const char *hdr_hash_skip_headers[] = {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "Content-Length",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "Return-Path", /* Yahoo IMAP has Return-Path, Yahoo POP3 doesn't */
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "Status",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "X-IMAP",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "X-IMAPbase",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "X-Keywords",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "X-Message-Flag",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "X-Status",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "X-UID",
1e9296de32c9ddda40f33c06556cd568ddadf71fTimo Sirainen "X-UIDL",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch "X-Yahoo-Newman-Property"
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch};
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainenconst char *pop3_migration_plugin_version = DOVECOT_ABI_VERSION;
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainen
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module,
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch &mail_storage_module_register);
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module,
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch &mail_module_register);
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Boschstatic int imap_msg_map_uid_cmp(const struct imap_msg_map *map1,
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch const struct imap_msg_map *map2)
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch{
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch if (map1->uid < map2->uid)
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch return -1;
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch if (map1->uid > map2->uid)
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch return 1;
14bd2410de3a0261d9c53c6120915027262216bdTimo Sirainen return 0;
14bd2410de3a0261d9c53c6120915027262216bdTimo Sirainen}
14bd2410de3a0261d9c53c6120915027262216bdTimo Sirainen
9f8cef4cbc49797053c343209ea13022fdbc5a63Stephan Boschstatic int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
9f8cef4cbc49797053c343209ea13022fdbc5a63Stephan Bosch const struct pop3_uidl_map *map2)
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainen{
9f8cef4cbc49797053c343209ea13022fdbc5a63Stephan Bosch if (map1->pop3_seq < map2->pop3_seq)
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainen return -1;
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainen if (map1->pop3_seq > map2->pop3_seq)
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainen return 1;
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainen return 0;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch}
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Boschstatic int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch const struct pop3_uidl_map *map2)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch{
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return memcmp(map1->common.hdr_sha1, map2->common.hdr_sha1,
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch sizeof(map1->common.hdr_sha1));
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch}
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Boschstatic int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1,
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch const struct imap_msg_map *map2)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch{
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return memcmp(map1->common.hdr_sha1, map2->common.hdr_sha1,
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch sizeof(map1->common.hdr_sha1));
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch}
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Boschstruct pop3_hdr_context {
833bed942977673526c72e79bccc09314fc57104Phil Carmody bool have_eoh;
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch bool stop;
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch};
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Boschstatic bool header_name_is_valid(const char *name)
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch{
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch unsigned int i;
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch for (i = 0; name[i] != '\0'; i++) {
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch if ((uint8_t)name[i] <= 0x20 || name[i] >= 0x7f)
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch return FALSE;
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch }
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch return TRUE;
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch}
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Boschstatic void
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Boschpop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED,
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch struct message_header_line *hdr,
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch bool *matched, struct pop3_hdr_context *ctx)
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch{
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch if (hdr == NULL)
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch return;
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch if (hdr->eoh) {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch ctx->have_eoh = TRUE;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (ctx->stop)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch *matched = TRUE;
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch } else {
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch if (strspn(hdr->name, "\r") == hdr->name_len) {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch /* CR+CR+LF - some servers stop the header processing
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch here while others don't. To make sure they can be
ee2633056e67353157bfbce4d9e0d1c3ceaa627aStephan Bosch matched correctly we want to stop here entirely. */
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch ctx->stop = TRUE;
91a482473f200152d6713181c0e36f7a4f03ef6dTimo Sirainen }
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (ctx->stop)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch *matched = TRUE;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch else if (!header_name_is_valid(hdr->name)) {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch /* Yahoo IMAP drops headers with invalid names, while
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch Yahoo POP3 preserves them. Drop them all. */
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch *matched = TRUE;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch }
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch }
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch}
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Boschint pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input,
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN],
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch bool *have_eoh_r)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch{
85f3bd5926fff0e70b6d259a5c8074bd8cdeb9adTimo Sirainen const unsigned char *data;
85f3bd5926fff0e70b6d259a5c8074bd8cdeb9adTimo Sirainen size_t size;
85f3bd5926fff0e70b6d259a5c8074bd8cdeb9adTimo Sirainen struct sha1_ctxt sha1_ctx;
85f3bd5926fff0e70b6d259a5c8074bd8cdeb9adTimo Sirainen struct pop3_hdr_context hdr_ctx;
85f3bd5926fff0e70b6d259a5c8074bd8cdeb9adTimo Sirainen
85f3bd5926fff0e70b6d259a5c8074bd8cdeb9adTimo Sirainen memset(&hdr_ctx, 0, sizeof(hdr_ctx));
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch /* hide headers that might change or be different in IMAP vs. POP3 */
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch input = i_stream_create_header_filter(input, HEADER_FILTER_HIDE_BODY |
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch hdr_hash_skip_headers,
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch N_ELEMENTS(hdr_hash_skip_headers),
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch pop3_header_filter_callback, &hdr_ctx);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch sha1_init(&sha1_ctx);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch while (i_stream_read_more(input, &data, &size) > 0) {
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch message_header_hash_more(&hash_method_sha1, &sha1_ctx, 2,
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch data, size);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch i_stream_skip(input, size);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch }
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch if (input->stream_errno != 0) {
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch i_error("pop3_migration: Failed to read header for msg %u: %s",
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch mail_seq, i_stream_get_error(input));
93ed69606237a08623f8294c060fa148880058f8Timo Sirainen i_stream_unref(&input);
93ed69606237a08623f8294c060fa148880058f8Timo Sirainen return -1;
93ed69606237a08623f8294c060fa148880058f8Timo Sirainen }
93ed69606237a08623f8294c060fa148880058f8Timo Sirainen sha1_result(&sha1_ctx, sha1_r);
93ed69606237a08623f8294c060fa148880058f8Timo Sirainen i_stream_unref(&input);
93ed69606237a08623f8294c060fa148880058f8Timo Sirainen
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch *have_eoh_r = hdr_ctx.have_eoh;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return 0;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch}
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Boschstatic unsigned int get_cache_idx(struct mail *mail)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch{
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(mail->box);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (mbox->cache_field_registered)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return mbox->cache_field.idx;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
711e8e4c5c5d702dfa062f42a1ede5de14c151c9Stephan Bosch mbox->cache_field.name = "pop3-migration.hdr";
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch mbox->cache_field.type = MAIL_CACHE_FIELD_FIXED_SIZE;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch mbox->cache_field.field_size = SHA1_RESULTLEN;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch mail_cache_register_fields(mail->box->cache, &mbox->cache_field, 1);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch mbox->cache_field_registered = TRUE;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return mbox->cache_field.idx;
711e8e4c5c5d702dfa062f42a1ede5de14c151c9Stephan Bosch}
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Boschstatic int
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Boschget_hdr_sha1(struct mail *mail, unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN])
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch{
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch struct istream *input;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch const char *errstr;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch enum mail_error error;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch bool have_eoh;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch int ret;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (mail_get_hdr_stream(mail, NULL, &input) < 0) {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch errstr = mailbox_get_last_error(mail->box, &error);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch i_error("pop3_migration: Failed to get header for msg %u: %s",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch mail->seq, errstr);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return error == MAIL_ERROR_EXPUNGED ? 0 : -1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch }
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (pop3_migration_get_hdr_sha1(mail->seq, input, sha1_r, &have_eoh) < 0)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return -1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (have_eoh) {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch struct index_mail *imail = (struct index_mail *)mail;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch index_mail_cache_add_idx(imail, get_cache_idx(mail),
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch sha1_r, SHA1_RESULTLEN);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return 1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch }
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch /* The empty "end of headers" line is missing. Either this means that
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch the headers ended unexpectedly (which is ok) or that the remote
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch server is buggy. Some servers have problems with
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch 1) header line continuations that contain only whitespace and
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch 2) headers that have no ":". The header gets truncated when such
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch line is reached.
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch
2f64a4c88de91c483fb378bc80d10e1caa6f2305Stephan Bosch At least Oracle IMS IMAP FETCH BODY[HEADER] handles 1) by not
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch returning the whitespace line and 2) by returning the line but
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch truncating the rest. POP3 TOP instead returns the entire header.
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch This causes the IMAP and POP3 hashes not to match.
833bed942977673526c72e79bccc09314fc57104Phil Carmody
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch If there's LF+CR+CR+LF in the middle of headers, Courier IMAP's
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch FETCH BODY[HEADER] stops after that, but Courier POP3's TOP doesn't.
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch So we'll try to avoid this by falling back to full FETCH BODY[]
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch (and/or RETR) and we'll parse the header ourself from it. This
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch should work around any similar bugs in all IMAP/POP3 servers. */
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (mail_get_stream_because(mail, NULL, NULL, "pop3-migration", &input) < 0) {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch errstr = mailbox_get_last_error(mail->box, &error);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch i_error("pop3_migration: Failed to get body for msg %u: %s",
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch mail->seq, errstr);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return error == MAIL_ERROR_EXPUNGED ? 0 : -1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch }
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainen ret = pop3_migration_get_hdr_sha1(mail->seq, input, sha1_r, &have_eoh);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (ret == 0) {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch if (!have_eoh)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch i_warning("pop3_migration: Truncated email with UID %u stored as truncated", mail->uid);
faa8995f1d300e7a8917407a52bbd1b98e10bf25Timo Sirainen struct index_mail *imail = (struct index_mail *)mail;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch index_mail_cache_add_idx(imail, get_cache_idx(mail),
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch sha1_r, SHA1_RESULTLEN);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return 1;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch } else {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return -1;
711e8e4c5c5d702dfa062f42a1ede5de14c151c9Stephan Bosch }
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch}
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Boschstatic bool
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Boschget_cached_hdr_sha1(struct mail *mail, buffer_t *cache_buf,
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN])
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch{
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch struct index_mail *imail = (struct index_mail *)mail;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch
b37e11d37fb1ebf50511eef5d9d96d1205818458Stephan Bosch buffer_set_used_size(cache_buf, 0);
b37e11d37fb1ebf50511eef5d9d96d1205818458Stephan Bosch if (index_mail_cache_lookup_field(imail, cache_buf,
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch get_cache_idx(mail)) > 0 &&
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch cache_buf->used == SHA1_RESULTLEN) {
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch memcpy(sha1_r, cache_buf->data, cache_buf->used);
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return TRUE;
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch }
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch return FALSE;
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch}
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Boschstatic struct mailbox *pop3_mailbox_alloc(struct mail_storage *storage)
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch{
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch struct pop3_migration_mail_storage *mstorage =
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch POP3_MIGRATION_CONTEXT(storage);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch struct mail_namespace *ns;
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch ns = mail_namespace_find(storage->user->namespaces,
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch mstorage->pop3_box_vname);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch i_assert(ns != NULL);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch return mailbox_alloc(ns->list, mstorage->pop3_box_vname,
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen MAILBOX_FLAG_READONLY | MAILBOX_FLAG_POP3_SESSION);
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen}
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainenstatic int pop3_map_read(struct mail_storage *storage, struct mailbox *pop3_box)
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch{
3fcb3d2d1f3583025ff62bae95ec706920f398b1Stephan Bosch struct pop3_migration_mail_storage *mstorage =
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen POP3_MIGRATION_CONTEXT(storage);
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen struct mailbox_transaction_context *t;
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen struct mail_search_args *search_args;
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen struct mail_search_context *ctx;
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen struct mail *mail;
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen struct pop3_uidl_map *map;
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen const char *uidl;
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen uoff_t size = (uoff_t)-1;
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen int ret = 0;
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen if (array_is_created(&mstorage->pop3_uidl_map)) {
95e0b82fdff1bb511067d703bb8b67c22f242c38Timo Sirainen /* already read these, just reset the imap_uids */
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch map->imap_uid = 0;
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch return 0;
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch }
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch i_array_init(&mstorage->pop3_uidl_map, 128);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch if (mailbox_sync(pop3_box, 0) < 0) {
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch i_error("pop3_migration: Couldn't sync mailbox %s: %s",
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch return -1;
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch }
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch t = mailbox_transaction_begin(pop3_box, 0);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch search_args = mail_search_build_init();
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch mail_search_build_add_all(search_args);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch ctx = mailbox_search_init(t, search_args, NULL,
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch mstorage->skip_size_check ? 0 :
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch MAIL_FETCH_PHYSICAL_SIZE, NULL);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch mail_search_args_unref(&search_args);
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch while (mailbox_search_next(ctx, &mail)) {
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch /* get the size with LIST instead of RETR */
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch
564e117d86ce5b659f9b9570edddc566f9ebb5dfStephan Bosch if (mstorage->skip_size_check)
;
else if (mail_get_physical_size(mail, &size) < 0) {
i_error("pop3_migration: Failed to get size for msg %u: %s",
mail->seq,
mailbox_get_last_error(pop3_box, NULL));
ret = -1;
break;
}
if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
mail->seq,
mailbox_get_last_error(pop3_box, NULL));
ret = -1;
break;
}
if (*uidl == '\0') {
i_warning("pop3_migration: UIDL for msg %u is empty",
mail->seq);
continue;
}
map = array_append_space(&mstorage->pop3_uidl_map);
map->pop3_seq = mail->seq;
map->pop3_uidl = p_strdup(storage->pool, uidl);
map->size = size;
}
if (mailbox_search_deinit(&ctx) < 0) {
i_error("pop3_migration: Failed to search all POP3 mails: %s",
mailbox_get_last_error(pop3_box, NULL));
ret = -1;
}
(void)mailbox_transaction_commit(&t);
return ret;
}
static void
pop3_map_read_cached_hdr_hashes(struct mailbox_transaction_context *t,
struct mail_search_args *search_args,
struct array *msg_map)
{
struct mail_search_context *ctx;
struct mail *mail;
struct msg_map_common *map;
buffer_t *cache_buf;
ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
cache_buf = buffer_create_dynamic(pool_datastack_create(), SHA1_RESULTLEN);
while (mailbox_search_next(ctx, &mail)) {
map = array_idx_modifiable_i(msg_map, mail->seq-1);
if (get_cached_hdr_sha1(mail, cache_buf, map->hdr_sha1))
map->hdr_sha1_set = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0) {
i_warning("pop3_migration: Failed to search all cached POP3 header hashes: %s - ignoring",
mailbox_get_last_error(t->box, NULL));
}
}
static void map_remove_found_seqs(struct mail_search_arg *search_arg,
struct array *msg_map, uint32_t seq1)
{
const struct msg_map_common *map;
uint32_t seq, count = array_count_i(msg_map);
i_assert(search_arg->type == SEARCH_SEQSET);
for (seq = seq1; seq <= count; seq++) {
map = array_idx_i(msg_map, seq-1);
if (map->hdr_sha1_set)
seq_range_array_remove(&search_arg->value.seqset, seq);
}
}
static int
map_read_hdr_hashes(struct mailbox *box, struct array *msg_map, uint32_t seq1)
{
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct msg_map_common *map;
int ret = 0;
t = mailbox_transaction_begin(box, 0);
/* get all the cached hashes */
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq1, array_count_i(msg_map));
pop3_map_read_cached_hdr_hashes(t, search_args, msg_map);
/* read all the non-cached hashes. doing this in two passes allows
us to set wanted_fields=MAIL_FETCH_STREAM_HEADER, which allows
prefetching to work without downloading all the headers even
for mails that already are cached. */
map_remove_found_seqs(search_args->args, msg_map, seq1);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_STREAM_HEADER, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
map = array_idx_modifiable_i(msg_map, mail->seq-1);
if ((ret = get_hdr_sha1(mail, map->hdr_sha1)) < 0) {
ret = -1;
break;
}
if (ret > 0)
map->hdr_sha1_set = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0) {
i_error("pop3_migration: Failed to search all mail headers: %s",
mailbox_get_last_error(box, NULL));
ret = -1;
}
(void)mailbox_transaction_commit(&t);
return ret < 0 ? -1 : 0;
}
static int
pop3_map_read_hdr_hashes(struct mail_storage *storage, struct mailbox *pop3_box,
unsigned first_seq)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
if (mstorage->pop3_all_hdr_sha1_set)
return 0;
if (mstorage->all_mailboxes) {
/* we may be matching against multiple mailboxes.
read all the hashes only once. */
first_seq = 1;
}
if (map_read_hdr_hashes(pop3_box, &mstorage->pop3_uidl_map.arr,
first_seq) < 0)
return -1;
if (first_seq == 1)
mstorage->pop3_all_hdr_sha1_set = TRUE;
return 0;
}
static int imap_map_read(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct mailbox_status status;
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct imap_msg_map *map;
uoff_t psize = (uoff_t)-1;
int ret = 0;
mailbox_get_open_status(box, STATUS_MESSAGES, &status);
i_assert(!array_is_created(&mbox->imap_msg_map));
p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
t = mailbox_transaction_begin(box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(t, search_args, NULL,
mstorage->skip_size_check ? 0 :
MAIL_FETCH_PHYSICAL_SIZE, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
if (mstorage->skip_size_check)
;
else if (mail_get_physical_size(mail, &psize) < 0) {
i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
mail->uid,
mailbox_get_last_error(box, NULL));
ret = -1;
break;
}
map = array_append_space(&mbox->imap_msg_map);
map->uid = mail->uid;
map->psize = psize;
}
if (mailbox_search_deinit(&ctx) < 0) {
i_error("pop3_migration: Failed to search all IMAP mails: %s",
mailbox_get_last_error(box, NULL));
ret = -1;
}
(void)mailbox_transaction_commit(&t);
return ret;
}
static int imap_map_read_hdr_hashes(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
return map_read_hdr_hashes(box, &mbox->imap_msg_map.arr,
mbox->first_unfound_idx+1);
}
static bool pop3_uidl_assign_by_size(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct pop3_uidl_map *pop3_map;
struct imap_msg_map *imap_map;
unsigned int i, pop3_count, imap_count, count;
if (mstorage->skip_size_check)
return FALSE;
pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
count = I_MIN(pop3_count, imap_count);
/* see if we can match the messages using sizes */
for (i = 0; i < count; i++) {
if (pop3_map[i].size != imap_map[i].psize)
break;
if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
/* two messages with same size, don't trust them */
break;
}
pop3_map[i].imap_uid = imap_map[i].uid;
imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
imap_map[i].pop3_seq = pop3_map[i].pop3_seq;
}
mbox->first_unfound_idx = i;
if (box->storage->user->mail_debug)
i_debug("pop3_migration: %u/%u mails matched by size", i, count);
return i == count;
}
static int
pop3_uidl_assign_by_hdr_hash(struct mailbox *box, struct mailbox *pop3_box)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_uidl_map *pop3_map;
struct imap_msg_map *imap_map;
unsigned int pop3_idx, imap_idx, pop3_count, imap_count;
unsigned int first_seq, missing_uids_count;
uint32_t first_missing_idx = (uint32_t)-1;
int ret;
first_seq = mbox->first_unfound_idx+1;
if (pop3_map_read_hdr_hashes(box->storage, pop3_box, first_seq) < 0 ||
imap_map_read_hdr_hashes(box) < 0)
return -1;
array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_hdr_cmp);
array_sort(&mbox->imap_msg_map, imap_msg_map_hdr_cmp);
pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
pop3_idx = imap_idx = 0;
while (pop3_idx < pop3_count && imap_idx < imap_count) {
if (!pop3_map[pop3_idx].common.hdr_sha1_set ||
pop3_map[pop3_idx].imap_uid != 0) {
pop3_idx++;
continue;
}
if (!imap_map[imap_idx].common.hdr_sha1_set ||
imap_map[imap_idx].pop3_uidl != NULL) {
imap_idx++;
continue;
}
ret = memcmp(pop3_map[pop3_idx].common.hdr_sha1,
imap_map[imap_idx].common.hdr_sha1,
sizeof(pop3_map[pop3_idx].common.hdr_sha1));
if (ret < 0)
pop3_idx++;
else if (ret > 0)
imap_idx++;
else {
pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
imap_map[imap_idx].pop3_uidl =
pop3_map[pop3_idx].pop3_uidl;
imap_map[imap_idx].pop3_seq =
pop3_map[pop3_idx].pop3_seq;
}
}
missing_uids_count = 0;
for (pop3_idx = 0; pop3_idx < pop3_count; pop3_idx++) {
if (pop3_map[pop3_idx].imap_uid != 0) {
/* matched */
} else if (!pop3_map[pop3_idx].common.hdr_sha1_set) {
/* we treated this mail as expunged - ignore */
} else {
if (first_missing_idx == (uint32_t)-1)
first_missing_idx = pop3_idx;
missing_uids_count++;
}
}
if (missing_uids_count > 0 && !mstorage->all_mailboxes) {
string_t *str = t_str_new(128);
str_printfa(str, "pop3_migration: %u POP3 messages have no "
"matching IMAP messages (first POP3 msg %u UIDL %s)",
missing_uids_count,
pop3_map[first_missing_idx].pop3_seq,
pop3_map[first_missing_idx].pop3_uidl);
if (imap_count + missing_uids_count == pop3_count) {
str_append(str, " - all IMAP messages were found "
"(POP3 contains more than IMAP INBOX - you may want to set pop3_migration_all_mailboxes=yes)");
}
if (!mstorage->ignore_missing_uidls) {
i_error("%s - set pop3_migration_ignore_missing_uidls=yes to continue anyway",
str_c(str));
return -1;
}
i_warning("%s", str_c(str));
} else if (box->storage->user->mail_debug) {
i_debug("pop3_migration: %u mails matched by headers", pop3_count);
}
array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);
return 0;
}
static int pop3_migration_uidl_sync(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct mailbox *pop3_box;
const struct pop3_uidl_map *pop3_map;
unsigned int i, count;
uint32_t prev_uid;
pop3_box = pop3_mailbox_alloc(box->storage);
/* the POP3 server isn't connected to yet. handle all IMAP traffic
first before connecting, so POP3 server won't disconnect us due to
idling. */
if (imap_map_read(box) < 0 ||
pop3_map_read(box->storage, pop3_box) < 0) {
mailbox_free(&pop3_box);
return -1;
}
if (!pop3_uidl_assign_by_size(box)) {
/* everything wasn't assigned, figure out the rest with
header hashes */
if (pop3_uidl_assign_by_hdr_hash(box, pop3_box) < 0) {
mailbox_free(&pop3_box);
return -1;
}
}
/* see if the POP3 UIDL order is the same as IMAP UID order */
mbox->uidl_ordered = TRUE;
pop3_map = array_get(&mstorage->pop3_uidl_map, &count);
prev_uid = 0;
for (i = 0; i < count; i++) {
if (pop3_map[i].imap_uid == 0)
continue;
if (prev_uid > pop3_map[i].imap_uid) {
mbox->uidl_ordered = FALSE;
break;
}
prev_uid = pop3_map[i].imap_uid;
}
mbox->uidl_synced = TRUE;
mailbox_free(&pop3_box);
return 0;
}
static int pop3_migration_uidl_sync_if_needed(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
if (mbox->uidl_synced)
return 0;
if (mbox->uidl_sync_failed ||
pop3_migration_uidl_sync(box) < 0) {
mbox->uidl_sync_failed = TRUE;
mail_storage_set_error(box->storage, MAIL_ERROR_TEMP,
"POP3 UIDLs couldn't be synced");
return -1;
}
return 0;
}
static int
pop3_migration_get_special(struct mail *_mail, enum mail_fetch_field field,
const char **value_r)
{
struct mail_private *mail = (struct mail_private *)_mail;
union mail_module_context *mmail = POP3_MIGRATION_MAIL_CONTEXT(mail);
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(_mail->box);
struct imap_msg_map map_key, *map;
if (field == MAIL_FETCH_UIDL_BACKEND ||
field == MAIL_FETCH_POP3_ORDER) {
if (pop3_migration_uidl_sync_if_needed(_mail->box) < 0)
return -1;
memset(&map_key, 0, sizeof(map_key));
map_key.uid = _mail->uid;
map = array_bsearch(&mbox->imap_msg_map, &map_key,
imap_msg_map_uid_cmp);
if (map != NULL && map->pop3_uidl != NULL) {
if (field == MAIL_FETCH_UIDL_BACKEND)
*value_r = map->pop3_uidl;
else
*value_r = t_strdup_printf("%u", map->pop3_seq);
return 0;
}
/* not found from POP3 server, fallback to default */
}
return mmail->super.get_special(_mail, field, value_r);
}
static void pop3_migration_mail_allocated(struct mail *_mail)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(_mail->box->storage);
struct mail_private *mail = (struct mail_private *)_mail;
struct mail_vfuncs *v = mail->vlast;
union mail_module_context *mmail;
struct mail_namespace *ns;
if (mstorage == NULL ||
(!mstorage->all_mailboxes && !_mail->box->inbox_user)) {
/* assigns UIDLs only for INBOX */
return;
}
ns = mail_namespace_find(_mail->box->storage->user->namespaces,
mstorage->pop3_box_vname);
if (ns == mailbox_get_namespace(_mail->box)) {
/* we're accessing the pop3-migration namespace itself */
return;
}
mmail = p_new(mail->pool, union mail_module_context, 1);
mmail->super = *v;
mail->vlast = &mmail->super;
v->get_special = pop3_migration_get_special;
MODULE_CONTEXT_SET_SELF(mail, pop3_migration_mail_module, mmail);
}
static struct mail_search_context *
pop3_migration_mailbox_search_init(struct mailbox_transaction_context *t,
struct mail_search_args *args,
const enum mail_sort_type *sort_program,
enum mail_fetch_field wanted_fields,
struct mailbox_header_lookup_ctx *wanted_headers)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(t->box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(t->box->storage);
if ((wanted_fields & (MAIL_FETCH_UIDL_BACKEND |
MAIL_FETCH_POP3_ORDER)) != 0 &&
(mstorage->all_mailboxes || t->box->inbox_user)) {
/* Start POP3 UIDL syncing before the search, so we'll do it
before we start sending any FETCH BODY[]s to IMAP. It
shouldn't matter much, except this works around a bug in
Yahoo IMAP where it sometimes breaks its state when doing
a FETCH BODY[] followed by FETCH BODY[HEADER].. */
(void)pop3_migration_uidl_sync_if_needed(t->box);
}
return mbox->module_ctx.super.search_init(t, args, sort_program,
wanted_fields, wanted_headers);
}
static void pop3_migration_mailbox_allocated(struct mailbox *box)
{
struct mailbox_vfuncs *v = box->vlast;
struct pop3_migration_mailbox *mbox;
mbox = p_new(box->pool, struct pop3_migration_mailbox, 1);
mbox->module_ctx.super = *v;
box->vlast = &mbox->module_ctx.super;
v->search_init = pop3_migration_mailbox_search_init;
MODULE_CONTEXT_SET(box, pop3_migration_storage_module, mbox);
}
static void pop3_migration_mail_storage_destroy(struct mail_storage *storage)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
if (array_is_created(&mstorage->pop3_uidl_map))
array_free(&mstorage->pop3_uidl_map);
mstorage->module_ctx.super.destroy(storage);
}
static void pop3_migration_mail_storage_created(struct mail_storage *storage)
{
struct pop3_migration_mail_storage *mstorage;
struct mail_storage_vfuncs *v = storage->vlast;
const char *pop3_box_vname;
pop3_box_vname = mail_user_plugin_getenv(storage->user,
"pop3_migration_mailbox");
if (pop3_box_vname == NULL) {
if (storage->user->mail_debug)
i_debug("pop3_migration: No pop3_migration_mailbox setting - disabled");
return;
}
mstorage = p_new(storage->pool, struct pop3_migration_mail_storage, 1);
mstorage->module_ctx.super = *v;
storage->vlast = &mstorage->module_ctx.super;
v->destroy = pop3_migration_mail_storage_destroy;
mstorage->pop3_box_vname = p_strdup(storage->pool, pop3_box_vname);
mstorage->all_mailboxes =
mail_user_plugin_getenv_bool(storage->user,
"pop3_migration_all_mailboxes");
mstorage->ignore_missing_uidls =
mail_user_plugin_getenv_bool(storage->user,
"pop3_migration_ignore_missing_uidls");
mstorage->skip_size_check =
mail_user_plugin_getenv_bool(storage->user,
"pop3_migration_skip_size_check");
MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage);
}
static struct mail_storage_hooks pop3_migration_mail_storage_hooks = {
.mail_allocated = pop3_migration_mail_allocated,
.mailbox_allocated = pop3_migration_mailbox_allocated,
.mail_storage_created = pop3_migration_mail_storage_created
};
void pop3_migration_plugin_init(struct module *module)
{
mail_storage_hooks_add(module, &pop3_migration_mail_storage_hooks);
}
void pop3_migration_plugin_deinit(void)
{
mail_storage_hooks_remove(&pop3_migration_mail_storage_hooks);
}