src/imap/imap-fetch.c

	imap-fetch.c revision 34875ff8cd23cfb80b3dd9784a3e53b97c361c9c
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (c) 2002-2009 Dovecot authors, see the included COPYING file */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "imap-common.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "array.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "buffer.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "istream.h"
1e21e6be70994b1aa9e52ca0e2f51afefca6d0dfTimo Sirainen#include "ostream.h"
e76073ebaf90fa29abfdc364873acf78983949aaTimo Sirainen#include "str.h"
1e21e6be70994b1aa9e52ca0e2f51afefca6d0dfTimo Sirainen#include "message-send.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "message-size.h"
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen#include "imap-date.h"
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen#include "mail-search-build.h"
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen#include "imap-commands.h"
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen#include "imap-quote.h"
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen#include "imap-fetch.h"
86bea1f8bffc2d98196f8655eecea9174c4f458aTimo Sirainen#include "imap-util.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include <stdlib.h>
b82474d60c15409eda71c55971710fd3b12b8a0fTimo Sirainen#include <ctype.h>
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#define BODY_NIL_REPLY \
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen    "\"text\" \"plain\" NIL NIL NIL \"7bit\" 0 0"
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen#define ENVELOPE_NIL_REPLY \
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    "(NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL)"
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenstatic ARRAY_DEFINE(fetch_handlers, struct imap_fetch_handler);
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen
484e12acec34f16e5a8adc001e23ae48f1dda8c7Timo Sirainenstatic int imap_fetch_handler_cmp(const struct imap_fetch_handler *h1,
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen                  const struct imap_fetch_handler *h2)
22535a9e685e29214082878e37a267157044618eTimo Sirainen{
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    return strcmp(h1->name, h2->name);
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen}
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainenvoid imap_fetch_handlers_register(const struct imap_fetch_handler *handlers,
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen                  size_t count)
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen{
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    array_append(&fetch_handlers, handlers, count);
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    array_sort(&fetch_handlers, imap_fetch_handler_cmp);
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen}
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainenstatic int
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainenimap_fetch_handler_bsearch(const char *name, const struct imap_fetch_handler *h)
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen{
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen    return strcmp(name, h->name);
7d6389e4053c2dac1fb37180b5756b00785983dcTimo Sirainen}
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainenbool imap_fetch_init_handler(struct imap_fetch_context *ctx, const char *name,
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen                 const struct imap_arg **args)
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen{
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen    const struct imap_fetch_handler *handler;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    const char *lookup_name, *p;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    for (p = name; i_isalnum(*p) || *p == '-'; p++) ;
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    lookup_name = t_strdup_until(name, p);
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    handler = array_bsearch(&fetch_handlers, lookup_name,
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen                imap_fetch_handler_bsearch);
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    if (handler == NULL) {
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen        client_send_command_error(ctx->cmd,
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen            t_strconcat("Unknown parameter ", name, NULL));
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen        return FALSE;
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen    }
3cf67672fdc87583cb23ce088c95bb5dee60e74dTimo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen    return handler->init(ctx, name, args);
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen}
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainen
25ee72451d16374ed27fdbf829f4ec756c778352Timo Sirainenstruct imap_fetch_context *
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainenimap_fetch_init(struct client_command_context *cmd, struct mailbox *box)
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen{
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    struct client *client = cmd->client;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    struct imap_fetch_context *ctx;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    ctx = p_new(cmd->pool, struct imap_fetch_context, 1);
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    ctx->client = client;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    ctx->cmd = cmd;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    ctx->box = box;
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen
b44650b0f48a4b5f0dc240ed836833a00b643b9fTimo Sirainen    ctx->cur_str = str_new(default_pool, 8192);
a3dd97fb6d92a89c3de0597fed2d4b044c7aeb84Timo Sirainen    p_array_init(&ctx->all_headers, cmd->pool, 64);
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    p_array_init(&ctx->handlers, cmd->pool, 16);
dc9bfb7dc057964238e181d3d8b08751527bb08aTimo Sirainen    p_array_init(&ctx->tmp_keywords, cmd->pool,
473080c7c0d25ddfdf77e7dfa0ba8f73c6c669d5Timo Sirainen             client->keywords.announce_count + 8);
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    ctx->line_finished = TRUE;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    return ctx;
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen}
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainenbool imap_fetch_add_changed_since(struct imap_fetch_context *ctx,
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen                  uint64_t modseq)
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen{
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    struct mail_search_arg *search_arg;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    search_arg = p_new(ctx->search_args->pool, struct mail_search_arg, 1);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    search_arg->type = SEARCH_MODSEQ;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen    search_arg->value.modseq =
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen        p_new(ctx->cmd->pool, struct mail_search_modseq, 1);
1e21e6be70994b1aa9e52ca0e2f51afefca6d0dfTimo Sirainen    search_arg->value.modseq->modseq = modseq + 1;
1e21e6be70994b1aa9e52ca0e2f51afefca6d0dfTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen    search_arg->next = ctx->search_args->args->next;
    ctx->search_args->args->next = search_arg;

    return imap_fetch_init_handler(ctx, "MODSEQ", NULL);
}

#undef imap_fetch_add_handler
void imap_fetch_add_handler(struct imap_fetch_context *ctx,
                bool buffered, bool want_deinit,
                const char *name, const char *nil_reply,
                imap_fetch_handler_t *handler, void *context)
{
    /* partially because of broken clients, but also partially because
       it potentially can make client implementations faster, we have a
       buffered parameter which basically means that the handler promises
       to write the output in ctx->cur_str. The cur_str is then sent to
       client before calling any non-buffered handlers.

       We try to keep the handler registration order the same as the
       client requested them. This is especially useful to get UID
       returned first, which some clients rely on..
    */
    const struct imap_fetch_context_handler *handlers;
    struct imap_fetch_context_handler h;
    unsigned int i, size;

    if (context == NULL) {
        /* don't allow duplicate handlers */
        handlers = array_get(&ctx->handlers, &size);

        for (i = 0; i < size; i++) {
            if (handlers[i].handler == handler &&
                handlers[i].context == NULL)
                return;
        }
    }

    memset(&h, 0, sizeof(h));
    h.handler = handler;
    h.context = context;
    h.buffered = buffered;
    h.want_deinit = want_deinit;
    h.name = p_strdup(ctx->cmd->pool, name);
    h.nil_reply = p_strdup(ctx->cmd->pool, nil_reply);

    if (!buffered)
        array_append(&ctx->handlers, &h, 1);
    else {
        array_insert(&ctx->handlers, ctx->buffered_handlers_count,
                 &h, 1);
                ctx->buffered_handlers_count++;
    }
}

static void
expunges_drop_known(struct imap_fetch_context *ctx, struct mail *mail,
            ARRAY_TYPE(seq_range) *expunges)
{
    const uint32_t *seqs, *uids;
    unsigned int i, count;

    seqs = array_get(ctx->qresync_sample_seqset, &count);
    uids = array_idx(ctx->qresync_sample_uidset, 0);
    i_assert(array_count(ctx->qresync_sample_uidset) == count);
    i_assert(count > 0);

    /* FIXME: we could do removals from the middle as well */
    for (i = 0; i < count; i++) {
        mail_set_seq(mail, seqs[i]);
        if (uids[i] != mail->uid)
            break;
    }
    if (i > 0)
        seq_range_array_remove_range(expunges, 1, uids[i-1]);
}

static int get_expunges_fallback(struct imap_fetch_context *ctx,
                 const ARRAY_TYPE(seq_range) *uids,
                 ARRAY_TYPE(seq_range) *expunges)
{
    struct mailbox_transaction_context *trans;
    struct mail_search_args *search_args;
    struct mail_search_context *search_ctx;
    struct mail *mail;
    const struct seq_range *uid_range;
    struct mailbox_status status;
    unsigned int i, count;
    uint32_t next_uid;
    int ret = 0;

    uid_range = array_get(uids, &count);
    i_assert(count > 0);
    i = 0;
    next_uid = uid_range[0].seq1;

    /* search UIDs in given range */
    search_args = mail_search_build_init();
    search_args->args = p_new(search_args->pool, struct mail_search_arg, 1);
    search_args->args->type = SEARCH_UIDSET;
    i_array_init(&search_args->args->value.seqset, array_count(uids));
    array_append_array(&search_args->args->value.seqset, uids);

    trans = mailbox_transaction_begin(ctx->box, 0);
    mail = mail_alloc(trans, 0, NULL);
    search_ctx = mailbox_search_init(trans, search_args, NULL);
    mail_search_args_unref(&search_args);

    while (mailbox_search_next(search_ctx, mail) > 0) {
        if (mail->uid == next_uid) {
            if (next_uid < uid_range[i].seq2)
                next_uid++;
            else if (++i < count)
                next_uid = uid_range[++i].seq1;
            else
                break;
        } else {
            /* next_uid .. mail->uid-1 are expunged */
            i_assert(mail->uid > next_uid);
            while (mail->uid > uid_range[i].seq2) {
                seq_range_array_add_range(expunges, next_uid,
                              uid_range[i].seq2);
                i++;
                i_assert(i < count);
                next_uid = uid_range[i].seq1;
            }
            if (next_uid != mail->uid) {
                seq_range_array_add_range(expunges, next_uid,
                              mail->uid - 1);
            }
            if (uid_range[i].seq2 == mail->uid)
                next_uid = uid_range[++i].seq1;
            else
                next_uid = mail->uid + 1;
        }
    }
    if (i < count) {
        i_assert(next_uid <= uid_range[i].seq2);
        seq_range_array_add_range(expunges, next_uid,
                      uid_range[i].seq2);
        i++;
    }
    for (; i < count; i++) {
        seq_range_array_add_range(expunges, uid_range[i].seq1,
                      uid_range[i].seq2);
    }

    mailbox_get_status(ctx->box, STATUS_UIDNEXT, &status);
    seq_range_array_remove_range(expunges, status.uidnext, (uint32_t)-1);

    if (mailbox_search_deinit(&search_ctx) < 0)
        ret = -1;

    if (ret == 0 && ctx->qresync_sample_seqset != NULL)
        expunges_drop_known(ctx, mail, expunges);

    mail_free(&mail);
    (void)mailbox_transaction_commit(&trans);
    return ret;
}

static void
mailbox_expunge_to_range(const ARRAY_TYPE(mailbox_expunge_rec) *input,
             ARRAY_TYPE(seq_range) *output)
{
    const struct mailbox_expunge_rec *expunges;
    unsigned int i, count;

    expunges = array_get(input, &count);
    for (i = 0; i < count; i++)
        seq_range_array_add(output, 0, expunges[i].uid);
}

static int
imap_fetch_send_vanished(struct imap_fetch_context *ctx)
{
    const struct mail_search_arg *uidarg = ctx->search_args->args;
    const struct mail_search_arg *modseqarg = uidarg->next;
    const ARRAY_TYPE(seq_range) *uids = &uidarg->value.seqset;
    uint64_t modseq = modseqarg->value.modseq->modseq;
    ARRAY_TYPE(mailbox_expunge_rec) expunges;
    ARRAY_TYPE(seq_range) expunges_range;
    string_t *str;
    int ret = 0;

    i_array_init(&expunges, array_count(uids));
    i_array_init(&expunges_range, array_count(uids));
    if (mailbox_get_expunges(ctx->box, modseq, uids, &expunges))
        mailbox_expunge_to_range(&expunges, &expunges_range);
    else {
        /* return all expunged UIDs */
        if (get_expunges_fallback(ctx, uids, &expunges_range) < 0) {
            array_clear(&expunges_range);
            ret = -1;
        }
    }
    if (array_count(&expunges_range) > 0) {
        str = str_new(default_pool, 128);
        str_append(str, "* VANISHED (EARLIER) ");
        imap_write_seq_range(str, &expunges_range);
        str_append(str, "\r\n");
        o_stream_send(ctx->client->output, str_data(str), str_len(str));
        str_free(&str);
    }
    array_free(&expunges);
    array_free(&expunges_range);
    return ret;
}

int imap_fetch_begin(struct imap_fetch_context *ctx)
{
    const void *data;

    if (ctx->send_vanished) {
        if (imap_fetch_send_vanished(ctx) < 0) {
            ctx->failed = TRUE;
            return -1;
        }
    }

    if (ctx->flags_update_seen) {
        if (mailbox_is_readonly(ctx->box))
            ctx->flags_update_seen = FALSE;
        else if (!ctx->flags_have_handler) {
            ctx->flags_show_only_seen_changes = TRUE;
            (void)imap_fetch_init_handler(ctx, "FLAGS", NULL);
        }
    }

    if (array_count(&ctx->all_headers) > 0 &&
        ((ctx->fetch_data & (MAIL_FETCH_STREAM_HEADER |
                 MAIL_FETCH_STREAM_BODY)) == 0)) {
        (void)array_append_space(&ctx->all_headers);

        data = array_idx(&ctx->all_headers, 0);
        ctx->all_headers_ctx =
            mailbox_header_lookup_init(ctx->box, data);
    }

    if ((ctx->fetch_data &
         (MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY)) != 0)
        ctx->fetch_data |= MAIL_FETCH_NUL_STATE;

    ctx->trans = mailbox_transaction_begin(ctx->box,
        MAILBOX_TRANSACTION_FLAG_HIDE);
    ctx->select_counter = ctx->client->select_counter;
    ctx->mail = mail_alloc(ctx->trans, ctx->fetch_data,
                   ctx->all_headers_ctx);

    /* Delayed uidset -> seqset conversion. VANISHED needs the uidset. */
    mail_search_args_init(ctx->search_args, ctx->box, TRUE,
                  &ctx->cmd->client->search_saved_uidset);
    ctx->search_ctx =
        mailbox_search_init(ctx->trans, ctx->search_args, NULL);
    return 0;
}

static int imap_fetch_flush_buffer(struct imap_fetch_context *ctx)
{
    const unsigned char *data;
    size_t len;

    data = str_data(ctx->cur_str);
    len = str_len(ctx->cur_str);

    if (len == 0)
        return 0;

    /* there's an extra space at the end if we added any fetch items
       to buffer */
    if (data[len-1] == ' ') {
        len--;
        ctx->first = FALSE;
    }

    if (o_stream_send(ctx->client->output, data, len) < 0)
        return -1;

    str_truncate(ctx->cur_str, 0);
    return 0;
}

static int imap_fetch_send_nil_reply(struct imap_fetch_context *ctx)
{
    const struct imap_fetch_context_handler *handler;

    if (!ctx->first)
        str_append_c(ctx->cur_str, ' ');

    handler = array_idx(&ctx->handlers, ctx->cur_handler);
    str_printfa(ctx->cur_str, "%s %s ",
            handler->name, handler->nil_reply);

    if (!handler->buffered) {
        if (imap_fetch_flush_buffer(ctx) < 0)
            return -1;
    }
    return 0;
}

static int imap_fetch_more_int(struct imap_fetch_context *ctx)
{
    struct client *client = ctx->client;
    const struct imap_fetch_context_handler *handlers;
    unsigned int count;
    int ret;

    if (ctx->cont_handler != NULL) {
        ret = ctx->cont_handler(ctx);
        if (ret == 0)
            return 0;

        if (ret < 0) {
            if (client->output->closed)
                return -1;

            if (ctx->cur_mail->expunged) {
                /* not an error, just lost it. */
                ctx->partial_fetch = TRUE;
                if (imap_fetch_send_nil_reply(ctx) < 0)
                    return -1;
            } else {
                return -1;
            }
        }

        ctx->cont_handler = NULL;
        ctx->cur_offset = 0;
                ctx->cur_handler++;
        if (ctx->cur_input != NULL)
            i_stream_unref(&ctx->cur_input);
    }

    handlers = array_get(&ctx->handlers, &count);
    for (;;) {
        if (o_stream_get_buffer_used_size(client->output) >=
            CLIENT_OUTPUT_OPTIMAL_SIZE) {
            ret = o_stream_flush(client->output);
            if (ret <= 0)
                return ret;
        }

        if (ctx->cur_mail == NULL) {
            if (ctx->cmd->cancel)
                return 1;

            if (mailbox_search_next(ctx->search_ctx,
                        ctx->mail) <= 0)
                break;
            ctx->cur_mail = ctx->mail;

            str_printfa(ctx->cur_str, "* %u FETCH (",
                    ctx->cur_mail->seq);
            ctx->first = TRUE;
            ctx->line_finished = FALSE;
        }

        for (; ctx->cur_handler < count; ctx->cur_handler++) {
            if (str_len(ctx->cur_str) > 0 &&
                !handlers[ctx->cur_handler].buffered) {
                /* first non-buffered handler.
                   flush the buffer. */
                ctx->line_partial = TRUE;
                if (imap_fetch_flush_buffer(ctx) < 0)
                    return -1;
            }

            i_assert(ctx->cur_input == NULL);
            T_BEGIN {
                const struct imap_fetch_context_handler *h =
                    &handlers[ctx->cur_handler];

                ret = h->handler(ctx, ctx->cur_mail,
                         h->context);
            } T_END;

            if (ret == 0)
                return 0;

            if (ret < 0) {
                if (ctx->cur_mail->expunged) {
                    /* not an error, just lost it. */
                    ctx->partial_fetch = TRUE;
                    if (imap_fetch_send_nil_reply(ctx) < 0)
                        return -1;
                } else {
                    i_assert(ret < 0 ||
                         ctx->cont_handler != NULL);
                    return -1;
                }
            }

            ctx->cont_handler = NULL;
            ctx->cur_offset = 0;
            if (ctx->cur_input != NULL)
                i_stream_unref(&ctx->cur_input);
        }

        if (str_len(ctx->cur_str) > 0) {
            /* no non-buffered handlers */
            if (imap_fetch_flush_buffer(ctx) < 0)
                return -1;
        }

        ctx->line_finished = TRUE;
        ctx->line_partial = FALSE;
        if (o_stream_send(client->output, ")\r\n", 3) < 0)
            return -1;
        client->last_output = ioloop_time;

        ctx->cur_mail = NULL;
        ctx->cur_handler = 0;
    }

    return 1;
}

int imap_fetch_more(struct imap_fetch_context *ctx)
{
    int ret;

    i_assert(ctx->client->output_lock == NULL ||
         ctx->client->output_lock == ctx->cmd);

    ret = imap_fetch_more_int(ctx);
    if (ret < 0)
        ctx->failed = TRUE;
    if (ctx->line_partial) {
        /* nothing can be sent until FETCH is finished */
        ctx->client->output_lock = ctx->cmd;
    }
    return ret;
}

int imap_fetch_deinit(struct imap_fetch_context *ctx)
{
    const struct imap_fetch_context_handler *handlers;
    unsigned int i, count;

    handlers = array_get(&ctx->handlers, &count);
    for (i = 0; i < count; i++) {
        if (handlers[i].want_deinit)
            handlers[i].handler(ctx, NULL, handlers[i].context);
    }

    if (!ctx->line_finished) {
        if (imap_fetch_flush_buffer(ctx) < 0)
            ctx->failed = TRUE;
        if (o_stream_send(ctx->client->output, ")\r\n", 3) < 0)
            ctx->failed = TRUE;
    }
    str_free(&ctx->cur_str);

    if (ctx->cur_input != NULL)
        i_stream_unref(&ctx->cur_input);

    if (ctx->mail != NULL)
        mail_free(&ctx->mail);

    mail_search_args_unref(&ctx->search_args);
    if (ctx->search_ctx != NULL) {
        if (mailbox_search_deinit(&ctx->search_ctx) < 0)
            ctx->failed = TRUE;
    }
    if (ctx->all_headers_ctx != NULL)
        mailbox_header_lookup_unref(&ctx->all_headers_ctx);

    if (ctx->trans != NULL) {
        /* even if something failed, we want to commit changes to
           cache, as well as possible \Seen flag changes for FETCH
           replies we returned so far. */
        if (mailbox_transaction_commit(&ctx->trans) < 0)
            ctx->failed = TRUE;
    }
    return ctx->failed ? -1 : 0;
}

static int fetch_body(struct imap_fetch_context *ctx, struct mail *mail,
              void *context ATTR_UNUSED)
{
    const char *body;

    if (mail_get_special(mail, MAIL_FETCH_IMAP_BODY, &body) < 0)
        return -1;

    if (ctx->first)
        ctx->first = FALSE;
    else {
        if (o_stream_send(ctx->client->output, " ", 1) < 0)
            return -1;
    }

    if (o_stream_send(ctx->client->output, "BODY (", 6) < 0 ||
        o_stream_send_str(ctx->client->output, body) < 0 ||
        o_stream_send(ctx->client->output, ")", 1) < 0)
        return -1;
    return 1;
}

static bool fetch_body_init(struct imap_fetch_context *ctx, const char *name,
                const struct imap_arg **args)
{
    if (name[4] == '\0') {
        ctx->fetch_data |= MAIL_FETCH_IMAP_BODY;
        imap_fetch_add_handler(ctx, FALSE, FALSE, name,
                       "("BODY_NIL_REPLY")", fetch_body, NULL);
        return TRUE;
    }
    return fetch_body_section_init(ctx, name, args);
}

static int fetch_bodystructure(struct imap_fetch_context *ctx,
                   struct mail *mail, void *context ATTR_UNUSED)
{
    const char *bodystructure;

    if (mail_get_special(mail, MAIL_FETCH_IMAP_BODYSTRUCTURE,
                 &bodystructure) < 0)
        return -1;

    if (ctx->first)
        ctx->first = FALSE;
    else {
        if (o_stream_send(ctx->client->output, " ", 1) < 0)
            return -1;
    }

    if (o_stream_send(ctx->client->output, "BODYSTRUCTURE (", 15) < 0 ||
        o_stream_send_str(ctx->client->output, bodystructure) < 0 ||
        o_stream_send(ctx->client->output, ")", 1) < 0)
        return -1;

    return 1;
}

static bool
fetch_bodystructure_init(struct imap_fetch_context *ctx, const char *name,
             const struct imap_arg **args ATTR_UNUSED)
{
    ctx->fetch_data |= MAIL_FETCH_IMAP_BODYSTRUCTURE;
    imap_fetch_add_handler(ctx, FALSE, FALSE, name,
                   "("BODY_NIL_REPLY" NIL NIL NIL NIL)",
                   fetch_bodystructure, NULL);
    return TRUE;
}

static int fetch_envelope(struct imap_fetch_context *ctx, struct mail *mail,
              void *context ATTR_UNUSED)
{
    const char *envelope;

    if (mail_get_special(mail, MAIL_FETCH_IMAP_ENVELOPE, &envelope) < 0)
        return -1;

    if (ctx->first)
        ctx->first = FALSE;
    else {
        if (o_stream_send(ctx->client->output, " ", 1) < 0)
            return -1;
    }

    if (o_stream_send(ctx->client->output, "ENVELOPE (", 10) < 0 ||
        o_stream_send_str(ctx->client->output, envelope) < 0 ||
        o_stream_send(ctx->client->output, ")", 1) < 0)
        return -1;
    return 1;
}

static bool
fetch_envelope_init(struct imap_fetch_context *ctx, const char *name,
            const struct imap_arg **args ATTR_UNUSED)
{
    ctx->fetch_data |= MAIL_FETCH_IMAP_ENVELOPE;
    imap_fetch_add_handler(ctx, FALSE, FALSE, name, ENVELOPE_NIL_REPLY,
                   fetch_envelope, NULL);
    return TRUE;
}

static int fetch_flags(struct imap_fetch_context *ctx, struct mail *mail,
               void *context ATTR_UNUSED)
{
    enum mail_flags flags;
    const char *const *keywords;

    flags = mail_get_flags(mail);
    if (ctx->flags_update_seen && (flags & MAIL_SEEN) == 0) {
        /* Add \Seen flag */
        ctx->seen_flags_changed = TRUE;
        flags |= MAIL_SEEN;
        mail_update_flags(mail, MODIFY_ADD, MAIL_SEEN);
    } else if (ctx->flags_show_only_seen_changes) {
        return 1;
    }

    keywords = client_get_keyword_names(ctx->client, &ctx->tmp_keywords,
            mail_get_keyword_indexes(mail));

    str_append(ctx->cur_str, "FLAGS (");
    imap_write_flags(ctx->cur_str, flags, keywords);
    str_append(ctx->cur_str, ") ");
    return 1;
}

static bool
fetch_flags_init(struct imap_fetch_context *ctx, const char *name,
         const struct imap_arg **args ATTR_UNUSED)
{
    ctx->flags_have_handler = TRUE;
    ctx->fetch_data |= MAIL_FETCH_FLAGS;
    imap_fetch_add_handler(ctx, TRUE, FALSE, name, "()", fetch_flags, NULL);
    return TRUE;
}

static int fetch_internaldate(struct imap_fetch_context *ctx, struct mail *mail,
                  void *context ATTR_UNUSED)
{
    time_t date;

    if (mail_get_received_date(mail, &date) < 0)
        return -1;

    str_printfa(ctx->cur_str, "INTERNALDATE \"%s\" ",
            imap_to_datetime(date));
    return 1;
}

static bool
fetch_internaldate_init(struct imap_fetch_context *ctx, const char *name,
            const struct imap_arg **args ATTR_UNUSED)
{
    ctx->fetch_data |= MAIL_FETCH_RECEIVED_DATE;
    imap_fetch_add_handler(ctx, TRUE, FALSE, name,
                   "\"01-Jan-1970 00:00:00 +0000\"",
                   fetch_internaldate, NULL);
    return TRUE;
}

static int fetch_modseq(struct imap_fetch_context *ctx, struct mail *mail,
            void *context ATTR_UNUSED)
{
    uint64_t modseq;

    modseq = mail_get_modseq(mail);
    if (ctx->client->highest_fetch_modseq < modseq)
        ctx->client->highest_fetch_modseq = modseq;
    str_printfa(ctx->cur_str, "MODSEQ (%llu) ",
            (unsigned long long)modseq);
    return 1;
}

static bool
fetch_modseq_init(struct imap_fetch_context *ctx, const char *name,
          const struct imap_arg **args ATTR_UNUSED)
{
    client_enable(ctx->client, MAILBOX_FEATURE_CONDSTORE);
    imap_fetch_add_handler(ctx, TRUE, FALSE, name, NULL,
                   fetch_modseq, NULL);
    return TRUE;
}

static int fetch_uid(struct imap_fetch_context *ctx, struct mail *mail,
             void *context ATTR_UNUSED)
{
    str_printfa(ctx->cur_str, "UID %u ", mail->uid);
    return 1;
}

static bool
fetch_uid_init(struct imap_fetch_context *ctx ATTR_UNUSED, const char *name,
           const struct imap_arg **args ATTR_UNUSED)
{
    imap_fetch_add_handler(ctx, TRUE, FALSE, name, NULL, fetch_uid, NULL);
    return TRUE;
}

static int fetch_guid(struct imap_fetch_context *ctx, struct mail *mail,
              void *context ATTR_UNUSED)
{
    const char *value;

    if (mail_get_special(mail, MAIL_FETCH_GUID, &value) < 0)
        return -1;

    str_append(ctx->cur_str, "X-GUID ");
    imap_quote_append_string(ctx->cur_str, value, FALSE);
    return 1;
}

static bool
fetch_guid_init(struct imap_fetch_context *ctx ATTR_UNUSED, const char *name,
        const struct imap_arg **args ATTR_UNUSED)
{
    imap_fetch_add_handler(ctx, TRUE, FALSE, name, "", fetch_guid, NULL);
    return TRUE;
}

static int fetch_x_mailbox(struct imap_fetch_context *ctx, struct mail *mail,
               void *context ATTR_UNUSED)
{
    const char *str;

    if (mail_get_special(mail, MAIL_FETCH_MAILBOX_NAME, &str) < 0)
        i_panic("mailbox name not returned");
    str_append(ctx->cur_str, "X-MAILBOX ");
    imap_quote_append_string(ctx->cur_str, str, FALSE);
    return 1;
}

static bool
fetch_x_mailbox_init(struct imap_fetch_context *ctx ATTR_UNUSED,
             const char *name,
             const struct imap_arg **args ATTR_UNUSED)
{
    imap_fetch_add_handler(ctx, TRUE, FALSE, name, NULL,
                   fetch_x_mailbox, NULL);
    return TRUE;
}

static int fetch_x_savedate(struct imap_fetch_context *ctx, struct mail *mail,
                void *context ATTR_UNUSED)
{
    time_t date;

    if (mail_get_save_date(mail, &date) < 0)
        return -1;

    str_printfa(ctx->cur_str, "X-SAVEDATE \"%s\" ",
            imap_to_datetime(date));
    return 1;
}

static bool
fetch_x_savedate_init(struct imap_fetch_context *ctx, const char *name,
              const struct imap_arg **args ATTR_UNUSED)
{
    ctx->fetch_data |= MAIL_FETCH_SAVE_DATE;
    imap_fetch_add_handler(ctx, TRUE, FALSE, name,
                   "\"01-Jan-1970 00:00:00 +0000\"",
                   fetch_x_savedate, NULL);
    return TRUE;
}

static const struct imap_fetch_handler
imap_fetch_default_handlers[] = {
    { "BODY", fetch_body_init },
    { "BODYSTRUCTURE", fetch_bodystructure_init },
    { "ENVELOPE", fetch_envelope_init },
    { "FLAGS", fetch_flags_init },
    { "INTERNALDATE", fetch_internaldate_init },
    { "MODSEQ", fetch_modseq_init },
    { "RFC822", fetch_rfc822_init },
    { "UID", fetch_uid_init },
    { "X-GUID", fetch_guid_init },
    { "X-MAILBOX", fetch_x_mailbox_init },
    { "X-SAVEDATE", fetch_x_savedate_init }
};

void imap_fetch_handlers_init(void)
{
    i_array_init(&fetch_handlers, 32);
    imap_fetch_handlers_register(imap_fetch_default_handlers,
                     N_ELEMENTS(imap_fetch_default_handlers));
}

void imap_fetch_handlers_deinit(void)
{
    array_free(&fetch_handlers);
}