src/doveadm/doveadm-mail-index.c

	doveadm-mail-index.c revision 6a262c9bd8f57cf1e57112e0522dbdab28ae8c29
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2010-2015 Dovecot authors, see the included COPYING file */
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "lib.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "str.h"
bdd36cfdba3ff66d25570a9ff568d69e1eb543cfTimo Sirainen#include "strescape.h"
b1f37113a5760bee842c5a7678bb5fa6f5bd8b60Timo Sirainen#include "net.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "write-full.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "mail-namespace.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "mail-storage.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "mail-search-build.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "mailbox-list-iter.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "doveadm-settings.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "doveadm-mail.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include <stdio.h>
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#define INDEXER_SOCKET_NAME "indexer"
530b5909a99e7575156496f1b8e4d1ff5b058484Timo Sirainen#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
530b5909a99e7575156496f1b8e4d1ff5b058484Timo Sirainen
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainenstruct index_cmd_context {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    struct doveadm_mail_cmd_context ctx;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    int queue_fd;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    unsigned int max_recent_msgs;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    unsigned int queue:1;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    unsigned int have_wildcards:1;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen};
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainenstatic int cmd_index_box_precache(struct mailbox *box)
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen{
bd63b5b860658b01b1f46f26d406e1e4a9dc019aTimo Sirainen    struct mailbox_status status;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    struct mailbox_transaction_context *trans;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    struct mail_search_args *search_args;
70ac869db925653b57f721cd045c467612fd5ee9Timo Sirainen    struct mail_search_context *ctx;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    struct mail *mail;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    struct mailbox_metadata metadata;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    uint32_t seq;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    unsigned int counter = 0, max;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    int ret = 0;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen                 &metadata) < 0) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        i_error("Mailbox %s: Precache-fields lookup failed: %s",
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen            mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen                   &status) < 0) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        i_error("Mailbox %s: Status lookup failed: %s",
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen            mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        return -1;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    seq = status.last_cached_seq + 1;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    if (seq > status.messages) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        if (doveadm_verbose) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen            i_info("%s: Cache is already up to date",
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen                   mailbox_get_vname(box));
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        return 0;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    if (doveadm_verbose) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        i_info("%s: Caching mails seq=%u..%u",
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen               mailbox_get_vname(box), seq, status.messages);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    search_args = mail_search_build_init();
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    mail_search_build_add_seqset(search_args, seq, status.messages);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    ctx = mailbox_search_init(trans, search_args, NULL,
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen                  metadata.precache_fields, NULL);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    mail_search_args_unref(&search_args);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    max = status.messages - seq + 1;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    while (mailbox_search_next(ctx, &mail)) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        mail_precache(mail);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        if (doveadm_verbose && ++counter % 100 == 0) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen            printf("\r%u/%u", counter, max);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen            fflush(stdout);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    if (doveadm_verbose)
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        printf("\r%u/%u\n", counter, max);
b1f37113a5760bee842c5a7678bb5fa6f5bd8b60Timo Sirainen    if (mailbox_search_deinit(&ctx) < 0) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        i_error("Mailbox %s: Mail search failed: %s",
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen            mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        ret = -1;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    if (mailbox_transaction_commit(&trans) < 0) {
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        i_error("Mailbox %s: Transaction commit failed: %s",
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen            mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        ret = -1;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    }
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    return ret;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen}
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainenstatic int
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainencmd_index_box(struct index_cmd_context *ctx, const struct mailbox_info *info)
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen{
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    struct mailbox *box;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    struct mailbox_status status;
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    int ret = 0;
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen
009217abb57a24a4076092e8e4e165545747839eStephan Bosch    box = mailbox_alloc(info->ns->list, info->vname,
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen                MAILBOX_FLAG_IGNORE_ACLS);
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen    if (ctx->max_recent_msgs != 0) {
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        /* index only if there aren't too many recent messages.
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen           don't bother syncing the mailbox, that alone can take a
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen           while with large maildirs. */
530b5909a99e7575156496f1b8e4d1ff5b058484Timo Sirainen        if (mailbox_open(box) < 0) {
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen            i_error("Opening mailbox %s failed: %s", info->vname,
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen                mail_storage_get_last_error(mailbox_get_storage(box), NULL));
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen            doveadm_mail_failed_mailbox(&ctx->ctx, box);
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen            mailbox_free(&box);
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen            return -1;
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        }
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        mailbox_get_open_status(box, STATUS_RECENT, &status);
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        if (status.recent > ctx->max_recent_msgs) {
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen            mailbox_free(&box);
530b5909a99e7575156496f1b8e4d1ff5b058484Timo Sirainen            return 0;
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        }
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen    }
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen
009217abb57a24a4076092e8e4e165545747839eStephan Bosch    if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        i_error("Syncing mailbox %s failed: %s", info->vname,
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen            mail_storage_get_last_error(mailbox_get_storage(box), NULL));
009217abb57a24a4076092e8e4e165545747839eStephan Bosch        doveadm_mail_failed_mailbox(&ctx->ctx, box);
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        ret = -1;
7c925149e49f7cce41c90d562ff3835b66ddca29Timo Sirainen    } else {
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        if (cmd_index_box_precache(box) < 0) {
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen            doveadm_mail_failed_mailbox(&ctx->ctx, box);
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen            ret = -1;
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen        }
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen    }
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen    mailbox_free(&box);
250a06cde50c4004ac1cc72279b75a229a7f90f5Timo Sirainen    return ret;
009217abb57a24a4076092e8e4e165545747839eStephan Bosch}
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainenstatic void index_queue_connect(struct index_cmd_context *ctx)
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen{
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen    const char *path;
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen    path = t_strconcat(doveadm_settings->base_dir,
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen               "/"INDEXER_SOCKET_NAME, NULL);
6a33ecc05e60b511c1ae176ef113546e1012c18fTimo Sirainen    ctx->queue_fd = net_connect_unix(path);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    if (ctx->queue_fd == -1)
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        i_fatal("net_connect_unix(%s) failed: %m", path);
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen    if (write_full(ctx->queue_fd, INDEXER_HANDSHAKE,
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen               strlen(INDEXER_HANDSHAKE)) < 0)
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen        i_fatal("write(indexer) failed: %m");
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen}
250a06cde50c4004ac1cc72279b75a229a7f90f5Timo Sirainen
250a06cde50c4004ac1cc72279b75a229a7f90f5Timo Sirainenstatic void cmd_index_queue(struct index_cmd_context *ctx,
250a06cde50c4004ac1cc72279b75a229a7f90f5Timo Sirainen                struct mail_user *user, const char *mailbox)
250a06cde50c4004ac1cc72279b75a229a7f90f5Timo Sirainen{
250a06cde50c4004ac1cc72279b75a229a7f90f5Timo Sirainen    if (ctx->queue_fd == -1)
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi        index_queue_connect(ctx);
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi    T_BEGIN {
23bdbb7b1831785c6ba6df190f6369da882d2b9dTimo Sirainen        string_t *str = t_str_new(256);
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi        str_append(str, "APPEND\t0\t");
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi        str_append_tabescaped(str, user->username);
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi        str_append_c(str, '\t');
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi        str_append_tabescaped(str, mailbox);
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi        str_printfa(str, "\t%u\n", ctx->max_recent_msgs);
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi        if (write_full(ctx->queue_fd, str_data(str), str_len(str)) < 0)
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi            i_fatal("write(indexer) failed: %m");
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi    } T_END;
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi}
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomistatic int
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomicmd_index_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
a23210844b06c53ebca34b162f3a268d5f1cd4d1Aki Tuomi{
    struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;
    const enum mailbox_list_iter_flags iter_flags =
        MAILBOX_LIST_ITER_NO_AUTO_BOXES |
        MAILBOX_LIST_ITER_RETURN_NO_FLAGS |
        MAILBOX_LIST_ITER_STAR_WITHIN_NS;
    const enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL;
    struct mailbox_list_iterate_context *iter;
    const struct mailbox_info *info;
    unsigned int i;
    int ret = 0;

    if (ctx->queue && !ctx->have_wildcards) {
        /* we can do this quickly without going through the mailboxes */
        for (i = 0; _ctx->args[i] != NULL; i++)
            cmd_index_queue(ctx, user, _ctx->args[i]);
        return 0;
    }

    iter = mailbox_list_iter_init_namespaces(user->namespaces, _ctx->args,
                         ns_mask, iter_flags);
    while ((info = mailbox_list_iter_next(iter)) != NULL) {
        if ((info->flags & (MAILBOX_NOSELECT |
                    MAILBOX_NONEXISTENT)) == 0) T_BEGIN {
            if (ctx->queue)
                cmd_index_queue(ctx, user, info->vname);
            else {
                if (cmd_index_box(ctx, info) < 0)
                    ret = -1;
            }
        } T_END;
    }
    if (mailbox_list_iter_deinit(&iter) < 0) {
        i_error("Listing mailboxes failed: %s",
            mailbox_list_get_last_error(user->namespaces->list, NULL));
        doveadm_mail_failed_error(_ctx, MAIL_ERROR_TEMP);
        ret = -1;
    }
    return ret;
}

static void cmd_index_init(struct doveadm_mail_cmd_context *_ctx,
               const char *const args[])
{
    struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;
    unsigned int i;

    if (args[0] == NULL)
        doveadm_mail_help_name("index");
    for (i = 0; args[i] != NULL; i++) {
        if (strchr(args[i], '*') != NULL ||
            strchr(args[i], '%') != NULL) {
            ctx->have_wildcards = TRUE;
            break;
        }
    }
}

static void cmd_index_deinit(struct doveadm_mail_cmd_context *_ctx)
{
    struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;

    if (ctx->queue_fd != -1) {
        net_disconnect(ctx->queue_fd);
        ctx->queue_fd = -1;
    }
}

static bool
cmd_index_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
{
    struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;

    switch (c) {
    case 'q':
        ctx->queue = TRUE;
        break;
    case 'n':
        if (str_to_uint(optarg, &ctx->max_recent_msgs) < 0) {
            i_fatal_status(EX_USAGE,
                "Invalid -n parameter number: %s", optarg);
        }
        break;
    default:
        return FALSE;
    }
    return TRUE;
}

static struct doveadm_mail_cmd_context *cmd_index_alloc(void)
{
    struct index_cmd_context *ctx;

    ctx = doveadm_mail_cmd_alloc(struct index_cmd_context);
    ctx->queue_fd = -1;
    ctx->ctx.getopt_args = "qn:";
    ctx->ctx.v.parse_arg = cmd_index_parse_arg;
    ctx->ctx.v.init = cmd_index_init;
    ctx->ctx.v.deinit = cmd_index_deinit;
    ctx->ctx.v.run = cmd_index_run;
    return &ctx->ctx;
}

struct doveadm_mail_cmd cmd_index = {
    cmd_index_alloc, "index", "[-q] [-n <max recent>] <mailbox mask>"
};