plugins/expire/doveadm-expire.c

	doveadm-expire.c revision bd63b5b860658b01b1f46f26d406e1e4a9dc019a
1029N/A/* Copyright (c) 2005-2012 Dovecot authors, see the included COPYING file */
1029N/A
1549N/A#include "lib.h"
1029N/A#include "module-dir.h"
1549N/A#include "str.h"
1029N/A#include "hash.h"
1029N/A#include "dict.h"
1042N/A#include "imap-match.h"
1042N/A#include "expire-set.h"
1042N/A#include "mail-search.h"
1042N/A#include "doveadm-settings.h"
1042N/A#include "doveadm-mail.h"
1042N/A
1042N/A#define DOVEADM_EXPIRE_MAIL_CMD_CONTEXT(obj) \
1042N/A    MODULE_CONTEXT(obj, doveadm_expire_mail_cmd_module)
1042N/A
1042N/Aenum expire_user_state {
1042N/A    EXPIRE_USER_STATE_NONEXISTENT = 0,
1042N/A    EXPIRE_USER_STATE_EXISTS = 1,
1042N/A    EXPIRE_USER_STATE_SEEN = 2
1042N/A};
1042N/A
1042N/Astruct expire_query {
1042N/A    const char *mailbox;
1029N/A    struct imap_match_glob *glob;
1029N/A    time_t before_time;
1029N/A};
1029N/A
1029N/Astruct doveadm_expire_mail_cmd_context {
1029N/A    union doveadm_mail_cmd_module_context module_ctx;
1549N/A
1029N/A    struct dict *dict;
1029N/A    struct dict_transaction_context *trans;
1549N/A    struct dict_iterate_context *iter;
1549N/A
1549N/A    /* username => enum expire_user_state */
1029N/A    HASH_TABLE(char *, void *) user_states;
1029N/A    ARRAY(struct expire_query) queries;
1029N/A    time_t oldest_before_time;
1549N/A    bool delete_nonexistent_users;
1549N/A};
1029N/A
1029N/Aconst char *doveadm_expire_plugin_version = DOVECOT_ABI_VERSION;
1029N/A
1029N/Avoid doveadm_expire_plugin_init(struct module *module);
1029N/Avoid doveadm_expire_plugin_deinit(void);
1549N/A
1029N/Astatic MODULE_CONTEXT_DEFINE_INIT(doveadm_expire_mail_cmd_module,
1029N/A                  &doveadm_mail_cmd_module_register);
1552N/Astatic void (*next_hook_doveadm_mail_init)(struct doveadm_mail_cmd_context *ctx);
1029N/A
1029N/Astatic bool
1029N/Adoveadm_expire_mail_match_mailbox(struct doveadm_expire_mail_cmd_context *ectx,
1181N/A                  const char *mailbox, time_t oldest_savedate)
1029N/A{
1029N/A    const struct expire_query *query;
1029N/A
1029N/A    array_foreach(&ectx->queries, query) {
1029N/A        if (oldest_savedate >= query->before_time)
1029N/A            continue;
1029N/A
1029N/A        if (query->glob == NULL) {
1029N/A            if (strcmp(query->mailbox, mailbox) == 0)
1029N/A                return TRUE;
1029N/A        } else {
1029N/A            if (imap_match(query->glob, mailbox) == IMAP_MATCH_YES)
1029N/A                return TRUE;
1029N/A        }
1029N/A    }
1029N/A    return FALSE;
1029N/A}
1196N/A
1196N/Astatic int
1196N/Adoveadm_expire_mail_want(struct doveadm_mail_cmd_context *ctx,
1196N/A             const char *dict_key, time_t oldest_savedate,
1029N/A             const char **username_r)
1029N/A{
1029N/A    struct doveadm_expire_mail_cmd_context *ectx =
1479N/A        DOVEADM_EXPIRE_MAIL_CMD_CONTEXT(ctx);
1479N/A    const char *username, *mailbox;
1479N/A    enum expire_user_state state;
1196N/A    char *orig_username;
1196N/A    void *value;
1196N/A
1196N/A    /* dict_key = DICT_EXPIRE_PREFIX<user>/<mailbox> */
1425N/A    username = dict_key + strlen(DICT_EXPIRE_PREFIX);
1196N/A    mailbox = strchr(username, '/');
1181N/A    if (mailbox == NULL) {
1181N/A        /* invalid record, ignore */
1181N/A        i_error("expire: Invalid key: %s", dict_key);
1029N/A        return -1;
1029N/A    }
1176N/A    username = t_strdup_until(username, mailbox++);
1029N/A
1029N/A    if (!hash_table_lookup_full(ectx->user_states, username,
1031N/A                    &orig_username, &value)) {
1031N/A        /* user no longer exists, delete the record */
1031N/A        return -1;
1031N/A    }
1031N/A    state = POINTER_CAST_TO(value, enum expire_user_state);
1031N/A    switch (state) {
1031N/A    case EXPIRE_USER_STATE_NONEXISTENT:
1031N/A        i_unreached();
1031N/A    case EXPIRE_USER_STATE_EXISTS:
1031N/A        break;
1031N/A    case EXPIRE_USER_STATE_SEEN:
1031N/A        /* seen this user already, skip the record */
1181N/A        return 0;
1181N/A    }
1181N/A
1181N/A    if (!doveadm_expire_mail_match_mailbox(ectx, mailbox,
1181N/A                           oldest_savedate)) {
1181N/A        /* this mailbox doesn't have any matching messages */
1181N/A        return 0;
    }
    state = EXPIRE_USER_STATE_SEEN;
    hash_table_update(ectx->user_states, orig_username,
              POINTER_CAST(state));
    *username_r = orig_username;
    return 1;
}

static int
doveadm_expire_mail_cmd_get_next_user(struct doveadm_mail_cmd_context *ctx,
                      const char **username_r)
{
    struct doveadm_expire_mail_cmd_context *ectx =
        DOVEADM_EXPIRE_MAIL_CMD_CONTEXT(ctx);
    const char *key, *value;
    unsigned long oldest_savedate;
    int ret;

    while (dict_iterate(ectx->iter, &key, &value)) {
        if (str_to_ulong(value, &oldest_savedate) < 0) {
            /* invalid record */
            i_error("expire: Invalid timestamp: %s", value);
            continue;
        }
        if ((time_t)oldest_savedate > ectx->oldest_before_time) {
            if (doveadm_debug) {
                i_debug("expire: Stopping iteration on key %s "
                    "(%lu > %ld)", key, oldest_savedate,
                    (long)ectx->oldest_before_time);
            }
            break;
        }

        T_BEGIN {
            ret = doveadm_expire_mail_want(ctx, key,
                               oldest_savedate,
                               username_r);
        } T_END;
        if (ret > 0)
            return TRUE;
        if (ret < 0 && ectx->delete_nonexistent_users) {
            /* user has been deleted */
            dict_unset(ectx->trans, key);
        }
    }

    /* finished */
    if (dict_iterate_deinit(&ectx->iter) < 0) {
        i_error("Dictionary iteration failed");
        return -1;
    }
    return 0;
}

static const char *const *doveadm_expire_get_patterns(void)
{
    ARRAY_TYPE(const_string) patterns;
    const char *str;
    char set_name[6+MAX_INT_STRLEN+1];
    unsigned int i;

    t_array_init(&patterns, 16);
    str = doveadm_plugin_getenv("expire");
    for (i = 2; str != NULL; i++) {
        array_append(&patterns, &str, 1);

        if (i_snprintf(set_name, sizeof(set_name), "expire%u", i) < 0)
            i_unreached();
        str = doveadm_plugin_getenv(set_name);
    }
    array_append_zero(&patterns);
    return array_idx(&patterns, 0);
}

static bool
doveadm_expire_get_or_mailboxes(struct doveadm_mail_cmd_context *ctx,
                const struct mail_search_arg *args,
                struct expire_query query)
{
    struct doveadm_expire_mail_cmd_context *ectx =
        DOVEADM_EXPIRE_MAIL_CMD_CONTEXT(ctx);
    const struct mail_search_arg *arg;
    unsigned int query_count;

    query.mailbox = NULL;
    query_count = array_count(&ectx->queries);
    for (arg = args; arg != NULL; arg = arg->next) {
        switch (arg->type) {
        case SEARCH_MAILBOX_GLOB:
            query.glob = imap_match_init(ctx->pool, arg->value.str,
                             TRUE, '/');
            /* fall through */
        case SEARCH_MAILBOX:
            /* require mailbox to be in expire patterns */
            query.mailbox = p_strdup(ctx->pool, arg->value.str);
            array_append(&ectx->queries, &query, 1);
            break;
        default:
            /* there are something else besides mailboxes,
               can't optimize this. */
            array_delete(&ectx->queries, query_count,
                     array_count(&ectx->queries) - query_count);
            return FALSE;
        }
    }
    return query.mailbox != NULL;
}

static bool
doveadm_expire_analyze_and_query(struct doveadm_mail_cmd_context *ctx,
                 const struct mail_search_arg *args)
{
    struct doveadm_expire_mail_cmd_context *ectx =
        DOVEADM_EXPIRE_MAIL_CMD_CONTEXT(ctx);
    const struct mail_search_arg *arg;
    struct expire_query query;
    bool have_or = FALSE;

    memset(&query, 0, sizeof(query));
    query.before_time = (time_t)-1;

    for (arg = args; arg != NULL; arg = arg->next) {
        switch (arg->type) {
        case SEARCH_OR:
            have_or = TRUE;
            break;
        case SEARCH_MAILBOX_GLOB:
            query.glob = imap_match_init(ctx->pool, arg->value.str,
                             TRUE, '/');
            /* fall through */
        case SEARCH_MAILBOX:
            /* require mailbox to be in expire patterns */
            query.mailbox = p_strdup(ctx->pool, arg->value.str);
            break;
        case SEARCH_BEFORE:
            if (arg->value.date_type != MAIL_SEARCH_DATE_TYPE_SAVED)
                break;
            if ((arg->value.search_flags &
                 MAIL_SEARCH_ARG_FLAG_USE_TZ) == 0)
                break;
            query.before_time = arg->value.time;
            break;
        default:
            break;
        }
    }

    if (query.before_time == (time_t)-1) {
        /* no SAVEDBEFORE, can't optimize */
        return FALSE;
    }

    if (query.mailbox != NULL) {
        /* one mailbox */
        array_append(&ectx->queries, &query, 1);
        return TRUE;
    }

    /* no MAILBOX, but check if one of the ORs lists mailboxes */
    if (!have_or)
        return FALSE;

    for (arg = args; arg != NULL; arg = arg->next) {
        if (arg->type == SEARCH_OR &&
            doveadm_expire_get_or_mailboxes(ctx, arg->value.subargs,
                            query))
            return TRUE;
    }
    return FALSE;
}

static time_t
doveadm_expire_analyze_or_query(struct doveadm_mail_cmd_context *ctx,
                const struct mail_search_arg *args)
{
    const struct mail_search_arg *arg;

    /* all of the subqueries must have mailbox and savedbefore */
    for (arg = args; arg != NULL; arg = arg->next) {
        if (arg->type != SEARCH_SUB)
            return FALSE;

        if (!doveadm_expire_analyze_and_query(ctx, arg->value.subargs))
            return FALSE;
    }
    return TRUE;
}

static bool doveadm_expire_analyze_query(struct doveadm_mail_cmd_context *ctx)
{
    struct doveadm_expire_mail_cmd_context *ectx =
        DOVEADM_EXPIRE_MAIL_CMD_CONTEXT(ctx);
    struct mail_search_arg *args = ctx->search_args->args;
    struct expire_set *set;
    const struct expire_query *queries;
    unsigned int i, count;

    i_assert(args != NULL);

    /* we support two kinds of queries:

       1) mailbox-pattern savedbefore <stamp> ...
       2) or 2*(mailbox-pattern savedbefore <stamp> ...)

       mailbox-pattern can be:

       a) mailbox <name>
       b) or 2*(mailbox <name>)
    */
    p_array_init(&ectx->queries, ctx->pool, 8);
    if (!doveadm_expire_analyze_and_query(ctx, args) &&
        (args->type != SEARCH_OR || args->next != NULL ||
         !doveadm_expire_analyze_or_query(ctx, args->value.subargs))) {
        if (doveadm_debug)
            i_debug("expire: Couldn't optimize search query");
        return FALSE;
    }

    /* make sure all mailboxes match expire patterns */
    set = expire_set_init(doveadm_expire_get_patterns());
    queries = array_get(&ectx->queries, &count);
    for (i = 0; i < count; i++) {
        if (!expire_set_lookup(set, queries[i].mailbox)) {
            if (doveadm_debug) {
                i_debug("expire: Couldn't optimize search query: "
                    "mailbox %s not in expire database",
                    queries[i].mailbox);
            }
            break;
        }
    }
    expire_set_deinit(&set);

    return i == count;
}

static void doveadm_expire_mail_cmd_deinit(struct doveadm_mail_cmd_context *ctx)
{
    struct doveadm_expire_mail_cmd_context *ectx =
        DOVEADM_EXPIRE_MAIL_CMD_CONTEXT(ctx);

    if (ectx->iter != NULL) {
        if (dict_iterate_deinit(&ectx->iter) < 0)
            i_error("expire: Dictionary iteration failed");
    }
    if (dict_transaction_commit(&ectx->trans) < 0)
        i_error("expire: Dictionary commit failed");
    dict_deinit(&ectx->dict);
    hash_table_destroy(&ectx->user_states);

    ectx->module_ctx.super.deinit(ctx);
}

static void doveadm_expire_mail_init(struct doveadm_mail_cmd_context *ctx)
{
    struct doveadm_expire_mail_cmd_context *ectx;
    struct dict *dict;
    const struct expire_query *query;
    const char *expire_dict, *username, *value;
    char *username_dup;
    enum expire_user_state state;

    if (ctx->search_args == NULL)
        return;

    expire_dict = doveadm_plugin_getenv("expire_dict");
    if (expire_dict == NULL)
        return;

    if (ctx->iterate_single_user) {
        if (doveadm_debug) {
            i_debug("expire: Iterating only a single user, "
                "ignoring expire database");
        }
        return;
    }

    ectx = p_new(ctx->pool, struct doveadm_expire_mail_cmd_context, 1);
    ectx->module_ctx.super = ctx->v;
    value = doveadm_plugin_getenv("expire_keep_nonexistent_users");
    ectx->delete_nonexistent_users =
        value == NULL || strcmp(value, "yes") != 0;
    MODULE_CONTEXT_SET(ctx, doveadm_expire_mail_cmd_module, ectx);

    /* we can potentially optimize this query. see if the search args
       are valid for optimization. */
    if (!doveadm_expire_analyze_query(ctx))
        return;

    if (doveadm_debug)
        i_debug("expire: Searching only users listed in expire database");

    if (dict_init(expire_dict, DICT_DATA_TYPE_UINT32, "",
              doveadm_settings->base_dir, &dict) < 0) {
        i_error("dict_init(%s) failed, not using it", expire_dict);
        return;
    }

    ectx->oldest_before_time = (time_t)-1;
    array_foreach(&ectx->queries, query) {
        if (ectx->oldest_before_time > query->before_time ||
            ectx->oldest_before_time == (time_t)-1)
            ectx->oldest_before_time = query->before_time;
    }

    ctx->v.deinit = doveadm_expire_mail_cmd_deinit;
    ctx->v.get_next_user = doveadm_expire_mail_cmd_get_next_user;

    hash_table_create(&ectx->user_states, ctx->pool, 0, str_hash, strcmp);
    while (mail_storage_service_all_next(ctx->storage_service, &username) > 0) {
        username_dup = p_strdup(ctx->pool, username);
        state = EXPIRE_USER_STATE_EXISTS;
        hash_table_insert(ectx->user_states, username_dup,
                  POINTER_CAST(state));
    }

    ectx->dict = dict;
    ectx->trans = dict_transaction_begin(dict);
    ectx->iter = dict_iterate_init(dict, DICT_EXPIRE_PREFIX,
                       DICT_ITERATE_FLAG_RECURSE |
                       DICT_ITERATE_FLAG_SORT_BY_VALUE);
}

void doveadm_expire_plugin_init(struct module *module ATTR_UNUSED)
{
    next_hook_doveadm_mail_init = hook_doveadm_mail_init;
    hook_doveadm_mail_init = doveadm_expire_mail_init;
}

void doveadm_expire_plugin_deinit(void)
{
    i_assert(hook_doveadm_mail_init == doveadm_expire_mail_init);
    hook_doveadm_mail_init = next_hook_doveadm_mail_init;
}