doveadm/dsync/dsync-mailbox-tree-fill.c

/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "hash.h"
#include "guid.h"
#include "str.h"
#include "wildcard-match.h"
#include "mailbox-log.h"
#include "mail-namespace.h"
#include "mail-storage.h"
#include "mailbox-list-iter.h"
#include "dsync-mailbox-tree-private.h"

static int
dsync_mailbox_tree_add_node(struct dsync_mailbox_tree *tree,
                const struct mailbox_info *info,
                struct dsync_mailbox_node **node_r)
{
    struct dsync_mailbox_node *node;

    node = dsync_mailbox_tree_get(tree, info->vname);
    if (node->ns == info->ns)
        ;
    else if (node->ns == NULL) {
        i_assert(tree->root.ns == NULL);
        node->ns = info->ns;
    } else {
        i_error("Mailbox '%s' exists in two namespaces: '%s' and '%s'",
            info->vname, node->ns->prefix, info->ns->prefix);
        return -1;
    }
    *node_r = node;
    return 0;
}

static int
dsync_mailbox_tree_add_exists_node(struct dsync_mailbox_tree *tree,
                   const struct mailbox_info *info,
                   struct dsync_mailbox_node **node_r,
                   enum mail_error *error_r)
{
    if (dsync_mailbox_tree_add_node(tree, info, node_r) < 0) {
        *error_r = MAIL_ERROR_TEMP;
        return -1;
    }
    (*node_r)->existence = DSYNC_MAILBOX_NODE_EXISTS;
    return 0;
}

static int
dsync_mailbox_tree_get_selectable(struct mailbox *box,
                  struct mailbox_metadata *metadata_r,
                  struct mailbox_status *status_r)
{
    /* try the fast path */
    if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, metadata_r) < 0)
        return -1;
    if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0)
        return -1;

    i_assert(!guid_128_is_empty(metadata_r->guid));
    if (status_r->uidvalidity != 0)
        return 0;

    /* no UIDVALIDITY assigned yet. syncing a mailbox should add it. */
    if (mailbox_sync(box, 0) < 0)
        return -1;
    if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0)
        return -1;
    i_assert(status_r->uidvalidity != 0);
    return 0;
}

static int dsync_mailbox_tree_add(struct dsync_mailbox_tree *tree,
                  const struct mailbox_info *info,
                  const guid_128_t box_guid,
                  enum mail_error *error_r)
{
    struct dsync_mailbox_node *node;
    struct mailbox *box;
    struct mailbox_metadata metadata;
    struct mailbox_status status;
    const char *errstr;
    enum mail_error error;
    int ret = 0;

    if ((info->flags & MAILBOX_NONEXISTENT) != 0)
        return 0;
    if ((info->flags & MAILBOX_NOSELECT) != 0) {
        return !guid_128_is_empty(box_guid) ? 0 :
            dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r);
    }

    /* get GUID and UIDVALIDITY for selectable mailbox */
    box = mailbox_alloc(info->ns->list, info->vname, MAILBOX_FLAG_READONLY);
    if (dsync_mailbox_tree_get_selectable(box, &metadata, &status) < 0) {
        errstr = mailbox_get_last_internal_error(box, &error);
        switch (error) {
        case MAIL_ERROR_NOTFOUND:
            /* mailbox was just deleted? */
            break;
        case MAIL_ERROR_NOTPOSSIBLE:
            /* invalid mbox files? ignore */
            break;
        default:
            i_error("Failed to access mailbox %s: %s",
                info->vname, errstr);
            *error_r = error;
            ret = -1;
        }
        mailbox_free(&box);
        return ret;
    }
    mailbox_free(&box);

    if (!guid_128_is_empty(box_guid) &&
        !guid_128_equals(box_guid, metadata.guid)) {
        /* unwanted mailbox */
        return 0;
    }
    if (dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r) < 0)
        return -1;
    memcpy(node->mailbox_guid, metadata.guid,
           sizeof(node->mailbox_guid));
    node->uid_validity = status.uidvalidity;
    node->uid_next = status.uidnext;
    return 0;
}

static struct dsync_mailbox_node *
dsync_mailbox_tree_find_sha(struct dsync_mailbox_tree *tree,
                struct mail_namespace *ns, const guid_128_t sha128)
{
    struct dsync_mailbox_node *node;

    if (!hash_table_is_created(tree->name128_hash))
        dsync_mailbox_tree_build_name128_hash(tree);

    node = hash_table_lookup(tree->name128_hash, sha128);
    return node == NULL || node->ns != ns ? NULL : node;
}

static int
dsync_mailbox_tree_add_change_timestamps(struct dsync_mailbox_tree *tree,
                     struct mail_namespace *ns)
{
    struct dsync_mailbox_node *node;
    struct dsync_mailbox_delete *del;
    struct mailbox_log *log;
    struct mailbox_log_iter *iter;
    const struct mailbox_log_record *rec;
    const uint8_t *guid_p;
    time_t timestamp;

    log = mailbox_list_get_changelog(ns->list);
    if (log == NULL)
        return 0;

    iter = mailbox_log_iter_init(log);
    while ((rec = mailbox_log_iter_next(iter)) != NULL) {
        node = rec->type == MAILBOX_LOG_RECORD_DELETE_MAILBOX ? NULL :
            dsync_mailbox_tree_find_sha(tree, ns, rec->mailbox_guid);

        timestamp = mailbox_log_record_get_timestamp(rec);
        switch (rec->type) {
        case MAILBOX_LOG_RECORD_DELETE_MAILBOX:
            guid_p = rec->mailbox_guid;
            if (hash_table_lookup(tree->guid_hash, guid_p) != NULL) {
                /* mailbox still exists. maybe it was restored
                   from backup or something. */
                break;
            }
            del = array_append_space(&tree->deletes);
            del->type = DSYNC_MAILBOX_DELETE_TYPE_MAILBOX;
            del->timestamp = timestamp;
            memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
            break;
        case MAILBOX_LOG_RECORD_DELETE_DIR:
            if (node != NULL &&
                node->existence == DSYNC_MAILBOX_NODE_EXISTS) {
                /* directory exists again, skip it */
                break;
            }
            /* we don't know what directory name was deleted,
               just its hash. if the name still exists on the other
               dsync side, it can match this deletion to the
               name. */
            del = array_append_space(&tree->deletes);
            del->type = DSYNC_MAILBOX_DELETE_TYPE_DIR;
            del->timestamp = timestamp;
            memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
            break;
        case MAILBOX_LOG_RECORD_CREATE_DIR:
            if (node == NULL) {
                /* directory has been deleted again, skip it */
                break;
            }
            /* notify the remote that we want to keep this
               directory created (unless remote has a newer delete
               timestamp) */
            node->last_renamed_or_created = timestamp;
            break;
        case MAILBOX_LOG_RECORD_RENAME:
            if (node != NULL)
                node->last_renamed_or_created = timestamp;
            break;
        case MAILBOX_LOG_RECORD_SUBSCRIBE:
            if (node != NULL)
                node->last_subscription_change = timestamp;
            break;
        case MAILBOX_LOG_RECORD_UNSUBSCRIBE:
            if (node != NULL) {
                node->last_subscription_change = timestamp;
                break;
            }
            /* The mailbox is already deleted, but it may still
               exist on the other side (even the subscription
               alone). */
            del = array_append_space(&tree->deletes);
            del->type = DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE;
            del->timestamp = timestamp;
            memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
            break;
        }
    }
    if (mailbox_log_iter_deinit(&iter) < 0) {
        i_error("Mailbox log iteration for namespace '%s' failed",
            ns->prefix);
        return -1;
    }
    return 0;
}

static int
dsync_mailbox_tree_fix_guid_duplicate(struct dsync_mailbox_tree *tree,
                      struct dsync_mailbox_node *node1,
                      struct dsync_mailbox_node *node2)
{
    struct mailbox *box;
    struct mailbox_update update;
    struct dsync_mailbox_node *change_node;
    const char *change_vname;
    int ret = 0;

    i_zero(&update);
    guid_128_generate(update.mailbox_guid);

    /* just in case the duplication exists in both sides,
       make them choose the same node */
    if (strcmp(dsync_mailbox_node_get_full_name(tree, node1),
           dsync_mailbox_node_get_full_name(tree, node2)) <= 0)
        change_node = node1;
    else
        change_node = node2;

    change_vname = dsync_mailbox_node_get_full_name(tree, change_node);
    i_error("Duplicate mailbox GUID %s for mailboxes %s and %s - "
        "giving a new GUID %s to %s",
        guid_128_to_string(node1->mailbox_guid),
        dsync_mailbox_node_get_full_name(tree, node1),
        dsync_mailbox_node_get_full_name(tree, node2),
        guid_128_to_string(update.mailbox_guid), change_vname);

    i_assert(node1->ns != NULL && node2->ns != NULL);
    box = mailbox_alloc(change_node->ns->list, change_vname, 0);
    if (mailbox_update(box, &update) < 0) {
        i_error("Couldn't update mailbox %s GUID: %s",
            change_vname, mailbox_get_last_internal_error(box, NULL));
        ret = -1;
    } else {
        memcpy(change_node->mailbox_guid, update.mailbox_guid,
               sizeof(change_node->mailbox_guid));
    }
    mailbox_free(&box);
    return ret;
}

static bool
dsync_mailbox_info_is_wanted(const struct mailbox_info *info,
                 const char *box_name,
                 const char *const *exclude_mailboxes)
{
    const char *const *info_specialuses;
    unsigned int i;

    if (exclude_mailboxes == NULL &&
        (box_name == NULL || box_name[0] != '\\'))
        return TRUE;

    info_specialuses = info->special_use == NULL ? NULL :
        t_strsplit(info->special_use, " ");
    /* include */
    if (box_name != NULL && box_name[0] == '\\') {
        if (info_specialuses == NULL ||
            !str_array_icase_find(info_specialuses, box_name))
            return FALSE;
    }
    /* exclude */
    if (exclude_mailboxes == NULL)
        return TRUE;
    for (i = 0; exclude_mailboxes[i] != NULL; i++) {
        const char *exclude = exclude_mailboxes[i];

        if (exclude[0] == '\\') {
            /* special-use */
            if (info_specialuses != NULL &&
                str_array_icase_find(info_specialuses, exclude))
                return FALSE;
        } else {
            /* mailbox with wildcards */
            if (wildcard_match(info->vname, exclude))
                return FALSE;
        }
    }
    return TRUE;
}

int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree,
                struct mail_namespace *ns, const char *box_name,
                const guid_128_t box_guid,
                const char *const *exclude_mailboxes,
                enum mail_error *error_r)
{
    const enum mailbox_list_iter_flags list_flags =
        /* FIXME: we'll skip symlinks, because we can't handle them
           currently. in future we could detect them and create them
           by creating the symlink. */
        MAILBOX_LIST_ITER_SKIP_ALIASES |
        MAILBOX_LIST_ITER_NO_AUTO_BOXES;
    const enum mailbox_list_iter_flags subs_list_flags =
        MAILBOX_LIST_ITER_NO_AUTO_BOXES |
        MAILBOX_LIST_ITER_SELECT_SUBSCRIBED |
        MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
    struct mailbox_list_iterate_context *iter;
    struct dsync_mailbox_node *node, *dup_node1, *dup_node2;
    const struct mailbox_info *info;
    const char *list_pattern =
        box_name != NULL && box_name[0] != '\\' ? box_name : "*";
    int ret = 0;

    i_assert(mail_namespace_get_sep(ns) == tree->sep);

    /* assign namespace to its root, so it gets copied to children */
    if (ns->prefix_len > 0) {
        node = dsync_mailbox_tree_get(tree,
            t_strndup(ns->prefix, ns->prefix_len-1));
        node->ns = ns;
    } else {
        tree->root.ns = ns;
    }

    /* first add all of the existing mailboxes */
    iter = mailbox_list_iter_init(ns->list, list_pattern, list_flags);
    while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
        if (dsync_mailbox_info_is_wanted(info, box_name,
                         exclude_mailboxes)) {
            if (dsync_mailbox_tree_add(tree, info, box_guid, error_r) < 0)
                ret = -1;
        }
    } T_END;
    if (mailbox_list_iter_deinit(&iter) < 0) {
        i_error("Mailbox listing for namespace '%s' failed: %s",
            ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r));
        ret = -1;
    }

    /* add subscriptions */
    iter = mailbox_list_iter_init(ns->list, list_pattern, subs_list_flags);
    while ((info = mailbox_list_iter_next(iter)) != NULL) {
        if (dsync_mailbox_tree_add_node(tree, info, &node) == 0)
            node->subscribed = TRUE;
        else {
            *error_r = MAIL_ERROR_TEMP;
            ret = -1;
        }
    }
    if (mailbox_list_iter_deinit(&iter) < 0) {
        i_error("Mailbox listing for namespace '%s' failed: %s",
            ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r));
        ret = -1;
    }
    if (ret < 0)
        return -1;

    while (dsync_mailbox_tree_build_guid_hash(tree, &dup_node1,
                          &dup_node2) < 0) {
        if (dsync_mailbox_tree_fix_guid_duplicate(tree, dup_node1, dup_node2) < 0)
            return -1;
    }

    /* add timestamps */
    if (dsync_mailbox_tree_add_change_timestamps(tree, ns) < 0)
        return -1;
    return 0;
}