i86pc/io/immu_dvma.c

	immu_dvma.c revision 15e1afcd5e908ae29b1e6018838638befdc225a2
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Portions Copyright (c) 2010, Oracle and/or its affiliates.
 * All rights reserved.
 */
/*
 * Copyright (c) 2009, Intel Corporation.
 * All rights reserved.
 */

/*
 * DVMA code
 * This file contains Intel IOMMU code that deals with DVMA
 * i.e. DMA remapping.
 */

#include <sys/sysmacros.h>
#include <sys/pcie.h>
#include <sys/pci_cfgspace.h>
#include <vm/hat_i86.h>
#include <sys/memlist.h>
#include <sys/acpi/acpi.h>
#include <sys/acpica.h>
#include <sys/modhash.h>
#include <sys/immu.h>
#include <sys/x86_archext.h>
#include <sys/archsystm.h>

#undef  TEST

/*
 * Macros based on PCI spec
 */
#define IMMU_PCI_REV2CLASS(r)   ((r) >> 8)  /* classcode from revid */
#define IMMU_PCI_CLASS2BASE(c)  ((c) >> 16) /* baseclass from classcode */
#define IMMU_PCI_CLASS2SUB(c)   (((c) >> 8) & 0xff); /* classcode */

#define IMMU_CONTIG_PADDR(d, p) \
    ((d).dck_paddr && ((d).dck_paddr + IMMU_PAGESIZE) == (p))

typedef struct dvma_arg {
    immu_t *dva_immu;
    dev_info_t *dva_rdip;
    dev_info_t *dva_ddip;
    domain_t *dva_domain;
    int dva_level;
    immu_flags_t dva_flags;
    list_t *dva_list;
    int dva_error;
} dvma_arg_t;

static domain_t *domain_create(immu_t *immu, dev_info_t *ddip,
    dev_info_t *rdip, immu_flags_t immu_flags);
static immu_devi_t *create_immu_devi(dev_info_t *rdip, int bus,
    int dev, int func, immu_flags_t immu_flags);
static void destroy_immu_devi(immu_devi_t *immu_devi);
static boolean_t dvma_map(domain_t *domain, uint64_t sdvma,
    uint64_t nvpages, immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,
    immu_flags_t immu_flags);

/* Extern globals */
extern struct memlist  *phys_install;

/*
 * iommulib interface functions.
 */
static int immu_probe(iommulib_handle_t unitp, dev_info_t *dip);
static int immu_allochdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
    int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep);
static int immu_freehdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
static int immu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dma_req,
    ddi_dma_cookie_t *cookiep, uint_t *ccountp);
static int immu_unbindhdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
static int immu_sync(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, size_t len,
    uint_t cachefl);
static int immu_win(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
    off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp);
static int immu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao);
static int immu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao);
static int immu_map(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, struct ddi_dma_req *dmareq,
    ddi_dma_handle_t *dma_handle);
static int immu_mctl(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    enum ddi_dma_ctlops request, off_t *offp, size_t *lenp,
    caddr_t *objpp, uint_t cachefl);

/* static Globals */

/*
 * Used to setup DMA objects (memory regions)
 * for DMA reads by IOMMU units
 */
static ddi_dma_attr_t immu_dma_attr = {
    DMA_ATTR_V0,
    0U,
    0xffffffffffffffffULL,
    0xffffffffU,
    MMU_PAGESIZE, /* MMU page aligned */
    0x1,
    0x1,
    0xffffffffU,
    0xffffffffffffffffULL,
    1,
    4,
    0
};

static ddi_device_acc_attr_t immu_acc_attr = {
    DDI_DEVICE_ATTR_V0,
    DDI_NEVERSWAP_ACC,
    DDI_STRICTORDER_ACC
};

struct iommulib_ops immulib_ops = {
    IOMMU_OPS_VERSION,
    INTEL_IOMMU,
    "Intel IOMMU",
    NULL,
    immu_probe,
    immu_allochdl,
    immu_freehdl,
    immu_bindhdl,
    immu_unbindhdl,
    immu_sync,
    immu_win,
    immu_mapobject,
    immu_unmapobject,
    immu_map,
    immu_mctl
};

/*
 * Fake physical address range used to set up initial prealloc mappings.
 * This memory is never actually accessed. It is mapped read-only,
 * and is overwritten as soon as the first DMA bind operation is
 * performed. Since 0 is a special case, just start at the 2nd
 * physical page.
 */

static immu_dcookie_t immu_precookie = { MMU_PAGESIZE, IMMU_NPREPTES };

/* globals private to this file */
static kmutex_t immu_domain_lock;
static list_t immu_unity_domain_list;
static list_t immu_xlate_domain_list;

/* structure used to store idx into each level of the page tables */
typedef struct xlate {
    int xlt_level;
    uint_t xlt_idx;
    pgtable_t *xlt_pgtable;
} xlate_t;

/* 0 is reserved by Vt-d spec. Solaris reserves 1 */
#define IMMU_UNITY_DID   1

static mod_hash_t *bdf_domain_hash;

int immu_use_alh;
int immu_use_tm;

static domain_t *
bdf_domain_lookup(immu_devi_t *immu_devi)
{
    domain_t *domain;
    int16_t seg = immu_devi->imd_seg;
    int16_t bus = immu_devi->imd_bus;
    int16_t devfunc = immu_devi->imd_devfunc;
    uintptr_t bdf = (seg << 16 | bus << 8 | devfunc);

    if (seg < 0 || bus < 0 || devfunc < 0) {
        return (NULL);
    }

    domain = NULL;
    if (mod_hash_find(bdf_domain_hash,
        (void *)bdf, (void *)&domain) == 0) {
        ASSERT(domain);
        ASSERT(domain->dom_did > 0);
        return (domain);
    } else {
        return (NULL);
    }
}

static void
bdf_domain_insert(immu_devi_t *immu_devi, domain_t *domain)
{
    int16_t seg = immu_devi->imd_seg;
    int16_t bus = immu_devi->imd_bus;
    int16_t devfunc = immu_devi->imd_devfunc;
    uintptr_t bdf = (seg << 16 | bus << 8 | devfunc);

    if (seg < 0 || bus < 0 || devfunc < 0) {
        return;
    }

    (void) mod_hash_insert(bdf_domain_hash, (void *)bdf, (void *)domain);
}

static int
match_lpc(dev_info_t *pdip, void *arg)
{
    immu_devi_t *immu_devi;
    dvma_arg_t *dvap = (dvma_arg_t *)arg;

    if (list_is_empty(dvap->dva_list)) {
        return (DDI_WALK_TERMINATE);
    }

    immu_devi = list_head(dvap->dva_list);
    for (; immu_devi; immu_devi = list_next(dvap->dva_list,
        immu_devi)) {
        if (immu_devi->imd_dip == pdip) {
            dvap->dva_ddip = pdip;
            dvap->dva_error = DDI_SUCCESS;
            return (DDI_WALK_TERMINATE);
        }
    }

    return (DDI_WALK_CONTINUE);
}

static void
immu_devi_set_spclist(dev_info_t *dip, immu_t *immu)
{
    list_t *spclist = NULL;
    immu_devi_t *immu_devi;

    immu_devi = IMMU_DEVI(dip);
    if (immu_devi->imd_display == B_TRUE) {
        spclist = &(immu->immu_dvma_gfx_list);
    } else if (immu_devi->imd_lpc == B_TRUE) {
        spclist = &(immu->immu_dvma_lpc_list);
    }

    if (spclist) {
        mutex_enter(&(immu->immu_lock));
        list_insert_head(spclist, immu_devi);
        mutex_exit(&(immu->immu_lock));
    }
}

/*
 * Set the immu_devi struct in the immu_devi field of a devinfo node
 */
int
immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags)
{
    int bus, dev, func;
    immu_devi_t *new_imd;
    immu_devi_t *immu_devi;

    immu_devi = immu_devi_get(dip);
    if (immu_devi != NULL) {
        return (DDI_SUCCESS);
    }

    bus = dev = func = -1;

    /*
     * Assume a new immu_devi struct is needed
     */
    if (!DEVI_IS_PCI(dip) || acpica_get_bdf(dip, &bus, &dev, &func) != 0) {
        /*
         * No BDF. Set bus = -1 to indicate this.
         * We still need to create a immu_devi struct
         * though
         */
        bus = -1;
        dev = 0;
        func = 0;
    }

    new_imd = create_immu_devi(dip, bus, dev, func, immu_flags);
    if (new_imd  == NULL) {
        ddi_err(DER_WARN, dip, "Failed to create immu_devi "
            "structure");
        return (DDI_FAILURE);
    }

    /*
     * Check if some other thread allocated a immu_devi while we
     * didn't own the lock.
     */
    mutex_enter(&(DEVI(dip)->devi_lock));
    if (IMMU_DEVI(dip) == NULL) {
        IMMU_DEVI_SET(dip, new_imd);
    } else {
        destroy_immu_devi(new_imd);
    }
    mutex_exit(&(DEVI(dip)->devi_lock));

    return (DDI_SUCCESS);
}

static dev_info_t *
get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)
{
    dvma_arg_t dvarg = {0};
    dvarg.dva_list = &(immu->immu_dvma_lpc_list);
    dvarg.dva_rdip = rdip;
    dvarg.dva_error = DDI_FAILURE;

    if (immu_walk_ancestor(rdip, NULL, match_lpc,
        &dvarg, NULL, immu_flags) != DDI_SUCCESS) {
        ddi_err(DER_MODE, rdip, "Could not walk ancestors to "
            "find lpc_devinfo for ISA device");
        return (NULL);
    }

    if (dvarg.dva_error != DDI_SUCCESS || dvarg.dva_ddip == NULL) {
        ddi_err(DER_MODE, rdip, "Could not find lpc_devinfo for "
            "ISA device");
        return (NULL);
    }

    return (dvarg.dva_ddip);
}

static dev_info_t *
get_gfx_devinfo(dev_info_t *rdip)
{
    immu_t *immu;
    immu_devi_t *immu_devi;
    list_t *list_gfx;

    /*
     * The GFX device may not be on the same iommu unit as "agpgart"
     * so search globally
     */
    immu_devi = NULL;
    immu = list_head(&immu_list);
    for (; immu; immu = list_next(&immu_list, immu)) {
        list_gfx = &(immu->immu_dvma_gfx_list);
        if (!list_is_empty(list_gfx)) {
            immu_devi = list_head(list_gfx);
            break;
        }
    }

    if (immu_devi == NULL) {
        ddi_err(DER_WARN, rdip, "iommu: No GFX device. "
            "Cannot redirect agpgart");
        return (NULL);
    }

    ddi_err(DER_LOG, rdip, "iommu: GFX redirect to %s",
        ddi_node_name(immu_devi->imd_dip));

    return (immu_devi->imd_dip);
}

static immu_flags_t
dma_to_immu_flags(struct ddi_dma_req *dmareq)
{
    immu_flags_t flags = 0;

    if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
        flags |= IMMU_FLAGS_SLEEP;
    } else {
        flags |= IMMU_FLAGS_NOSLEEP;
    }

#ifdef BUGGY_DRIVERS

    flags |= (IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

#else
    /*
     * Read and write flags need to be reversed.
     * DMA_READ means read from device and write
     * to memory. So DMA read means DVMA write.
     */
    if (dmareq->dmar_flags & DDI_DMA_READ)
        flags |= IMMU_FLAGS_WRITE;

    if (dmareq->dmar_flags & DDI_DMA_WRITE)
        flags |= IMMU_FLAGS_READ;

    /*
     * Some buggy drivers specify neither READ or WRITE
     * For such drivers set both read and write permissions
     */
    if ((dmareq->dmar_flags & (DDI_DMA_READ | DDI_DMA_WRITE)) == 0) {
        flags |= (IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
    }
#endif

    return (flags);
}

/*ARGSUSED*/
int
pgtable_ctor(void *buf, void *arg, int kmflag)
{
    size_t actual_size = 0;
    pgtable_t *pgtable;
    int (*dmafp)(caddr_t);
    caddr_t vaddr;
    void *next;
    uint_t flags;
    immu_t *immu = arg;

    pgtable = (pgtable_t *)buf;

    dmafp = (kmflag & KM_NOSLEEP) ? DDI_DMA_DONTWAIT : DDI_DMA_SLEEP;

    next = kmem_zalloc(IMMU_PAGESIZE, kmflag);
    if (next == NULL) {
        return (-1);
    }

    if (ddi_dma_alloc_handle(root_devinfo, &immu_dma_attr,
        dmafp, NULL, &pgtable->hwpg_dmahdl) != DDI_SUCCESS) {
        kmem_free(next, IMMU_PAGESIZE);
        return (-1);
    }

    flags = DDI_DMA_CONSISTENT;
    if (!immu->immu_dvma_coherent)
        flags |= IOMEM_DATA_UC_WR_COMBINE;

    if (ddi_dma_mem_alloc(pgtable->hwpg_dmahdl, IMMU_PAGESIZE,
        &immu_acc_attr, flags,
        dmafp, NULL, &vaddr, &actual_size,
        &pgtable->hwpg_memhdl) != DDI_SUCCESS) {
        ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
        kmem_free(next, IMMU_PAGESIZE);
        return (-1);
    }

    /*
     * Memory allocation failure. Maybe a temporary condition
     * so return error rather than panic, so we can try again
     */
    if (actual_size < IMMU_PAGESIZE) {
        ddi_dma_mem_free(&pgtable->hwpg_memhdl);
        ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
        kmem_free(next, IMMU_PAGESIZE);
        return (-1);
    }

    pgtable->hwpg_paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, vaddr));
    pgtable->hwpg_vaddr = vaddr;
    pgtable->swpg_next_array = next;

    rw_init(&(pgtable->swpg_rwlock), NULL, RW_DEFAULT, NULL);

    return (0);
}

/*ARGSUSED*/
void
pgtable_dtor(void *buf, void *arg)
{
    pgtable_t *pgtable;

    pgtable = (pgtable_t *)buf;

    /* destroy will panic if lock is held. */
    rw_destroy(&(pgtable->swpg_rwlock));

    ddi_dma_mem_free(&pgtable->hwpg_memhdl);
    ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
    kmem_free(pgtable->swpg_next_array, IMMU_PAGESIZE);
}

/*
 * pgtable_alloc()
 *  alloc a IOMMU pgtable structure.
 *  This same struct is used for root and context tables as well.
 *  This routine allocs the f/ollowing:
 *  - a pgtable_t struct
 *  - a HW page which holds PTEs/entries which is accesssed by HW
 *        so we set up DMA for this page
 *  - a SW page which is only for our bookeeping
 *        (for example to  hold pointers to the next level pgtable).
 *        So a simple kmem_alloc suffices
 */
static pgtable_t *
pgtable_alloc(immu_t *immu, immu_flags_t immu_flags)
{
    pgtable_t *pgtable;
    int kmflags;

    kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;

    pgtable = kmem_cache_alloc(immu->immu_pgtable_cache, kmflags);
    if (pgtable == NULL) {
        return (NULL);
    }
    return (pgtable);
}

static void
pgtable_zero(pgtable_t *pgtable)
{
    bzero(pgtable->hwpg_vaddr, IMMU_PAGESIZE);
    bzero(pgtable->swpg_next_array, IMMU_PAGESIZE);
}

static void
pgtable_free(immu_t *immu, pgtable_t *pgtable)
{
    kmem_cache_free(immu->immu_pgtable_cache, pgtable);
}

/*
 * Function to identify a display device from the PCI class code
 */
static boolean_t
device_is_display(uint_t classcode)
{
    static uint_t disp_classes[] = {
        0x000100,
        0x030000,
        0x030001
    };
    int i, nclasses = sizeof (disp_classes) / sizeof (uint_t);

    for (i = 0; i < nclasses; i++) {
        if (classcode == disp_classes[i])
            return (B_TRUE);
    }
    return (B_FALSE);
}

/*
 * Function that determines if device is PCIEX and/or PCIEX bridge
 */
static boolean_t
device_is_pciex(
    uchar_t bus, uchar_t dev, uchar_t func, boolean_t *is_pcib)
{
    ushort_t cap;
    ushort_t capsp;
    ushort_t cap_count = PCI_CAP_MAX_PTR;
    ushort_t status;
    boolean_t is_pciex = B_FALSE;

    *is_pcib = B_FALSE;

    status = pci_getw_func(bus, dev, func, PCI_CONF_STAT);
    if (!(status & PCI_STAT_CAP))
        return (B_FALSE);

    capsp = pci_getb_func(bus, dev, func, PCI_CONF_CAP_PTR);
    while (cap_count-- && capsp >= PCI_CAP_PTR_OFF) {
        capsp &= PCI_CAP_PTR_MASK;
        cap = pci_getb_func(bus, dev, func, capsp);

        if (cap == PCI_CAP_ID_PCI_E) {
            status = pci_getw_func(bus, dev, func, capsp + 2);
            /*
             * See section 7.8.2 of PCI-Express Base Spec v1.0a
             * for Device/Port Type.
             * PCIE_PCIECAP_DEV_TYPE_PCIE2PCI implies that the
             * device is a PCIE2PCI bridge
             */
            *is_pcib =
                ((status & PCIE_PCIECAP_DEV_TYPE_MASK) ==
                PCIE_PCIECAP_DEV_TYPE_PCIE2PCI) ? B_TRUE : B_FALSE;
            is_pciex = B_TRUE;
        }

        capsp = (*pci_getb_func)(bus, dev, func,
            capsp + PCI_CAP_NEXT_PTR);
    }

    return (is_pciex);
}

static boolean_t
device_use_premap(uint_t classcode)
{
    if (IMMU_PCI_CLASS2BASE(classcode) == PCI_CLASS_NET)
        return (B_TRUE);
    return (B_FALSE);
}


/*
 * immu_dvma_get_immu()
 *   get the immu unit structure for a dev_info node
 */
immu_t *
immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags)
{
    immu_devi_t *immu_devi;
    immu_t *immu;

    /*
     * check if immu unit was already found earlier.
     * If yes, then it will be stashed in immu_devi struct.
     */
    immu_devi = immu_devi_get(dip);
    if (immu_devi == NULL) {
        if (immu_devi_set(dip, immu_flags) != DDI_SUCCESS) {
            /*
             * May fail because of low memory. Return error rather
             * than panic as we want driver to rey again later
             */
            ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: "
                "No immu_devi structure");
            /*NOTREACHED*/
        }
        immu_devi = immu_devi_get(dip);
    }

    mutex_enter(&(DEVI(dip)->devi_lock));
    if (immu_devi->imd_immu) {
        immu = immu_devi->imd_immu;
        mutex_exit(&(DEVI(dip)->devi_lock));
        return (immu);
    }
    mutex_exit(&(DEVI(dip)->devi_lock));

    immu = immu_dmar_get_immu(dip);
    if (immu == NULL) {
        ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: "
            "Cannot find immu_t for device");
        /*NOTREACHED*/
    }

    /*
     * Check if some other thread found immu
     * while lock was not held
     */
    immu_devi = immu_devi_get(dip);
    /* immu_devi should be present as we found it earlier */
    if (immu_devi == NULL) {
        ddi_err(DER_PANIC, dip,
            "immu_dvma_get_immu: No immu_devi structure");
        /*NOTREACHED*/
    }

    mutex_enter(&(DEVI(dip)->devi_lock));
    if (immu_devi->imd_immu == NULL) {
        /* nobody else set it, so we should do it */
        immu_devi->imd_immu = immu;
        immu_devi_set_spclist(dip, immu);
    } else {
        /*
         * if some other thread got immu before
         * us, it should get the same results
         */
        if (immu_devi->imd_immu != immu) {
            ddi_err(DER_PANIC, dip, "Multiple "
                "immu units found for device. Expected (%p), "
                "actual (%p)", (void *)immu,
                (void *)immu_devi->imd_immu);
            mutex_exit(&(DEVI(dip)->devi_lock));
            /*NOTREACHED*/
        }
    }
    mutex_exit(&(DEVI(dip)->devi_lock));

    return (immu);
}


/* ############################# IMMU_DEVI code ############################ */

/*
 * Allocate a immu_devi structure and initialize it
 */
static immu_devi_t *
create_immu_devi(dev_info_t *rdip, int bus, int dev, int func,
    immu_flags_t immu_flags)
{
    uchar_t baseclass, subclass;
    uint_t classcode, revclass;
    immu_devi_t *immu_devi;
    boolean_t pciex = B_FALSE;
    int kmflags;
    boolean_t is_pcib = B_FALSE;

    /* bus ==  -1 indicate non-PCI device (no BDF) */
    ASSERT(bus == -1 || bus >= 0);
    ASSERT(dev >= 0);
    ASSERT(func >= 0);

    kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
    immu_devi = kmem_zalloc(sizeof (immu_devi_t), kmflags);
    if (immu_devi == NULL) {
        ddi_err(DER_WARN, rdip, "Failed to allocate memory for "
            "Intel IOMMU immu_devi structure");
        return (NULL);
    }
    immu_devi->imd_dip = rdip;
    immu_devi->imd_seg = 0; /* Currently seg can only be 0 */
    immu_devi->imd_bus = bus;
    immu_devi->imd_pcib_type = IMMU_PCIB_BAD;

    if (bus == -1) {
        immu_devi->imd_pcib_type = IMMU_PCIB_NOBDF;
        return (immu_devi);
    }

    immu_devi->imd_devfunc = IMMU_PCI_DEVFUNC(dev, func);
    immu_devi->imd_sec = 0;
    immu_devi->imd_sub = 0;

    revclass = pci_getl_func(bus, dev, func, PCI_CONF_REVID);

    classcode = IMMU_PCI_REV2CLASS(revclass);
    baseclass = IMMU_PCI_CLASS2BASE(classcode);
    subclass = IMMU_PCI_CLASS2SUB(classcode);

    if (baseclass == PCI_CLASS_BRIDGE && subclass == PCI_BRIDGE_PCI) {

        immu_devi->imd_sec = pci_getb_func(bus, dev, func,
            PCI_BCNF_SECBUS);
        immu_devi->imd_sub = pci_getb_func(bus, dev, func,
            PCI_BCNF_SUBBUS);

        pciex = device_is_pciex(bus, dev, func, &is_pcib);
        if (pciex  == B_TRUE && is_pcib == B_TRUE) {
            immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCI;
        } else if (pciex == B_TRUE) {
            immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCIE;
        } else {
            immu_devi->imd_pcib_type = IMMU_PCIB_PCI_PCI;
        }
    } else {
        immu_devi->imd_pcib_type = IMMU_PCIB_ENDPOINT;
    }

    /* check for certain special devices */
    immu_devi->imd_display = device_is_display(classcode);
    immu_devi->imd_lpc = ((baseclass == PCI_CLASS_BRIDGE) &&
        (subclass == PCI_BRIDGE_ISA)) ? B_TRUE : B_FALSE;
    immu_devi->imd_use_premap = device_use_premap(classcode);

    immu_devi->imd_domain = NULL;

    immu_devi->imd_dvma_flags = immu_global_dvma_flags;

    return (immu_devi);
}

static void
destroy_immu_devi(immu_devi_t *immu_devi)
{
    kmem_free(immu_devi, sizeof (immu_devi_t));
}

static domain_t *
immu_devi_domain(dev_info_t *rdip, dev_info_t **ddipp)
{
    immu_devi_t *immu_devi;
    domain_t *domain;
    dev_info_t *ddip;

    *ddipp = NULL;

    immu_devi = immu_devi_get(rdip);
    if (immu_devi == NULL) {
        return (NULL);
    }

    mutex_enter(&(DEVI(rdip)->devi_lock));
    domain = immu_devi->imd_domain;
    ddip = immu_devi->imd_ddip;
    mutex_exit(&(DEVI(rdip)->devi_lock));

    if (domain)
        *ddipp = ddip;

    return (domain);

}

/* ############################# END IMMU_DEVI code ######################## */
/* ############################# DOMAIN code ############################### */

/*
 * This routine always succeeds
 */
static int
did_alloc(immu_t *immu, dev_info_t *rdip,
    dev_info_t *ddip, immu_flags_t immu_flags)
{
    int did;

    did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1,
        (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP);

    if (did == 0) {
        ddi_err(DER_WARN, rdip, "device domain-id alloc error"
            " domain-device: %s%d. immu unit is %s. Using "
            "unity domain with domain-id (%d)",
            ddi_driver_name(ddip), ddi_get_instance(ddip),
            immu->immu_name, immu->immu_unity_domain->dom_did);
        did = immu->immu_unity_domain->dom_did;
    }

    return (did);
}

static int
get_branch_domain(dev_info_t *pdip, void *arg)
{
    immu_devi_t *immu_devi;
    domain_t *domain;
    dev_info_t *ddip;
    immu_t *immu;
    dvma_arg_t *dvp = (dvma_arg_t *)arg;

    /*
     * The field dvp->dva_rdip is a work-in-progress
     * and gets updated as we walk up the ancestor
     * tree. The final ddip is set only when we reach
     * the top of the tree. So the dvp->dva_ddip field cannot
     * be relied on until we reach the top of the field.
     */

    /* immu_devi may not be set. */
    immu_devi = immu_devi_get(pdip);
    if (immu_devi == NULL) {
        if (immu_devi_set(pdip, dvp->dva_flags) != DDI_SUCCESS) {
            dvp->dva_error = DDI_FAILURE;
            return (DDI_WALK_TERMINATE);
        }
    }

    immu_devi = immu_devi_get(pdip);
    immu = immu_devi->imd_immu;
    if (immu == NULL)
        immu = immu_dvma_get_immu(pdip, dvp->dva_flags);

    /*
     * If we encounter a PCIE_PCIE bridge *ANCESTOR* we need to
     * terminate the walk (since the device under the PCIE bridge
     * is a PCIE device and has an independent entry in the
     * root/context table)
     */
    if (dvp->dva_rdip != pdip &&
        immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCIE) {
        return (DDI_WALK_TERMINATE);
    }

    /*
     * In order to be a domain-dim, it must be a PCI device i.e.
     * must have valid BDF. This also eliminates the root complex.
     */
    if (immu_devi->imd_pcib_type != IMMU_PCIB_BAD &&
        immu_devi->imd_pcib_type != IMMU_PCIB_NOBDF) {
        ASSERT(immu_devi->imd_bus >= 0);
        ASSERT(immu_devi->imd_devfunc >= 0);
        dvp->dva_ddip = pdip;
    }

    if (immu_devi->imd_display == B_TRUE ||
        (dvp->dva_flags & IMMU_FLAGS_UNITY)) {
        dvp->dva_domain = immu->immu_unity_domain;
        /* continue walking to find ddip */
        return (DDI_WALK_CONTINUE);
    }

    mutex_enter(&(DEVI(pdip)->devi_lock));
    domain = immu_devi->imd_domain;
    ddip = immu_devi->imd_ddip;
    mutex_exit(&(DEVI(pdip)->devi_lock));

    if (domain && ddip) {
        /* if domain is set, it must be the same */
        if (dvp->dva_domain) {
            ASSERT(domain == dvp->dva_domain);
        }
        dvp->dva_domain = domain;
        dvp->dva_ddip = ddip;
        return (DDI_WALK_TERMINATE);
    }

    /* Domain may already be set, continue walking so that ddip gets set */
    if (dvp->dva_domain) {
        return (DDI_WALK_CONTINUE);
    }

    /* domain is not set in either immu_devi or dvp */
    domain = bdf_domain_lookup(immu_devi);
    if (domain == NULL) {
        return (DDI_WALK_CONTINUE);
    }

    /* ok, the BDF hash had a domain for this BDF. */

    /* Grab lock again to check if something else set immu_devi fields */
    mutex_enter(&(DEVI(pdip)->devi_lock));
    if (immu_devi->imd_domain != NULL) {
        dvp->dva_domain = domain;
    } else {
        dvp->dva_domain = domain;
    }
    mutex_exit(&(DEVI(pdip)->devi_lock));

    /*
     * walk upwards until the topmost PCI bridge is found
     */
    return (DDI_WALK_CONTINUE);

}

static void
map_unity_domain(domain_t *domain)
{
    struct memlist *mp;
    uint64_t start;
    uint64_t npages;
    immu_dcookie_t dcookies[1] = {0};
    int dcount = 0;

    /*
     * UNITY arenas are a mirror of the physical memory
     * installed on the system.
     */

#ifdef BUGGY_DRIVERS
    /*
     * Dont skip page0. Some broken HW/FW access it.
     */
    dcookies[0].dck_paddr = 0;
    dcookies[0].dck_npages = 1;
    dcount = 1;
    (void) dvma_map(domain, 0, 1, dcookies, dcount, NULL,
        IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1);
#endif

    memlist_read_lock();

    mp = phys_install;

    if (mp->ml_address == 0) {
        /* since we already mapped page1 above */
        start = IMMU_PAGESIZE;
    } else {
        start = mp->ml_address;
    }
    npages = mp->ml_size/IMMU_PAGESIZE + 1;

    dcookies[0].dck_paddr = start;
    dcookies[0].dck_npages = npages;
    dcount = 1;
    (void) dvma_map(domain, start, npages, dcookies,
        dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

    ddi_err(DER_LOG, domain->dom_dip, "iommu: mapping PHYS span [0x%" PRIx64
        " - 0x%" PRIx64 "]", start, start + mp->ml_size);

    mp = mp->ml_next;
    while (mp) {
        ddi_err(DER_LOG, domain->dom_dip,
            "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]",
            mp->ml_address, mp->ml_address + mp->ml_size);

        start = mp->ml_address;
        npages = mp->ml_size/IMMU_PAGESIZE + 1;

        dcookies[0].dck_paddr = start;
        dcookies[0].dck_npages = npages;
        dcount = 1;
        (void) dvma_map(domain, start, npages,
            dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
        mp = mp->ml_next;
    }

    mp = bios_rsvd;
    while (mp) {
        ddi_err(DER_LOG, domain->dom_dip,
            "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]",
            mp->ml_address, mp->ml_address + mp->ml_size);

        start = mp->ml_address;
        npages = mp->ml_size/IMMU_PAGESIZE + 1;

        dcookies[0].dck_paddr = start;
        dcookies[0].dck_npages = npages;
        dcount = 1;
        (void) dvma_map(domain, start, npages,
            dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

        mp = mp->ml_next;
    }

    memlist_read_unlock();
}

/*
 * create_xlate_arena()
 *  Create the dvma arena for a domain with translation
 *  mapping
 */
static void
create_xlate_arena(immu_t *immu, domain_t *domain,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
    char *arena_name;
    struct memlist *mp;
    int vmem_flags;
    uint64_t start;
    uint_t mgaw;
    uint64_t size;
    uint64_t maxaddr;
    void *vmem_ret;

    arena_name = domain->dom_dvma_arena_name;

    /* Note, don't do sizeof (arena_name) - it is just a pointer */
    (void) snprintf(arena_name,
        sizeof (domain->dom_dvma_arena_name),
        "%s-domain-%d-xlate-DVMA-arena", immu->immu_name,
        domain->dom_did);

    vmem_flags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP;

    /* Restrict mgaddr (max guest addr) to MGAW */
    mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);

    /*
     * To ensure we avoid ioapic and PCI MMIO ranges we just
     * use the physical memory address range of the system as the
     * range
     */
    maxaddr = ((uint64_t)1 << mgaw);

    memlist_read_lock();

    mp = phys_install;

    if (mp->ml_address == 0)
        start = MMU_PAGESIZE;
    else
        start = mp->ml_address;

    if (start + mp->ml_size > maxaddr)
        size = maxaddr - start;
    else
        size = mp->ml_size;

    ddi_err(DER_VERB, rdip,
        "iommu: %s: Creating dvma vmem arena [0x%" PRIx64
        " - 0x%" PRIx64 "]", arena_name, start, start + size);

    /*
     * We always allocate in quanta of IMMU_PAGESIZE
     */
    domain->dom_dvma_arena = vmem_create(arena_name,
        (void *)(uintptr_t)start,   /* start addr */
        size,           /* size */
        IMMU_PAGESIZE,      /* quantum */
        NULL,           /* afunc */
        NULL,           /* ffunc */
        NULL,           /* source */
        0,              /* qcache_max */
        vmem_flags);

    if (domain->dom_dvma_arena == NULL) {
        ddi_err(DER_PANIC, rdip,
            "Failed to allocate DVMA arena(%s) "
            "for domain ID (%d)", arena_name, domain->dom_did);
        /*NOTREACHED*/
    }

    mp = mp->ml_next;
    while (mp) {

        if (mp->ml_address == 0)
            start = MMU_PAGESIZE;
        else
            start = mp->ml_address;

        if (start + mp->ml_size > maxaddr)
            size = maxaddr - start;
        else
            size = mp->ml_size;

        ddi_err(DER_VERB, rdip,
            "iommu: %s: Adding dvma vmem span [0x%" PRIx64
            " - 0x%" PRIx64 "]", arena_name, start,
            start + size);

        vmem_ret = vmem_add(domain->dom_dvma_arena,
            (void *)(uintptr_t)start, size,  vmem_flags);

        if (vmem_ret == NULL) {
            ddi_err(DER_PANIC, rdip,
                "Failed to allocate DVMA arena(%s) "
                "for domain ID (%d)",
                arena_name, domain->dom_did);
            /*NOTREACHED*/
        }
        mp = mp->ml_next;
    }
    memlist_read_unlock();
}

/* ################################### DOMAIN CODE ######################### */

/*
 * Set the domain and domain-dip for a dip
 */
static void
set_domain(
    dev_info_t *dip,
    dev_info_t *ddip,
    domain_t *domain)
{
    immu_devi_t *immu_devi;
    domain_t *fdomain;
    dev_info_t *fddip;

    immu_devi = immu_devi_get(dip);

    mutex_enter(&(DEVI(dip)->devi_lock));
    fddip = immu_devi->imd_ddip;
    fdomain = immu_devi->imd_domain;

    if (fddip) {
        ASSERT(fddip == ddip);
    } else {
        immu_devi->imd_ddip = ddip;
    }

    if (fdomain) {
        ASSERT(fdomain == domain);
    } else {
        immu_devi->imd_domain = domain;
    }
    mutex_exit(&(DEVI(dip)->devi_lock));
}

/*
 * device_domain()
 *  Get domain for a device. The domain may be global in which case it
 *  is shared between all IOMMU units. Due to potential AGAW differences
 *      between IOMMU units, such global domains *have to be* UNITY mapping
 *      domains. Alternatively, the domain may be local to a IOMMU unit.
 *  Local domains may be shared or immu_devi, although the
 *      scope of sharing
 *  is restricted to devices controlled by the IOMMU unit to
 *      which the domain
 *  belongs. If shared, they (currently) have to be UNITY domains. If
 *      immu_devi a domain may be either UNITY or translation (XLATE) domain.
 */
static domain_t *
device_domain(dev_info_t *rdip, dev_info_t **ddipp, immu_flags_t immu_flags)
{
    dev_info_t *ddip; /* topmost dip in domain i.e. domain owner */
    immu_t *immu;
    domain_t *domain;
    dvma_arg_t dvarg = {0};
    int level;

    *ddipp = NULL;

    /*
     * Check if the domain is already set. This is usually true
     * if this is not the first DVMA transaction.
     */
    ddip = NULL;
    domain = immu_devi_domain(rdip, &ddip);
    if (domain) {
        *ddipp = ddip;
        return (domain);
    }

    immu = immu_dvma_get_immu(rdip, immu_flags);
    if (immu == NULL) {
        /*
         * possible that there is no IOMMU unit for this device
         * - BIOS bugs are one example.
         */
        ddi_err(DER_WARN, rdip, "No iommu unit found for device");
        return (NULL);
    }

    immu_flags |= immu_devi_get(rdip)->imd_dvma_flags;

    dvarg.dva_rdip = rdip;
    dvarg.dva_ddip = NULL;
    dvarg.dva_domain = NULL;
    dvarg.dva_flags = immu_flags;
    level = 0;
    if (immu_walk_ancestor(rdip, NULL, get_branch_domain,
        &dvarg, &level, immu_flags) != DDI_SUCCESS) {
        /*
         * maybe low memory. return error,
         * so driver tries again later
         */
        return (NULL);
    }

    /* should have walked at least 1 dip (i.e. edip) */
    ASSERT(level > 0);

    ddip = dvarg.dva_ddip;  /* must be present */
    domain = dvarg.dva_domain;  /* may be NULL */

    /*
     * We may find the domain during our ancestor walk on any one of our
     * ancestor dips, If the domain is found then the domain-dip
     * (i.e. ddip) will also be found in the same immu_devi struct.
     * The domain-dip is the highest ancestor dip which shares the
     * same domain with edip.
     * The domain may or may not be found, but the domain dip must
     * be found.
     */
    if (ddip == NULL) {
        ddi_err(DER_MODE, rdip, "Cannot find domain dip for device.");
        return (NULL);
    }

    /*
     * Did we find a domain ?
     */
    if (domain) {
        goto found;
    }

    /* nope, so allocate */
    domain = domain_create(immu, ddip, rdip, immu_flags);
    if (domain == NULL) {
        return (NULL);
    }

    /*FALLTHROUGH*/
found:
    /*
     * We know *domain *is* the right domain, so panic if
     * another domain is set for either the request-dip or
     * effective dip.
     */
    set_domain(ddip, ddip, domain);
    set_domain(rdip, ddip, domain);

    *ddipp = ddip;
    return (domain);
}

static void
create_unity_domain(immu_t *immu)
{
    domain_t *domain;

    /* domain created during boot and always use sleep flag */
    domain = kmem_zalloc(sizeof (domain_t), KM_SLEEP);

    rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);

    domain->dom_did = IMMU_UNITY_DID;
    domain->dom_maptype = IMMU_MAPTYPE_UNITY;

    domain->dom_immu = immu;
    immu->immu_unity_domain = domain;

    /*
     * Setup the domain's initial page table
     * should never fail.
     */
    domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
    pgtable_zero(domain->dom_pgtable_root);

    /*
     * Only map all physical memory in to the unity domain
     * if passthrough is not supported. If it is supported,
     * passthrough is set in the context entry instead.
     */
    if (!IMMU_ECAP_GET_PT(immu->immu_regs_excap))
        map_unity_domain(domain);


    /*
     * put it on the system-wide UNITY domain list
     */
    mutex_enter(&(immu_domain_lock));
    list_insert_tail(&immu_unity_domain_list, domain);
    mutex_exit(&(immu_domain_lock));
}

/*
 * ddip is the domain-dip - the topmost dip in a domain
 * rdip is the requesting-dip - the device which is
 * requesting DVMA setup
 * if domain is a non-shared domain rdip == ddip
 */
static domain_t *
domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,
    immu_flags_t immu_flags)
{
    int kmflags;
    domain_t *domain;
    char mod_hash_name[128];
    immu_devi_t *immu_devi;
    int did;
    immu_dcookie_t dcookies[1] = {0};
    int dcount = 0;

    immu_devi = immu_devi_get(rdip);

    /*
     * First allocate a domainid.
     * This routine will never fail, since if we run out
     * of domains the unity domain will be allocated.
     */
    did = did_alloc(immu, rdip, ddip, immu_flags);
    if (did == IMMU_UNITY_DID) {
        /* domain overflow */
        ASSERT(immu->immu_unity_domain);
        return (immu->immu_unity_domain);
    }

    kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
    domain = kmem_zalloc(sizeof (domain_t), kmflags);
    if (domain == NULL) {
        ddi_err(DER_PANIC, rdip, "Failed to alloc DVMA domain "
            "structure for device. IOMMU unit: %s", immu->immu_name);
        /*NOTREACHED*/
    }

    rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);

    (void) snprintf(mod_hash_name, sizeof (mod_hash_name),
        "immu%s-domain%d-pava-hash", immu->immu_name, did);

    domain->dom_did = did;
    domain->dom_immu = immu;
    domain->dom_maptype = IMMU_MAPTYPE_XLATE;
    domain->dom_dip = ddip;

    /*
     * Create xlate DVMA arena for this domain.
     */
    create_xlate_arena(immu, domain, rdip, immu_flags);

    /*
     * Setup the domain's initial page table
     */
    domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);
    if (domain->dom_pgtable_root == NULL) {
        ddi_err(DER_PANIC, rdip, "Failed to alloc root "
            "pgtable for domain (%d). IOMMU unit: %s",
            domain->dom_did, immu->immu_name);
        /*NOTREACHED*/
    }
    pgtable_zero(domain->dom_pgtable_root);

    /*
     * Since this is a immu unit-specific domain, put it on
     * the per-immu domain list.
     */
    mutex_enter(&(immu->immu_lock));
    list_insert_head(&immu->immu_domain_list, domain);
    mutex_exit(&(immu->immu_lock));

    /*
     * Also put it on the system-wide xlate domain list
     */
    mutex_enter(&(immu_domain_lock));
    list_insert_head(&immu_xlate_domain_list, domain);
    mutex_exit(&(immu_domain_lock));

    bdf_domain_insert(immu_devi, domain);

#ifdef BUGGY_DRIVERS
    /*
     * Map page0. Some broken HW/FW access it.
     */
    dcookies[0].dck_paddr = 0;
    dcookies[0].dck_npages = 1;
    dcount = 1;
    (void) dvma_map(domain, 0, 1, dcookies, dcount, NULL,
        IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1);
#endif
    return (domain);
}

/*
 * Create domainid arena.
 * Domainid 0 is reserved by Vt-d spec and cannot be used by
 * system software.
 * Domainid 1 is reserved by solaris and used for *all* of the following:
 *  as the "uninitialized" domain - For devices not yet controlled
 *  by Solaris
 *  as the "unity" domain - For devices that will always belong
 *  to the unity domain
 *  as the "overflow" domain - Used for any new device after we
 *  run out of domains
 * All of the above domains map into a single domain with
 * domainid 1 and UNITY DVMA mapping
 * Each IMMU unity has its own unity/uninit/overflow domain
 */
static void
did_init(immu_t *immu)
{
    (void) snprintf(immu->immu_did_arena_name,
        sizeof (immu->immu_did_arena_name),
        "%s_domainid_arena", immu->immu_name);

    ddi_err(DER_VERB, immu->immu_dip, "creating domainid arena %s",
        immu->immu_did_arena_name);

    immu->immu_did_arena = vmem_create(
        immu->immu_did_arena_name,
        (void *)(uintptr_t)(IMMU_UNITY_DID + 1),   /* start addr */
        immu->immu_max_domains - IMMU_UNITY_DID,
        1,              /* quantum */
        NULL,           /* afunc */
        NULL,           /* ffunc */
        NULL,           /* source */
        0,              /* qcache_max */
        VM_SLEEP);

    /* Even with SLEEP flag, vmem_create() can fail */
    if (immu->immu_did_arena == NULL) {
        ddi_err(DER_PANIC, NULL, "%s: Failed to create Intel "
            "IOMMU domainid allocator: %s", immu->immu_name,
            immu->immu_did_arena_name);
    }
}

/* #########################  CONTEXT CODE ################################# */

static void
context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,
    int bus, int devfunc)
{
    pgtable_t *context;
    pgtable_t *pgtable_root;
    hw_rce_t *hw_rent;
    hw_rce_t *hw_cent;
    hw_rce_t *ctxp;
    int sid;
    krw_t rwtype;
    boolean_t fill_root;
    boolean_t fill_ctx;

    pgtable_root = domain->dom_pgtable_root;

    ctxp = (hw_rce_t *)(root_table->swpg_next_array);
    context = *(pgtable_t **)(ctxp + bus);
    hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr) + bus;

    fill_root = B_FALSE;
    fill_ctx = B_FALSE;

    /* Check the most common case first with reader lock */
    rw_enter(&(immu->immu_ctx_rwlock), RW_READER);
    rwtype = RW_READER;
again:
    if (ROOT_GET_P(hw_rent)) {
        hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc;
        if (CONT_GET_AVAIL(hw_cent) == IMMU_CONT_INITED) {
            rw_exit(&(immu->immu_ctx_rwlock));
            return;
        } else {
            fill_ctx = B_TRUE;
        }
    } else {
        fill_root = B_TRUE;
        fill_ctx = B_TRUE;
    }

    if (rwtype == RW_READER &&
        rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) {
        rw_exit(&(immu->immu_ctx_rwlock));
        rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
        rwtype = RW_WRITER;
        goto again;
    }
    rwtype = RW_WRITER;

    if (fill_root == B_TRUE) {
        ROOT_SET_CONT(hw_rent, context->hwpg_paddr);
        ROOT_SET_P(hw_rent);
        immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t));
    }

    if (fill_ctx == B_TRUE) {
        hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc;
        /* need to disable context entry before reprogramming it */
        bzero(hw_cent, sizeof (hw_rce_t));

        /* flush caches */
        immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));

        sid = ((bus << 8) | devfunc);
        immu_flush_context_fsi(immu, 0, sid, domain->dom_did,
            &immu->immu_ctx_inv_wait);

        CONT_SET_AVAIL(hw_cent, IMMU_CONT_INITED);
        CONT_SET_DID(hw_cent, domain->dom_did);
        CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
        CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr);
        if (domain->dom_did == IMMU_UNITY_DID &&
            IMMU_ECAP_GET_PT(immu->immu_regs_excap))
            CONT_SET_TTYPE(hw_cent, TTYPE_PASSTHRU);
        else
            /*LINTED*/
            CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY);
        CONT_SET_P(hw_cent);
        if (IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {
            CONT_SET_EH(hw_cent);
            if (immu_use_alh)
                CONT_SET_ALH(hw_cent);
        }
        immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
    }
    rw_exit(&(immu->immu_ctx_rwlock));
}

static pgtable_t *
context_create(immu_t *immu)
{
    int bus;
    int devfunc;
    pgtable_t *root_table;
    pgtable_t *context;
    pgtable_t *pgtable_root;
    hw_rce_t *ctxp;
    hw_rce_t *hw_rent;
    hw_rce_t *hw_cent;

    /* Allocate a zeroed root table (4K 256b entries) */
    root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
    pgtable_zero(root_table);

    /*
     * Setup context tables for all possible root table entries.
     * Start out with unity domains for all entries.
     */
    ctxp = (hw_rce_t *)(root_table->swpg_next_array);
    hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr);
    for (bus = 0; bus < IMMU_ROOT_NUM; bus++, ctxp++, hw_rent++) {
        context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
        pgtable_zero(context);
        ROOT_SET_P(hw_rent);
        ROOT_SET_CONT(hw_rent, context->hwpg_paddr);
        hw_cent = (hw_rce_t *)(context->hwpg_vaddr);
        for (devfunc = 0; devfunc < IMMU_CONT_NUM;
            devfunc++, hw_cent++) {
            pgtable_root =
                immu->immu_unity_domain->dom_pgtable_root;
            CONT_SET_DID(hw_cent,
                immu->immu_unity_domain->dom_did);
            CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
            CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr);
            if (IMMU_ECAP_GET_PT(immu->immu_regs_excap))
                CONT_SET_TTYPE(hw_cent, TTYPE_PASSTHRU);
            else
                /*LINTED*/
                CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY);
            CONT_SET_AVAIL(hw_cent, IMMU_CONT_UNINITED);
            CONT_SET_P(hw_cent);
        }
        immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE);
        *((pgtable_t **)ctxp) = context;
    }

    return (root_table);
}

/*
 * Called during rootnex attach, so no locks needed
 */
static void
context_init(immu_t *immu)
{
    rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL);

    immu_init_inv_wait(&immu->immu_ctx_inv_wait, "ctxglobal", B_TRUE);

    immu_regs_wbf_flush(immu);

    immu->immu_ctx_root = context_create(immu);

    immu_regs_set_root_table(immu);

    rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
    immu_flush_context_gbl(immu, &immu->immu_ctx_inv_wait);
    immu_flush_iotlb_gbl(immu, &immu->immu_ctx_inv_wait);
    rw_exit(&(immu->immu_ctx_rwlock));
}


/*
 * Find top pcib
 */
static int
find_top_pcib(dev_info_t *dip, void *arg)
{
    immu_devi_t *immu_devi;
    dev_info_t **pcibdipp = (dev_info_t **)arg;

    immu_devi = immu_devi_get(dip);

    if (immu_devi->imd_pcib_type == IMMU_PCIB_PCI_PCI) {
        *pcibdipp = dip;
    }

    return (DDI_WALK_CONTINUE);
}

static int
immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
    immu_devi_t *r_immu_devi;
    immu_devi_t *d_immu_devi;
    int r_bus;
    int d_bus;
    int r_devfunc;
    int d_devfunc;
    immu_pcib_t d_pcib_type;
    dev_info_t *pcibdip;

    if (ddip == NULL || rdip == NULL ||
        ddip == root_devinfo || rdip == root_devinfo) {
        ddi_err(DER_MODE, rdip, "immu_contexts_update: domain-dip or "
            "request-dip are NULL or are root devinfo");
        return (DDI_FAILURE);
    }

    /*
     * We need to set the context fields
     * based on what type of device rdip and ddip are.
     * To do that we need the immu_devi field.
     * Set the immu_devi field (if not already set)
     */
    if (immu_devi_set(ddip, immu_flags) == DDI_FAILURE) {
        ddi_err(DER_MODE, rdip,
            "immu_context_update: failed to set immu_devi for ddip");
        return (DDI_FAILURE);
    }

    if (immu_devi_set(rdip, immu_flags) == DDI_FAILURE) {
        ddi_err(DER_MODE, rdip,
            "immu_context_update: failed to set immu_devi for rdip");
        return (DDI_FAILURE);
    }

    d_immu_devi = immu_devi_get(ddip);
    r_immu_devi = immu_devi_get(rdip);

    d_bus = d_immu_devi->imd_bus;
    d_devfunc = d_immu_devi->imd_devfunc;
    d_pcib_type = d_immu_devi->imd_pcib_type;
    r_bus = r_immu_devi->imd_bus;
    r_devfunc = r_immu_devi->imd_devfunc;

    if (rdip == ddip) {
        /* rdip is a PCIE device. set context for it only */
        context_set(immu, domain, immu->immu_ctx_root, r_bus,
            r_devfunc);
#ifdef BUGGY_DRIVERS
    } else if (r_immu_devi == d_immu_devi) {
#ifdef TEST
        ddi_err(DER_WARN, rdip, "Driver bug: Devices 0x%lx and "
            "0x%lx are identical", rdip, ddip);
#endif
        /* rdip is a PCIE device. set context for it only */
        context_set(immu, domain, immu->immu_ctx_root, r_bus,
            r_devfunc);
#endif
    } else if (d_pcib_type == IMMU_PCIB_PCIE_PCI) {
        /*
         * ddip is a PCIE_PCI bridge. Set context for ddip's
         * secondary bus. If rdip is on ddip's secondary
         * bus, set context for rdip. Else, set context
         * for rdip's PCI bridge on ddip's secondary bus.
         */
        context_set(immu, domain, immu->immu_ctx_root,
            d_immu_devi->imd_sec, 0);
        if (d_immu_devi->imd_sec == r_bus) {
            context_set(immu, domain, immu->immu_ctx_root,
                r_bus, r_devfunc);
        } else {
            pcibdip = NULL;
            if (immu_walk_ancestor(rdip, ddip, find_top_pcib,
                &pcibdip, NULL, immu_flags) == DDI_SUCCESS &&
                pcibdip != NULL) {
                r_immu_devi = immu_devi_get(pcibdip);
                r_bus = r_immu_devi->imd_bus;
                r_devfunc = r_immu_devi->imd_devfunc;
                context_set(immu, domain, immu->immu_ctx_root,
                    r_bus, r_devfunc);
            } else {
                ddi_err(DER_PANIC, rdip, "Failed to find PCI "
                    " bridge for PCI device");
                /*NOTREACHED*/
            }
        }
    } else if (d_pcib_type == IMMU_PCIB_PCI_PCI) {
        context_set(immu, domain, immu->immu_ctx_root, d_bus,
            d_devfunc);
    } else if (d_pcib_type == IMMU_PCIB_ENDPOINT) {
        /*
         * ddip is a PCIE device which has a non-PCI device under it
         * i.e. it is a PCI-nonPCI bridge. Example: pciicde-ata
         */
        context_set(immu, domain, immu->immu_ctx_root, d_bus,
            d_devfunc);
    } else {
        ddi_err(DER_PANIC, rdip, "unknown device type. Cannot "
            "set iommu context.");
        /*NOTREACHED*/
    }

    /* XXX do we need a membar_producer() here */
    return (DDI_SUCCESS);
}

/* ##################### END CONTEXT CODE ################################## */
/* ##################### MAPPING CODE ################################## */


#ifdef DEBUG
static boolean_t
PDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
    /* The PDTE must be set i.e. present bit is set */
    if (!PDTE_P(pdte)) {
        ddi_err(DER_MODE, rdip, "No present flag");
        return (B_FALSE);
    }

    /*
     * Just assert to check most significant system software field
     * (PDTE_SW4) as it is same as present bit and we
     * checked that above
     */
    ASSERT(PDTE_SW4(pdte));

    /*
     * TM field should be clear if not reserved.
     * non-leaf is always reserved
     */
    if (next == NULL && immu->immu_TM_reserved == B_FALSE) {
        if (PDTE_TM(pdte)) {
            ddi_err(DER_MODE, rdip, "TM flag set");
            return (B_FALSE);
        }
    }

    /*
     * The SW3 field is not used and must be clear
     */
    if (PDTE_SW3(pdte)) {
        ddi_err(DER_MODE, rdip, "SW3 set");
        return (B_FALSE);
    }

    /*
     * PFN (for PTE) or next level pgtable-paddr (for PDE) must be set
     */
    if (next == NULL) {
        ASSERT(paddr % IMMU_PAGESIZE == 0);
        if (PDTE_PADDR(pdte) != paddr) {
            ddi_err(DER_MODE, rdip,
                "PTE paddr mismatch: %lx != %lx",
                PDTE_PADDR(pdte), paddr);
            return (B_FALSE);
        }
    } else {
        if (PDTE_PADDR(pdte) != next->hwpg_paddr) {
            ddi_err(DER_MODE, rdip,
                "PDE paddr mismatch: %lx != %lx",
                PDTE_PADDR(pdte), next->hwpg_paddr);
            return (B_FALSE);
        }
    }

    /*
     * SNP field should be clear if not reserved.
     * non-leaf is always reserved
     */
    if (next == NULL && immu->immu_SNP_reserved == B_FALSE) {
        if (PDTE_SNP(pdte)) {
            ddi_err(DER_MODE, rdip, "SNP set");
            return (B_FALSE);
        }
    }

    /* second field available for system software should be clear */
    if (PDTE_SW2(pdte)) {
        ddi_err(DER_MODE, rdip, "SW2 set");
        return (B_FALSE);
    }

    /* Super pages field should be clear */
    if (PDTE_SP(pdte)) {
        ddi_err(DER_MODE, rdip, "SP set");
        return (B_FALSE);
    }

    /*
     * least significant field available for
     * system software should be clear
     */
    if (PDTE_SW1(pdte)) {
        ddi_err(DER_MODE, rdip, "SW1 set");
        return (B_FALSE);
    }

    if ((immu_flags & IMMU_FLAGS_READ) && !PDTE_READ(pdte)) {
        ddi_err(DER_MODE, rdip, "READ not set");
        return (B_FALSE);
    }

    if ((immu_flags & IMMU_FLAGS_WRITE) && !PDTE_WRITE(pdte)) {
        ddi_err(DER_MODE, rdip, "WRITE not set");
        return (B_FALSE);
    }

    return (B_TRUE);
}
#endif

/*ARGSUSED*/
static void
PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
    uint64_t *dvma_ptr, uint64_t *npages_ptr, dev_info_t *rdip)
{
    uint64_t npages;
    uint64_t dvma;
    pgtable_t *pgtable;
    hw_pdte_t *hwp;
    hw_pdte_t *shwp;
    int idx;

    pgtable = xlate->xlt_pgtable;
    idx = xlate->xlt_idx;

    dvma = *dvma_ptr;
    npages = *npages_ptr;

    /*
     * since a caller gets a unique dvma for a physical address,
     * no other concurrent thread will be writing to the same
     * PTE even if it has the same paddr. So no locks needed.
     */
    shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;

    hwp = shwp;
    for (; npages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) {
        PDTE_CLEAR_P(*hwp);
        dvma += IMMU_PAGESIZE;
        npages--;
    }

    *dvma_ptr = dvma;
    *npages_ptr = npages;

    xlate->xlt_idx = idx;
}

static void
xlate_setup(uint64_t dvma, xlate_t *xlate, int nlevels)
{
    int level;
    uint64_t offbits;

    /*
     * Skip the first 12 bits which is the offset into
     * 4K PFN (phys page frame based on IMMU_PAGESIZE)
     */
    offbits = dvma >> IMMU_PAGESHIFT;

    /* skip to level 1 i.e. leaf PTE */
    for (level = 1, xlate++; level <= nlevels; level++, xlate++) {
        xlate->xlt_level = level;
        xlate->xlt_idx = (offbits & IMMU_PGTABLE_LEVEL_MASK);
        ASSERT(xlate->xlt_idx <= IMMU_PGTABLE_MAXIDX);
        xlate->xlt_pgtable = NULL;
        offbits >>= IMMU_PGTABLE_LEVEL_STRIDE;
    }
}

/*
 * Read the pgtables
 */
static boolean_t
PDE_lookup(domain_t *domain, xlate_t *xlate, int nlevels)
{
    pgtable_t *pgtable;
    pgtable_t *next;
    uint_t idx;

    /* start with highest level pgtable i.e. root */
    xlate += nlevels;

    if (xlate->xlt_pgtable == NULL) {
        xlate->xlt_pgtable = domain->dom_pgtable_root;
    }

    for (; xlate->xlt_level > 1; xlate--) {
        idx = xlate->xlt_idx;
        pgtable = xlate->xlt_pgtable;

        if ((xlate - 1)->xlt_pgtable) {
            continue;
        }

        /* Lock the pgtable in read mode */
        rw_enter(&(pgtable->swpg_rwlock), RW_READER);

        /*
         * since we are unmapping, the pgtable should
         * already point to a leafier pgtable.
         */
        next = *(pgtable->swpg_next_array + idx);
        (xlate - 1)->xlt_pgtable = next;
        rw_exit(&(pgtable->swpg_rwlock));
        if (next == NULL)
            return (B_FALSE);
    }

    return (B_TRUE);
}

static void
immu_fault_walk(void *arg, void *base, size_t len)
{
    uint64_t dvma, start;

    dvma = *(uint64_t *)arg;
    start = (uint64_t)(uintptr_t)base;

    if (dvma >= start && dvma < (start + len)) {
        ddi_err(DER_WARN, NULL,
            "faulting DVMA address is in vmem arena "
            "(%" PRIx64 "-%" PRIx64 ")",
            start, start + len);
        *(uint64_t *)arg = ~0ULL;
    }
}

void
immu_print_fault_info(uint_t sid, uint64_t dvma)
{
    int nlevels;
    xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
    xlate_t *xlatep;
    hw_pdte_t pte;
    domain_t *domain;
    immu_t *immu;
    uint64_t dvma_arg;

    if (mod_hash_find(bdf_domain_hash,
        (void *)(uintptr_t)sid, (void *)&domain) != 0) {
        ddi_err(DER_WARN, NULL,
            "no domain for faulting SID %08x", sid);
        return;
    }

    immu = domain->dom_immu;

    dvma_arg = dvma;
    vmem_walk(domain->dom_dvma_arena, VMEM_ALLOC, immu_fault_walk,
        (void *)&dvma_arg);
    if (dvma_arg != ~0ULL)
        ddi_err(DER_WARN, domain->dom_dip,
            "faulting DVMA address is not in vmem arena");

    nlevels = immu->immu_dvma_nlevels;
    xlate_setup(dvma, xlate, nlevels);

    if (!PDE_lookup(domain, xlate, nlevels)) {
        ddi_err(DER_WARN, domain->dom_dip,
            "pte not found in domid %d for faulting addr %" PRIx64,
            domain->dom_did, dvma);
        return;
    }

    xlatep = &xlate[1];
    pte = *((hw_pdte_t *)
        (xlatep->xlt_pgtable->hwpg_vaddr) + xlatep->xlt_idx);

    ddi_err(DER_WARN, domain->dom_dip,
        "domid %d pte: %" PRIx64 "(paddr %" PRIx64 ")", domain->dom_did,
        (unsigned long long)pte, (unsigned long long)PDTE_PADDR(pte));
}

/*ARGSUSED*/
static void
PTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
    hw_pdte_t pte;

#ifndef DEBUG
    pte = immu->immu_ptemask;
    PDTE_SET_PADDR(pte, paddr);
#else
    pte = *hwp;

    if (PDTE_P(pte)) {
        if (PDTE_PADDR(pte) != paddr) {
            ddi_err(DER_MODE, rdip, "PTE paddr %lx != paddr %lx",
                PDTE_PADDR(pte), paddr);
        }
#ifdef BUGGY_DRIVERS
        return;
#else
        goto out;
#endif
    }

    /* clear TM field if not reserved */
    if (immu->immu_TM_reserved == B_FALSE) {
        PDTE_CLEAR_TM(pte);
    }

    /* Clear 3rd field for system software  - not used */
    PDTE_CLEAR_SW3(pte);

    /* Set paddr */
    ASSERT(paddr % IMMU_PAGESIZE == 0);
    PDTE_CLEAR_PADDR(pte);
    PDTE_SET_PADDR(pte, paddr);

    /*  clear SNP field if not reserved. */
    if (immu->immu_SNP_reserved == B_FALSE) {
        PDTE_CLEAR_SNP(pte);
    }

    /* Clear SW2 field available for software */
    PDTE_CLEAR_SW2(pte);


    /* SP is don't care for PTEs. Clear it for cleanliness */
    PDTE_CLEAR_SP(pte);

    /* Clear SW1 field available for software */
    PDTE_CLEAR_SW1(pte);

    /*
     * Now that we are done writing the PTE
     * set the "present" flag. Note this present
     * flag is a bit in the PDE/PTE that the
     * spec says is available for system software.
     * This is an implementation detail of Solaris
     * bare-metal Intel IOMMU.
     * The present field in a PDE/PTE is not defined
     * by the Vt-d spec
     */

    PDTE_SET_P(pte);

    pte |= immu->immu_ptemask;

out:
#endif /* DEBUG */
#ifdef BUGGY_DRIVERS
    PDTE_SET_READ(pte);
    PDTE_SET_WRITE(pte);
#else
    if (immu_flags & IMMU_FLAGS_READ)
        PDTE_SET_READ(pte);
    if (immu_flags & IMMU_FLAGS_WRITE)
        PDTE_SET_WRITE(pte);
#endif /* BUGGY_DRIVERS */

    *hwp = pte;
}

/*ARGSUSED*/
static void
PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
    uint64_t *dvma_ptr, uint64_t *nvpages_ptr, immu_dcookie_t *dcookies,
    int dcount, dev_info_t *rdip, immu_flags_t immu_flags)
{
    paddr_t paddr;
    uint64_t nvpages;
    uint64_t nppages;
    uint64_t dvma;
    pgtable_t *pgtable;
    hw_pdte_t *hwp;
    hw_pdte_t *shwp;
    int idx, nset;
    int j;

    pgtable = xlate->xlt_pgtable;
    idx = xlate->xlt_idx;

    dvma = *dvma_ptr;
    nvpages = *nvpages_ptr;

    /*
     * since a caller gets a unique dvma for a physical address,
     * no other concurrent thread will be writing to the same
     * PTE even if it has the same paddr. So no locks needed.
     */
    shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;

    hwp = shwp;
    for (j = dcount - 1; j >= 0; j--) {
        if (nvpages <= dcookies[j].dck_npages)
            break;
        nvpages -= dcookies[j].dck_npages;
    }

    nppages = nvpages;
    paddr = dcookies[j].dck_paddr +
        (dcookies[j].dck_npages - nppages) * IMMU_PAGESIZE;

    nvpages = *nvpages_ptr;
    nset = 0;
    for (; nvpages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) {
        PTE_set_one(immu, hwp, paddr, rdip, immu_flags);
        nset++;

        ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)
            == B_TRUE);
        nppages--;
        nvpages--;
        paddr += IMMU_PAGESIZE;
        dvma += IMMU_PAGESIZE;

        if (nppages == 0) {
            j++;
        }

        if (j == dcount)
            break;

        if (nppages == 0) {
            nppages = dcookies[j].dck_npages;
            paddr = dcookies[j].dck_paddr;
        }
    }

    if (nvpages) {
        *dvma_ptr = dvma;
        *nvpages_ptr = nvpages;
    } else {
        *dvma_ptr = 0;
        *nvpages_ptr = 0;
    }

    xlate->xlt_idx = idx;
}

/*ARGSUSED*/
static void
PDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
    hw_pdte_t pde;

    pde = *hwp;

    /* if PDE is already set, make sure it is correct */
    if (PDTE_P(pde)) {
        ASSERT(PDTE_PADDR(pde) == next->hwpg_paddr);
#ifdef BUGGY_DRIVERS
        return;
#else
        goto out;
#endif
    }

    /* Dont touch SW4, it is the present bit */

    /* don't touch TM field it is reserved for PDEs */

    /* 3rd field available for system software is not used */
    PDTE_CLEAR_SW3(pde);

    /* Set next level pgtable-paddr for PDE */
    PDTE_CLEAR_PADDR(pde);
    PDTE_SET_PADDR(pde, next->hwpg_paddr);

    /* don't touch SNP field it is reserved for PDEs */

    /* Clear second field available for system software */
    PDTE_CLEAR_SW2(pde);

    /* No super pages for PDEs */
    PDTE_CLEAR_SP(pde);

    /* Clear SW1 for software */
    PDTE_CLEAR_SW1(pde);

    /*
     * Now that we are done writing the PDE
     * set the "present" flag. Note this present
     * flag is a bit in the PDE/PTE that the
     * spec says is available for system software.
     * This is an implementation detail of Solaris
     * base-metal Intel IOMMU.
     * The present field in a PDE/PTE is not defined
     * by the Vt-d spec
     */

out:
#ifdef  BUGGY_DRIVERS
    PDTE_SET_READ(pde);
    PDTE_SET_WRITE(pde);
#else
    if (immu_flags & IMMU_FLAGS_READ)
        PDTE_SET_READ(pde);
    if (immu_flags & IMMU_FLAGS_WRITE)
        PDTE_SET_WRITE(pde);
#endif

    PDTE_SET_P(pde);

    *hwp = pde;
}

/*
 * Used to set PDEs
 */
static boolean_t
PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
    pgtable_t *pgtable;
    pgtable_t *new;
    pgtable_t *next;
    hw_pdte_t *hwp;
    int level;
    uint_t idx;
    krw_t rwtype;
    boolean_t set = B_FALSE;

    /* start with highest level pgtable i.e. root */
    xlate += nlevels;

    new = NULL;
    xlate->xlt_pgtable = domain->dom_pgtable_root;
    for (level = nlevels; level > 1; level--, xlate--) {
        idx = xlate->xlt_idx;
        pgtable = xlate->xlt_pgtable;

        /* Lock the pgtable in READ mode first */
        rw_enter(&(pgtable->swpg_rwlock), RW_READER);
        rwtype = RW_READER;
again:
        hwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;
        next = (pgtable->swpg_next_array)[idx];

        /*
         * check if leafier level already has a pgtable
         * if yes, verify
         */
        if (next == NULL) {
            if (new == NULL) {

                IMMU_DPROBE2(immu__pdp__alloc, dev_info_t *,
                    rdip, int, level);

                new = pgtable_alloc(immu, immu_flags);
                if (new == NULL) {
                    ddi_err(DER_PANIC, rdip,
                        "pgtable alloc err");
                }
                pgtable_zero(new);
            }

            /* Change to a write lock */
            if (rwtype == RW_READER &&
                rw_tryupgrade(&(pgtable->swpg_rwlock)) == 0) {
                rw_exit(&(pgtable->swpg_rwlock));
                rw_enter(&(pgtable->swpg_rwlock), RW_WRITER);
                rwtype = RW_WRITER;
                goto again;
            }
            rwtype = RW_WRITER;
            next = new;
            (pgtable->swpg_next_array)[idx] = next;
            new = NULL;
            PDE_set_one(immu, hwp, next, rdip, immu_flags);
            set = B_TRUE;
            rw_downgrade(&(pgtable->swpg_rwlock));
            rwtype = RW_READER;
        }
#ifndef  BUGGY_DRIVERS
        else {
            hw_pdte_t pde = *hwp;

            /*
             * If buggy driver we already set permission
             * READ+WRITE so nothing to do for that case
             * XXX Check that read writer perms change before
             * actually setting perms. Also need to hold lock
             */
            if (immu_flags & IMMU_FLAGS_READ)
                PDTE_SET_READ(pde);
            if (immu_flags & IMMU_FLAGS_WRITE)
                PDTE_SET_WRITE(pde);

            *hwp = pde;
        }
#endif

        ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)
            == B_TRUE);

        (xlate - 1)->xlt_pgtable = next;
        rw_exit(&(pgtable->swpg_rwlock));
    }

    if (new) {
        pgtable_free(immu, new);
    }

    return (set);
}

/*
 * dvma_map()
 *     map a contiguous range of DVMA pages
 *
 *     immu: IOMMU unit for which we are generating DVMA cookies
 *   domain: domain
 *    sdvma: Starting dvma
 *   spaddr: Starting paddr
 *   npages: Number of pages
 *     rdip: requesting device
 *     immu_flags: flags
 */
static boolean_t
dvma_map(domain_t *domain, uint64_t sdvma, uint64_t snvpages,
    immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,
    immu_flags_t immu_flags)
{
    uint64_t dvma;
    uint64_t n;
    immu_t *immu = domain->dom_immu;
    int nlevels = immu->immu_dvma_nlevels;
    xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
    boolean_t pde_set = B_FALSE;

    n = snvpages;
    dvma = sdvma;

    while (n > 0) {
        xlate_setup(dvma, xlate, nlevels);

        /* Lookup or allocate PGDIRs and PGTABLEs if necessary */
        if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)
            == B_TRUE) {
            pde_set = B_TRUE;
        }

        /* set all matching ptes that fit into this leaf pgtable */
        PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,
            dcount, rdip, immu_flags);
    }

    return (pde_set);
}

/*
 * dvma_unmap()
 *   unmap a range of DVMAs
 *
 * immu: IOMMU unit state
 * domain: domain for requesting device
 * ddip: domain-dip
 * dvma: starting DVMA
 * npages: Number of IMMU pages to be unmapped
 * rdip: requesting device
 */
static void
dvma_unmap(domain_t *domain, uint64_t sdvma, uint64_t snpages,
    dev_info_t *rdip)
{
    immu_t *immu = domain->dom_immu;
    int nlevels = immu->immu_dvma_nlevels;
    xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
    uint64_t n;
    uint64_t dvma;

    dvma = sdvma;
    n = snpages;

    while (n > 0) {
        /* setup the xlate array */
        xlate_setup(dvma, xlate, nlevels);

        /* just lookup existing pgtables. Should never fail */
        if (!PDE_lookup(domain, xlate, nlevels))
            ddi_err(DER_PANIC, rdip,
                "PTE not found for addr %" PRIx64,
                (unsigned long long)dvma);

        /* clear all matching ptes that fit into this leaf pgtable */
        PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);
    }

    /* No need to flush IOTLB after unmap */
}

static uint64_t
dvma_alloc(domain_t *domain, ddi_dma_attr_t *dma_attr, uint_t npages, int kmf)
{
    uint64_t dvma;
    size_t xsize, align;
    uint64_t minaddr, maxaddr;

    /* parameters */
    xsize = npages * IMMU_PAGESIZE;
    align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE);
    minaddr = dma_attr->dma_attr_addr_lo;
    maxaddr = dma_attr->dma_attr_addr_hi + 1;

    /* handle the rollover cases */
    if (maxaddr < dma_attr->dma_attr_addr_hi) {
        maxaddr = dma_attr->dma_attr_addr_hi;
    }

    /*
     * allocate from vmem arena.
     */
    dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,
        xsize, align, 0, 0, (void *)(uintptr_t)minaddr,
        (void *)(uintptr_t)maxaddr, kmf);

    return (dvma);
}

static void
dvma_prealloc(dev_info_t *rdip, immu_hdl_priv_t *ihp, ddi_dma_attr_t *dma_attr)
{
    int nlevels;
    xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0}, *xlp;
    uint64_t dvma, n;
    size_t xsize, align;
    uint64_t minaddr, maxaddr, dmamax;
    int on, npte, pindex;
    hw_pdte_t *shwp;
    immu_t *immu;
    domain_t *domain;

    /* parameters */
    domain = IMMU_DEVI(rdip)->imd_domain;
    immu = domain->dom_immu;
    nlevels = immu->immu_dvma_nlevels;
    xsize = IMMU_NPREPTES * IMMU_PAGESIZE;
    align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE);
    minaddr = dma_attr->dma_attr_addr_lo;
    if (dma_attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)
        dmamax = dma_attr->dma_attr_seg;
    else
        dmamax = dma_attr->dma_attr_addr_hi;
    maxaddr = dmamax + 1;

    if (maxaddr < dmamax)
        maxaddr = dmamax;

    dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,
        xsize, align, 0, dma_attr->dma_attr_seg + 1,
        (void *)(uintptr_t)minaddr, (void *)(uintptr_t)maxaddr, VM_NOSLEEP);

    ihp->ihp_predvma = dvma;
    ihp->ihp_npremapped = 0;
    if (dvma == 0)
        return;

    n = IMMU_NPREPTES;
    pindex = 0;

    /*
     * Set up a mapping at address 0, just so that all PDPs get allocated
     * now. Although this initial mapping should never be used,
     * explicitly set it to read-only, just to be safe.
     */
    while (n > 0) {
        xlate_setup(dvma, xlate, nlevels);

        (void) PDE_set_all(immu, domain, xlate, nlevels, rdip,
            IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

        xlp = &xlate[1];
        shwp = (hw_pdte_t *)(xlp->xlt_pgtable->hwpg_vaddr)
            + xlp->xlt_idx;
        on = n;

        PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie,
            1, rdip, IMMU_FLAGS_READ);

        npte = on - n;

        while (npte > 0) {
            ihp->ihp_preptes[pindex++] = shwp;
#ifdef BUGGY_DRIVERS
            PDTE_CLEAR_WRITE(*shwp);
#endif
            shwp++;
            npte--;
        }
    }
}

static void
dvma_prefree(dev_info_t *rdip, immu_hdl_priv_t *ihp)
{
    domain_t *domain;

    domain = IMMU_DEVI(rdip)->imd_domain;

    if (ihp->ihp_predvma != 0) {
        dvma_unmap(domain, ihp->ihp_predvma, IMMU_NPREPTES, rdip);
        vmem_free(domain->dom_dvma_arena,
            (void *)(uintptr_t)ihp->ihp_predvma,
            IMMU_NPREPTES * IMMU_PAGESIZE);
    }
}

static void
dvma_free(domain_t *domain, uint64_t dvma, uint64_t npages)
{
    uint64_t size = npages * IMMU_PAGESIZE;

    if (domain->dom_maptype != IMMU_MAPTYPE_XLATE)
        return;

    vmem_free(domain->dom_dvma_arena, (void *)(uintptr_t)dvma, size);
}

static int
immu_map_dvmaseg(dev_info_t *rdip, ddi_dma_handle_t handle,
    immu_hdl_priv_t *ihp, struct ddi_dma_req *dmareq,
    ddi_dma_obj_t *dma_out)
{
    domain_t *domain;
    immu_t *immu;
    immu_flags_t immu_flags;
    ddi_dma_atyp_t buftype;
    ddi_dma_obj_t *dmar_object;
    ddi_dma_attr_t *attrp;
    uint64_t offset, paddr, dvma, sdvma, rwmask;
    size_t npages, npgalloc;
    uint_t psize, size, pcnt, dmax;
    page_t **pparray;
    caddr_t vaddr;
    page_t *page;
    struct as *vas;
    immu_dcookie_t *dcookies;
    int pde_set;

    domain = IMMU_DEVI(rdip)->imd_domain;
    immu = domain->dom_immu;
    immu_flags = dma_to_immu_flags(dmareq);

    attrp = &((ddi_dma_impl_t *)handle)->dmai_attr;

    dmar_object = &dmareq->dmar_object;
    pparray = dmar_object->dmao_obj.virt_obj.v_priv;
    vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
    buftype = dmar_object->dmao_type;
    size = dmar_object->dmao_size;

    IMMU_DPROBE3(immu__map__dvma, dev_info_t *, rdip, ddi_dma_atyp_t,
        buftype, uint_t, size);

    dcookies = &ihp->ihp_dcookies[0];

    pcnt = dmax = 0;

    /* retrieve paddr, psize, offset from dmareq */
    if (buftype == DMA_OTYP_PAGES) {
        page = dmar_object->dmao_obj.pp_obj.pp_pp;
        offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
            MMU_PAGEOFFSET;
        paddr = pfn_to_pa(page->p_pagenum) + offset;
        psize = MIN((MMU_PAGESIZE - offset), size);
        page = page->p_next;
        vas = dmar_object->dmao_obj.virt_obj.v_as;
    } else {
        if (vas == NULL) {
            vas = &kas;
        }
        offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
        if (pparray != NULL) {
            paddr = pfn_to_pa(pparray[pcnt]->p_pagenum) + offset;
            psize = MIN((MMU_PAGESIZE - offset), size);
            pcnt++;
        } else {
            paddr = pfn_to_pa(hat_getpfnum(vas->a_hat,
                vaddr)) + offset;
            psize = MIN(size, (MMU_PAGESIZE - offset));
            vaddr += psize;
        }
    }

    npgalloc = IMMU_BTOPR(size + offset);

    if (npgalloc <= IMMU_NPREPTES && ihp->ihp_predvma != 0) {
#ifdef BUGGY_DRIVERS
        rwmask = PDTE_MASK_R | PDTE_MASK_W | immu->immu_ptemask;
#else
        rwmask = immu->immu_ptemask;
        if (immu_flags & IMMU_FLAGS_READ)
            rwmask |= PDTE_MASK_R;
        if (immu_flags & IMMU_FLAGS_WRITE)
            rwmask |= PDTE_MASK_W;
#endif
#ifdef DEBUG
        rwmask |= PDTE_MASK_P;
#endif
        sdvma = ihp->ihp_predvma;
        ihp->ihp_npremapped = npgalloc;
        *ihp->ihp_preptes[0] =
            PDTE_PADDR(paddr & ~MMU_PAGEOFFSET) | rwmask;
    } else {
        ihp->ihp_npremapped = 0;
        sdvma = dvma_alloc(domain, attrp, npgalloc,
            dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
        if (sdvma == 0)
            return (DDI_DMA_NORESOURCES);

        dcookies[0].dck_paddr = (paddr & ~MMU_PAGEOFFSET);
        dcookies[0].dck_npages = 1;
    }

    IMMU_DPROBE3(immu__dvma__alloc, dev_info_t *, rdip, uint64_t, npgalloc,
        uint64_t, sdvma);

    dvma = sdvma;
    pde_set = 0;
    npages = 1;
    size -= psize;
    while (size > 0) {
        /* get the size for this page (i.e. partial or full page) */
        psize = MIN(size, MMU_PAGESIZE);
        if (buftype == DMA_OTYP_PAGES) {
            /* get the paddr from the page_t */
            paddr = pfn_to_pa(page->p_pagenum);
            page = page->p_next;
        } else if (pparray != NULL) {
            /* index into the array of page_t's to get the paddr */
            paddr = pfn_to_pa(pparray[pcnt]->p_pagenum);
            pcnt++;
        } else {
            /* call into the VM to get the paddr */
            paddr = pfn_to_pa(hat_getpfnum(vas->a_hat, vaddr));
            vaddr += psize;
        }

        npages++;

        if (ihp->ihp_npremapped > 0) {
            *ihp->ihp_preptes[npages - 1] =
                PDTE_PADDR(paddr) | rwmask;
        } else if (IMMU_CONTIG_PADDR(dcookies[dmax], paddr)) {
            dcookies[dmax].dck_npages++;
        } else {
            /* No, we need a new dcookie */
            if (dmax == (IMMU_NDCK - 1)) {
                /*
                 * Ran out of dcookies. Map them now.
                 */
                if (dvma_map(domain, dvma,
                    npages, dcookies, dmax + 1, rdip,
                    immu_flags))
                    pde_set++;

                IMMU_DPROBE4(immu__dvmamap__early,
                    dev_info_t *, rdip, uint64_t, dvma,
                    uint_t, npages, uint_t, dmax+1);

                dvma += (npages << IMMU_PAGESHIFT);
                npages = 0;
                dmax = 0;
            } else
                dmax++;
            dcookies[dmax].dck_paddr = paddr;
            dcookies[dmax].dck_npages = 1;
        }
        size -= psize;
    }

    /*
     * Finish up, mapping all, or all of the remaining,
     * physical memory ranges.
     */
    if (ihp->ihp_npremapped == 0 && npages > 0) {
        IMMU_DPROBE4(immu__dvmamap__late, dev_info_t *, rdip, \
            uint64_t, dvma, uint_t, npages, uint_t, dmax+1);

        if (dvma_map(domain, dvma, npages, dcookies,
            dmax + 1, rdip, immu_flags))
            pde_set++;
    }

    /* Invalidate the IOTLB */
    immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc,
        pde_set > 0 ? TLB_IVA_WHOLE : TLB_IVA_LEAF,
        &ihp->ihp_inv_wait);

    ihp->ihp_ndvseg = 1;
    ihp->ihp_dvseg[0].dvs_start = sdvma;
    ihp->ihp_dvseg[0].dvs_len = dmar_object->dmao_size;

    dma_out->dmao_size = dmar_object->dmao_size;
    dma_out->dmao_obj.dvma_obj.dv_off = offset & IMMU_PAGEOFFSET;
    dma_out->dmao_obj.dvma_obj.dv_nseg = 1;
    dma_out->dmao_obj.dvma_obj.dv_seg = &ihp->ihp_dvseg[0];
    dma_out->dmao_type = DMA_OTYP_DVADDR;

    return (DDI_DMA_MAPPED);
}

static int
immu_unmap_dvmaseg(dev_info_t *rdip, ddi_dma_obj_t *dmao)
{
    uint64_t dvma, npages;
    domain_t *domain;
    struct dvmaseg *dvs;

    domain = IMMU_DEVI(rdip)->imd_domain;
    dvs = dmao->dmao_obj.dvma_obj.dv_seg;

    dvma = dvs[0].dvs_start;
    npages = IMMU_BTOPR(dvs[0].dvs_len + dmao->dmao_obj.dvma_obj.dv_off);

#ifdef DEBUG
    /* Unmap only in DEBUG mode */
    dvma_unmap(domain, dvma, npages, rdip);
#endif
    dvma_free(domain, dvma, npages);

    IMMU_DPROBE3(immu__dvma__free, dev_info_t *, rdip, uint_t, npages,
        uint64_t, dvma);

#ifdef DEBUG
    /*
     * In the DEBUG case, the unmap was actually done,
     * but an IOTLB flush was not done. So, an explicit
     * write back flush is needed.
     */
    immu_regs_wbf_flush(domain->dom_immu);
#endif

    return (DDI_SUCCESS);
}

/* ############################# Functions exported ######################## */

/*
 * setup the DVMA subsystem
 * this code runs only for the first IOMMU unit
 */
void
immu_dvma_setup(list_t *listp)
{
    immu_t *immu;
    uint_t kval;
    size_t nchains;

    /* locks */
    mutex_init(&immu_domain_lock, NULL, MUTEX_DEFAULT, NULL);

    /* Create lists */
    list_create(&immu_unity_domain_list, sizeof (domain_t),
        offsetof(domain_t, dom_maptype_node));
    list_create(&immu_xlate_domain_list, sizeof (domain_t),
        offsetof(domain_t, dom_maptype_node));

    /* Setup BDF domain hash */
    nchains = 0xff;
    kval = mod_hash_iddata_gen(nchains);

    bdf_domain_hash = mod_hash_create_extended("BDF-DOMAIN_HASH",
        nchains, mod_hash_null_keydtor, mod_hash_null_valdtor,
        mod_hash_byid, (void *)(uintptr_t)kval, mod_hash_idkey_cmp,
        KM_NOSLEEP);

    immu = list_head(listp);
    for (; immu; immu = list_next(listp, immu)) {
        create_unity_domain(immu);
        did_init(immu);
        context_init(immu);
        immu->immu_dvma_setup = B_TRUE;
    }
}

/*
 * Startup up one DVMA unit
 */
void
immu_dvma_startup(immu_t *immu)
{
    if (immu_gfxdvma_enable == B_FALSE &&
        immu->immu_dvma_gfx_only == B_TRUE) {
        return;
    }

    /*
     * DVMA will start once IOMMU is "running"
     */
    immu->immu_dvma_running = B_TRUE;
}

/*
 * immu_dvma_physmem_update()
 *       called when the installed memory on a
 *       system increases, to expand domain DVMA
 *       for domains with UNITY mapping
 */
void
immu_dvma_physmem_update(uint64_t addr, uint64_t size)
{
    uint64_t start;
    uint64_t npages;
    int dcount;
    immu_dcookie_t dcookies[1] = {0};
    domain_t *domain;

    /*
     * Just walk the system-wide list of domains with
     * UNITY mapping. Both the list of *all* domains
     * and *UNITY* domains is protected by the same
     * single lock
     */
    mutex_enter(&immu_domain_lock);
    domain = list_head(&immu_unity_domain_list);
    for (; domain; domain = list_next(&immu_unity_domain_list, domain)) {
        /*
         * Nothing to do if the IOMMU supports passthrough.
         */
        if (IMMU_ECAP_GET_PT(domain->dom_immu->immu_regs_excap))
            continue;

        /* There is no vmem_arena for unity domains. Just map it */
        ddi_err(DER_LOG, domain->dom_dip,
            "iommu: unity-domain: Adding map "
            "[0x%" PRIx64 " - 0x%" PRIx64 "]", addr, addr + size);

        start = IMMU_ROUNDOWN(addr);
        npages = (IMMU_ROUNDUP(size) / IMMU_PAGESIZE) + 1;

        dcookies[0].dck_paddr = start;
        dcookies[0].dck_npages = npages;
        dcount = 1;
        (void) dvma_map(domain, start, npages,
            dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

    }
    mutex_exit(&immu_domain_lock);
}

int
immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags)
{
    dev_info_t *ddip, *odip;
    immu_t *immu;
    domain_t *domain;

    odip = rdip;

    immu = immu_dvma_get_immu(rdip, immu_flags);
    if (immu == NULL) {
        /*
         * possible that there is no IOMMU unit for this device
         * - BIOS bugs are one example.
         */
        ddi_err(DER_WARN, rdip, "No iommu unit found for device");
        return (DDI_DMA_NORESOURCES);
    }

    /*
     * redirect isa devices attached under lpc to lpc dip
     */
    if (strcmp(ddi_node_name(ddi_get_parent(rdip)), "isa") == 0) {
        rdip = get_lpc_devinfo(immu, rdip, immu_flags);
        if (rdip == NULL) {
            ddi_err(DER_PANIC, rdip, "iommu redirect failed");
            /*NOTREACHED*/
        }
    }

    /* Reset immu, as redirection can change IMMU */
    immu = NULL;

    /*
     * for gart, redirect to the real graphic devinfo
     */
    if (strcmp(ddi_node_name(rdip), "agpgart") == 0) {
        rdip = get_gfx_devinfo(rdip);
        if (rdip == NULL) {
            ddi_err(DER_PANIC, rdip, "iommu redirect failed");
            /*NOTREACHED*/
        }
    }

    /*
     * Setup DVMA domain for the device. This does
     * work only the first time we do DVMA for a
     * device.
     */
    ddip = NULL;
    domain = device_domain(rdip, &ddip, immu_flags);
    if (domain == NULL) {
        ddi_err(DER_MODE, rdip, "Intel IOMMU setup failed for device");
        return (DDI_DMA_NORESOURCES);
    }

    immu = domain->dom_immu;

    /*
     * If a domain is found, we must also have a domain dip
     * which is the topmost ancestor dip of rdip that shares
     * the same domain with rdip.
     */
    if (domain->dom_did == 0 || ddip == NULL) {
        ddi_err(DER_MODE, rdip, "domain did 0(%d) or ddip NULL(%p)",
            domain->dom_did, ddip);
        return (DDI_DMA_NORESOURCES);
    }

    if (odip != rdip)
        set_domain(odip, ddip, domain);

    /*
     * Update the root and context entries
     */
    if (immu_context_update(immu, domain, ddip, rdip, immu_flags)
        != DDI_SUCCESS) {
        ddi_err(DER_MODE, rdip, "DVMA map: context update failed");
        return (DDI_DMA_NORESOURCES);
    }

    return (DDI_SUCCESS);
}

int
immu_map_memrange(dev_info_t *rdip, memrng_t *mrng)
{
    immu_dcookie_t dcookies[1] = {0};
    boolean_t pde_set;
    immu_t *immu;
    domain_t *domain;
    immu_inv_wait_t iw;

    dcookies[0].dck_paddr = mrng->mrng_start;
    dcookies[0].dck_npages = mrng->mrng_npages;

    domain = IMMU_DEVI(rdip)->imd_domain;
    immu = domain->dom_immu;

    pde_set = dvma_map(domain, mrng->mrng_start,
        mrng->mrng_npages, dcookies, 1, rdip,
        IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

    immu_init_inv_wait(&iw, "memrange", B_TRUE);

    immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start,
        mrng->mrng_npages, pde_set == B_TRUE ?
        TLB_IVA_WHOLE : TLB_IVA_LEAF, &iw);

    return (DDI_SUCCESS);
}

immu_devi_t *
immu_devi_get(dev_info_t *rdip)
{
    immu_devi_t *immu_devi;
    volatile uintptr_t *vptr = (uintptr_t *)&(DEVI(rdip)->devi_iommu);

    /* Just want atomic reads. No need for lock */
    immu_devi = (immu_devi_t *)(uintptr_t)atomic_or_64_nv((uint64_t *)vptr,
        0);
    return (immu_devi);
}

/*ARGSUSED*/
int
immu_hdl_priv_ctor(void *buf, void *arg, int kmf)
{
    immu_hdl_priv_t *ihp;

    ihp = buf;
    immu_init_inv_wait(&ihp->ihp_inv_wait, "dmahandle", B_FALSE);

    return (0);
}

/*
 * iommulib interface functions
 */
static int
immu_probe(iommulib_handle_t handle, dev_info_t *dip)
{
    immu_devi_t *immu_devi;
    int ret;

    if (!immu_enable)
        return (DDI_FAILURE);

    /*
     * Make sure the device has all the IOMMU structures
     * initialized. If this device goes through an IOMMU
     * unit (e.g. this probe function returns success),
     * this will be called at most N times, with N being
     * the number of IOMMUs in the system.
     *
     * After that, when iommulib_nex_open succeeds,
     * we can always assume that this device has all
     * the structures initialized. IOMMU_USED(dip) will
     * be true. There is no need to find the controlling
     * IOMMU/domain again.
     */
    ret = immu_dvma_device_setup(dip, IMMU_FLAGS_NOSLEEP);
    if (ret != DDI_SUCCESS)
        return (ret);

    immu_devi = IMMU_DEVI(dip);

    /*
     * For unity domains, there is no need to call in to
     * the IOMMU code.
     */
    if (immu_devi->imd_domain->dom_did == IMMU_UNITY_DID)
        return (DDI_FAILURE);

    if (immu_devi->imd_immu->immu_dip == iommulib_iommu_getdip(handle))
        return (DDI_SUCCESS);

    return (DDI_FAILURE);
}

/*ARGSUSED*/
static int
immu_allochdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
    int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep)
{
    int ret;
    immu_hdl_priv_t *ihp;
    immu_t *immu;

    ret = iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,
        arg, dma_handlep);
    if (ret == DDI_SUCCESS) {
        immu = IMMU_DEVI(rdip)->imd_immu;

        ihp = kmem_cache_alloc(immu->immu_hdl_cache,
            waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP);
        if (ihp == NULL) {
            (void) iommulib_iommu_dma_freehdl(dip, rdip,
                *dma_handlep);
            return (DDI_DMA_NORESOURCES);
        }

        if (IMMU_DEVI(rdip)->imd_use_premap)
            dvma_prealloc(rdip, ihp, attr);
        else {
            ihp->ihp_npremapped = 0;
            ihp->ihp_predvma = 0;
        }
        ret = iommulib_iommu_dmahdl_setprivate(dip, rdip, *dma_handlep,
            ihp);
    }
    return (ret);
}

/*ARGSUSED*/
static int
immu_freehdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
{
    immu_hdl_priv_t *ihp;

    ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
    if (ihp != NULL) {
        if (IMMU_DEVI(rdip)->imd_use_premap)
            dvma_prefree(rdip, ihp);
        kmem_cache_free(IMMU_DEVI(rdip)->imd_immu->immu_hdl_cache, ihp);
    }

    return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));
}


/*ARGSUSED*/
static int
immu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    struct ddi_dma_req *dma_req, ddi_dma_cookie_t *cookiep,
    uint_t *ccountp)
{
    int ret;
    immu_hdl_priv_t *ihp;

    ret = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,
        dma_req, cookiep, ccountp);

    if (ret == DDI_DMA_MAPPED) {
        ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
        immu_flush_wait(IMMU_DEVI(rdip)->imd_immu, &ihp->ihp_inv_wait);
    }

    return (ret);
}

/*ARGSUSED*/
static int
immu_unbindhdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
{
    return (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle));
}

/*ARGSUSED*/
static int
immu_sync(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
    size_t len, uint_t cachefl)
{
    return (iommulib_iommu_dma_sync(dip, rdip, dma_handle, off, len,
        cachefl));
}

/*ARGSUSED*/
static int
immu_win(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
    off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
    uint_t *ccountp)
{
    return (iommulib_iommu_dma_win(dip, rdip, dma_handle, win, offp,
        lenp, cookiep, ccountp));
}

/*ARGSUSED*/
static int
immu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao)
{
    immu_hdl_priv_t *ihp;

    ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);

    return (immu_map_dvmaseg(rdip, dma_handle, ihp, dmareq, dmao));
}

/*ARGSUSED*/
static int
immu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)
{
    immu_hdl_priv_t *ihp;

    ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
    if (ihp->ihp_npremapped > 0)
        return (DDI_SUCCESS);
    return (immu_unmap_dvmaseg(rdip, dmao));
}

/*ARGSUSED*/
static int
immu_map(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, struct ddi_dma_req *dmareq,
    ddi_dma_handle_t *dma_handle)
{
    ASSERT(0);
    return (DDI_FAILURE);
}

/*ARGSUSED*/
static int
immu_mctl(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    enum ddi_dma_ctlops request, off_t *offp, size_t *lenp,
    caddr_t *objpp, uint_t cachefl)
{
    ASSERT(0);
    return (DDI_FAILURE);
}