3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * CDDL HEADER START
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * The contents of this file are subject to the terms of the
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Common Development and Distribution License (the "License").
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * You may not use this file except in compliance with the License.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * See the License for the specific language governing permissions
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * and limitations under the License.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * When distributing Covered Code, include this CDDL HEADER in each
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * If applicable, add the following below this CDDL HEADER, with the
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * fields enclosed by brackets "[]" replaced with your own identifying
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * information: Portions Copyright [yyyy] [name of copyright owner]
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * CDDL HEADER END
9e986f0e5fb5e5ac09af90cd3b63f7836d983f9dFrank Van Der Linden * Portions Copyright (c) 2010, Oracle and/or its affiliates.
9e986f0e5fb5e5ac09af90cd3b63f7836d983f9dFrank Van Der Linden * All rights reserved.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Copyright (c) 2009, Intel Corporation.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * All rights reserved.
cd21e7c548ae2a3b5e522244bf798f2a6b4ba02dGarrett D'Amore * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * This file contains Intel IOMMU code that deals with DVMA
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * i.e. DMA remapping.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Macros based on PCI spec
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde#define IMMU_PCI_REV2CLASS(r) ((r) >> 8) /* classcode from revid */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde#define IMMU_PCI_CLASS2BASE(c) ((c) >> 16) /* baseclass from classcode */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde#define IMMU_PCI_CLASS2SUB(c) (((c) >> 8) & 0xff); /* classcode */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ((d).dck_paddr && ((d).dck_paddr + IMMU_PAGESIZE) == (p))
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdestatic domain_t *domain_create(immu_t *immu, dev_info_t *ddip,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdestatic immu_devi_t *create_immu_devi(dev_info_t *rdip, int bus,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdestatic void destroy_immu_devi(immu_devi_t *immu_devi);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic boolean_t dvma_map(domain_t *domain, uint64_t sdvma,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden uint64_t nvpages, immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde/* Extern globals */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * iommulib interface functions.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_probe(iommulib_handle_t unitp, dev_info_t *dip);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_allochdl(iommulib_handle_t handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_freehdl(iommulib_handle_t handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dma_req,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_dma_cookie_t *cookiep, uint_t *ccountp);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_unbindhdl(iommulib_handle_t handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_sync(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, size_t len,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_win(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int immu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* static Globals */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Used to setup DMA objects (memory regions)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * for DMA reads by IOMMU units
d2256d265bf2bcad0d811b81411de3802a4b97c6Frank Van Der Linden 0xffffffffffffffffULL,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde 0xffffffffU,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde 0xffffffffU,
d2256d265bf2bcad0d811b81411de3802a4b97c6Frank Van Der Linden 0xffffffffffffffffULL,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "Intel IOMMU",
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * Fake physical address range used to set up initial prealloc mappings.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * This memory is never actually accessed. It is mapped read-only,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * and is overwritten as soon as the first DMA bind operation is
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * performed. Since 0 is a special case, just start at the 2nd
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * physical page.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic immu_dcookie_t immu_precookie = { MMU_PAGESIZE, IMMU_NPREPTES };
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* globals private to this file */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* structure used to store idx into each level of the page tables */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* 0 is reserved by Vt-d spec. Solaris reserves 1 */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde uintptr_t bdf = (seg << 16 | bus << 8 | devfunc);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdebdf_domain_insert(immu_devi_t *immu_devi, domain_t *domain)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde uintptr_t bdf = (seg << 16 | bus << 8 | devfunc);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (void) mod_hash_insert(bdf_domain_hash, (void *)bdf, (void *)domain);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde for (; immu_devi; immu_devi = list_next(dvap->dva_list,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdeimmu_devi_set_spclist(dev_info_t *dip, immu_t *immu)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Set the immu_devi struct in the immu_devi field of a devinfo node
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdeimmu_devi_set(dev_info_t *dip, immu_flags_t immu_flags)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Assume a new immu_devi struct is needed
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (!DEVI_IS_PCI(dip) || acpica_get_bdf(dip, &bus, &dev, &func) != 0) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * No BDF. Set bus = -1 to indicate this.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * We still need to create a immu_devi struct
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde new_imd = create_immu_devi(dip, bus, dev, func, immu_flags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_WARN, dip, "Failed to create immu_devi "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "structure");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Check if some other thread allocated a immu_devi while we
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * didn't own the lock.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdeget_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_MODE, rdip, "Could not walk ancestors to "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "find lpc_devinfo for ISA device");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (dvarg.dva_error != DDI_SUCCESS || dvarg.dva_ddip == NULL) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_MODE, rdip, "Could not find lpc_devinfo for "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "ISA device");
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * The GFX device may not be on the same iommu unit as "agpgart"
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * so search globally
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde for (; immu; immu = list_next(&immu_list, immu)) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_err(DER_WARN, rdip, "iommu: No GFX device. "
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde "Cannot redirect agpgart");
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_err(DER_LOG, rdip, "iommu: GFX redirect to %s",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Read and write flags need to be reversed.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * DMA_READ means read from device and write
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * to memory. So DMA read means DVMA write.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Some buggy drivers specify neither READ or WRITE
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * For such drivers set both read and write permissions
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if ((dmareq->dmar_flags & (DDI_DMA_READ | DDI_DMA_WRITE)) == 0) {
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde dmafp = (kmflag & KM_NOSLEEP) ? DDI_DMA_DONTWAIT : DDI_DMA_SLEEP;
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde return (-1);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (ddi_dma_alloc_handle(root_devinfo, &immu_dma_attr,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde dmafp, NULL, &pgtable->hwpg_dmahdl) != DDI_SUCCESS) {
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde return (-1);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (ddi_dma_mem_alloc(pgtable->hwpg_dmahdl, IMMU_PAGESIZE,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde return (-1);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Memory allocation failure. Maybe a temporary condition
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * so return error rather than panic, so we can try again
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde return (-1);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde pgtable->hwpg_paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, vaddr));
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde rw_init(&(pgtable->swpg_rwlock), NULL, RW_DEFAULT, NULL);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /* destroy will panic if lock is held. */
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde kmem_free(pgtable->swpg_next_array, IMMU_PAGESIZE);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * pgtable_alloc()
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * alloc a IOMMU pgtable structure.
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * This same struct is used for root and context tables as well.
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * This routine allocs the f/ollowing:
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * - a pgtable_t struct
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * - a HW page which holds PTEs/entries which is accesssed by HW
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * so we set up DMA for this page
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * - a SW page which is only for our bookeeping
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * (for example to hold pointers to the next level pgtable).
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * So a simple kmem_alloc suffices
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegdepgtable_alloc(immu_t *immu, immu_flags_t immu_flags)
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden pgtable = kmem_cache_alloc(immu->immu_pgtable_cache, kmflags);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde bzero(pgtable->swpg_next_array, IMMU_PAGESIZE);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden kmem_cache_free(immu->immu_pgtable_cache, pgtable);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Function to identify a display device from the PCI class code
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde int i, nclasses = sizeof (disp_classes) / sizeof (uint_t);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde for (i = 0; i < nclasses; i++) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Function that determines if device is PCIEX and/or PCIEX bridge
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde uchar_t bus, uchar_t dev, uchar_t func, boolean_t *is_pcib)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde status = pci_getw_func(bus, dev, func, PCI_CONF_STAT);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde capsp = pci_getb_func(bus, dev, func, PCI_CONF_CAP_PTR);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde while (cap_count-- && capsp >= PCI_CAP_PTR_OFF) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde status = pci_getw_func(bus, dev, func, capsp + 2);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * See section 7.8.2 of PCI-Express Base Spec v1.0a
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * PCIE_PCIECAP_DEV_TYPE_PCIE2PCI implies that the
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * device is a PCIE2PCI bridge
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde PCIE_PCIECAP_DEV_TYPE_PCIE2PCI) ? B_TRUE : B_FALSE;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (IMMU_PCI_CLASS2BASE(classcode) == PCI_CLASS_NET)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * immu_dvma_get_immu()
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * get the immu unit structure for a dev_info node
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdeimmu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * check if immu unit was already found earlier.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * If yes, then it will be stashed in immu_devi struct.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (immu_devi_set(dip, immu_flags) != DDI_SUCCESS) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * May fail because of low memory. Return error rather
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * than panic as we want driver to rey again later
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "No immu_devi structure");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "Cannot find immu_t for device");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Check if some other thread found immu
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * while lock was not held
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* immu_devi should be present as we found it earlier */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "immu_dvma_get_immu: No immu_devi structure");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* nobody else set it, so we should do it */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * if some other thread got immu before
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * us, it should get the same results
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "immu units found for device. Expected (%p), "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* ############################# IMMU_DEVI code ############################ */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Allocate a immu_devi structure and initialize it
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdecreate_immu_devi(dev_info_t *rdip, int bus, int dev, int func,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* bus == -1 indicate non-PCI device (no BDF) */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi = kmem_zalloc(sizeof (immu_devi_t), kmflags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_WARN, rdip, "Failed to allocate memory for "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "Intel IOMMU immu_devi structure");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi->imd_seg = 0; /* Currently seg can only be 0 */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi->imd_devfunc = IMMU_PCI_DEVFUNC(dev, func);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde revclass = pci_getl_func(bus, dev, func, PCI_CONF_REVID);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (baseclass == PCI_CLASS_BRIDGE && subclass == PCI_BRIDGE_PCI) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi->imd_sec = pci_getb_func(bus, dev, func,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi->imd_sub = pci_getb_func(bus, dev, func,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde pciex = device_is_pciex(bus, dev, func, &is_pcib);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCIE;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* check for certain special devices */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi->imd_display = device_is_display(classcode);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi->imd_lpc = ((baseclass == PCI_CLASS_BRIDGE) &&
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde (subclass == PCI_BRIDGE_ISA)) ? B_TRUE : B_FALSE;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_devi->imd_use_premap = device_use_premap(classcode);
9e986f0e5fb5e5ac09af90cd3b63f7836d983f9dFrank Van Der Linden immu_devi->imd_dvma_flags = immu_global_dvma_flags;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdeimmu_devi_domain(dev_info_t *rdip, dev_info_t **ddipp)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* ############################# END IMMU_DEVI code ######################## */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* ############################# DOMAIN code ############################### */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * This routine always succeeds
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_WARN, rdip, "device domain-id alloc error"
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde " domain-device: %s%d. immu unit is %s. Using "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "unity domain with domain-id (%d)",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu->immu_name, immu->immu_unity_domain->dom_did);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * The field dvp->dva_rdip is a work-in-progress
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * and gets updated as we walk up the ancestor
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * tree. The final ddip is set only when we reach
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * the top of the tree. So the dvp->dva_ddip field cannot
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * be relied on until we reach the top of the field.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* immu_devi may not be set. */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (immu_devi_set(pdip, dvp->dva_flags) != DDI_SUCCESS) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu = immu_dvma_get_immu(pdip, dvp->dva_flags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * If we encounter a PCIE_PCIE bridge *ANCESTOR* we need to
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * terminate the walk (since the device under the PCIE bridge
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * is a PCIE device and has an independent entry in the
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCIE) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * In order to be a domain-dim, it must be a PCI device i.e.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * must have valid BDF. This also eliminates the root complex.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (immu_devi->imd_pcib_type != IMMU_PCIB_BAD &&
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* continue walking to find ddip */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* if domain is set, it must be the same */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Domain may already be set, continue walking so that ddip gets set */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* domain is not set in either immu_devi or dvp */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* ok, the BDF hash had a domain for this BDF. */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Grab lock again to check if something else set immu_devi fields */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * walk upwards until the topmost PCI bridge is found
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * UNITY arenas are a mirror of the physical memory
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * installed on the system.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Dont skip page0. Some broken HW/FW access it.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (void) dvma_map(domain, 0, 1, dcookies, dcount, NULL,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* since we already mapped page1 above */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (void) dvma_map(domain, start, npages, dcookies,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_err(DER_LOG, domain->dom_dip, "iommu: mapping PHYS span [0x%" PRIx64
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde " - 0x%" PRIx64 "]", start, start + mp->ml_size);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]",
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden mp->ml_address, mp->ml_address + mp->ml_size);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]",
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden mp->ml_address, mp->ml_address + mp->ml_size);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * create_xlate_arena()
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Create the dvma arena for a domain with translation
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdecreate_xlate_arena(immu_t *immu, domain_t *domain,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Note, don't do sizeof (arena_name) - it is just a pointer */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "%s-domain-%d-xlate-DVMA-arena", immu->immu_name,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde vmem_flags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Restrict mgaddr (max guest addr) to MGAW */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * To ensure we avoid ioapic and PCI MMIO ranges we just
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * use the physical memory address range of the system as the
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "iommu: %s: Creating dvma vmem arena [0x%" PRIx64
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde " - 0x%" PRIx64 "]", arena_name, start, start + size);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * We always allocate in quanta of IMMU_PAGESIZE
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde domain->dom_dvma_arena = vmem_create(arena_name,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde 0, /* qcache_max */
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde "Failed to allocate DVMA arena(%s) "
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde "for domain ID (%d)", arena_name, domain->dom_did);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /*NOTREACHED*/
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "iommu: %s: Adding dvma vmem span [0x%" PRIx64
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "Failed to allocate DVMA arena(%s) "
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde "for domain ID (%d)",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* ################################### DOMAIN CODE ######################### */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Set the domain and domain-dip for a dip
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * device_domain()
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Get domain for a device. The domain may be global in which case it
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * is shared between all IOMMU units. Due to potential AGAW differences
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * between IOMMU units, such global domains *have to be* UNITY mapping
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * domains. Alternatively, the domain may be local to a IOMMU unit.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Local domains may be shared or immu_devi, although the
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * scope of sharing
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * is restricted to devices controlled by the IOMMU unit to
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * which the domain
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * belongs. If shared, they (currently) have to be UNITY domains. If
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * immu_devi a domain may be either UNITY or translation (XLATE) domain.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdedevice_domain(dev_info_t *rdip, dev_info_t **ddipp, immu_flags_t immu_flags)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde dev_info_t *ddip; /* topmost dip in domain i.e. domain owner */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Check if the domain is already set. This is usually true
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * if this is not the first DVMA transaction.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * possible that there is no IOMMU unit for this device
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * - BIOS bugs are one example.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_err(DER_WARN, rdip, "No iommu unit found for device");
9e986f0e5fb5e5ac09af90cd3b63f7836d983f9dFrank Van Der Linden immu_flags |= immu_devi_get(rdip)->imd_dvma_flags;
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde if (immu_walk_ancestor(rdip, NULL, get_branch_domain,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * maybe low memory. return error,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * so driver tries again later
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* should have walked at least 1 dip (i.e. edip) */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * We may find the domain during our ancestor walk on any one of our
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * ancestor dips, If the domain is found then the domain-dip
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * (i.e. ddip) will also be found in the same immu_devi struct.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * The domain-dip is the highest ancestor dip which shares the
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * same domain with edip.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * The domain may or may not be found, but the domain dip must
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde ddi_err(DER_MODE, rdip, "Cannot find domain dip for device.");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Did we find a domain ?
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* nope, so allocate */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde domain = domain_create(immu, ddip, rdip, immu_flags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*FALLTHROUGH*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * We know *domain *is* the right domain, so panic if
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * another domain is set for either the request-dip or
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * effective dip.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* domain created during boot and always use sleep flag */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde domain = kmem_zalloc(sizeof (domain_t), KM_SLEEP);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Setup the domain's initial page table
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * should never fail.
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden pgtable_zero(domain->dom_pgtable_root);
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden * Only map all physical memory in to the unity domain
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden * if passthrough is not supported. If it is supported,
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden * passthrough is set in the context entry instead.
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden if (!IMMU_ECAP_GET_PT(immu->immu_regs_excap))
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * put it on the system-wide UNITY domain list
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde list_insert_tail(&immu_unity_domain_list, domain);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * ddip is the domain-dip - the topmost dip in a domain
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * rdip is the requesting-dip - the device which is
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * requesting DVMA setup
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * if domain is a non-shared domain rdip == ddip
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdedomain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * First allocate a domainid.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * This routine will never fail, since if we run out
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * of domains the unity domain will be allocated.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* domain overflow */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde domain = kmem_zalloc(sizeof (domain_t), kmflags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_PANIC, rdip, "Failed to alloc DVMA domain "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "structure for device. IOMMU unit: %s", immu->immu_name);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde (void) snprintf(mod_hash_name, sizeof (mod_hash_name),
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "immu%s-domain%d-pava-hash", immu->immu_name, did);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Create xlate DVMA arena for this domain.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde create_xlate_arena(immu, domain, rdip, immu_flags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Setup the domain's initial page table
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_PANIC, rdip, "Failed to alloc root "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "pgtable for domain (%d). IOMMU unit: %s",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden pgtable_zero(domain->dom_pgtable_root);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Since this is a immu unit-specific domain, put it on
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * the per-immu domain list.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde list_insert_head(&immu->immu_domain_list, domain);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Also put it on the system-wide xlate domain list
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde list_insert_head(&immu_xlate_domain_list, domain);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Map page0. Some broken HW/FW access it.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (void) dvma_map(domain, 0, 1, dcookies, dcount, NULL,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Create domainid arena.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Domainid 0 is reserved by Vt-d spec and cannot be used by
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * system software.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Domainid 1 is reserved by solaris and used for *all* of the following:
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * as the "uninitialized" domain - For devices not yet controlled
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * by Solaris
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * as the "unity" domain - For devices that will always belong
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * to the unity domain
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * as the "overflow" domain - Used for any new device after we
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * run out of domains
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * All of the above domains map into a single domain with
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * domainid 1 and UNITY DVMA mapping
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Each IMMU unity has its own unity/uninit/overflow domain
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_err(DER_VERB, immu->immu_dip, "creating domainid arena %s",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde (void *)(uintptr_t)(IMMU_UNITY_DID + 1), /* start addr */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde 0, /* qcache_max */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Even with SLEEP flag, vmem_create() can fail */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_PANIC, NULL, "%s: Failed to create Intel "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "IOMMU domainid allocator: %s", immu->immu_name,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* ######################### CONTEXT CODE ################################# */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdecontext_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ctxp = (hw_rce_t *)(root_table->swpg_next_array);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr) + bus;
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /* Check the most common case first with reader lock */
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc;
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde if (CONT_GET_AVAIL(hw_cent) == IMMU_CONT_INITED) {
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t));
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* need to disable context entry before reprogramming it */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* flush caches */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_flush_context_fsi(immu, 0, sid, domain->dom_did,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr);
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden if (domain->dom_did == IMMU_UNITY_DID &&
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden IMMU_ECAP_GET_PT(immu->immu_regs_excap))
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden CONT_SET_TTYPE(hw_cent, TTYPE_PASSTHRU);
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Allocate a zeroed root table (4K 256b entries) */
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Setup context tables for all possible root table entries.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Start out with unity domains for all entries.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ctxp = (hw_rce_t *)(root_table->swpg_next_array);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde for (bus = 0; bus < IMMU_ROOT_NUM; bus++, ctxp++, hw_rent++) {
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr);
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden if (IMMU_ECAP_GET_PT(immu->immu_regs_excap))
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden CONT_SET_TTYPE(hw_cent, TTYPE_PASSTHRU);
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Called during rootnex attach, so no locks needed
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_init_inv_wait(&immu->immu_ctx_inv_wait, "ctxglobal", B_TRUE);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_flush_context_gbl(immu, &immu->immu_ctx_inv_wait);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_flush_iotlb_gbl(immu, &immu->immu_ctx_inv_wait);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Find top pcib
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (immu_devi->imd_pcib_type == IMMU_PCIB_PCI_PCI) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdeimmu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddip == root_devinfo || rdip == root_devinfo) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_MODE, rdip, "immu_contexts_update: domain-dip or "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "request-dip are NULL or are root devinfo");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * We need to set the context fields
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * based on what type of device rdip and ddip are.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * To do that we need the immu_devi field.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Set the immu_devi field (if not already set)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (immu_devi_set(ddip, immu_flags) == DDI_FAILURE) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "immu_context_update: failed to set immu_devi for ddip");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (immu_devi_set(rdip, immu_flags) == DDI_FAILURE) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "immu_context_update: failed to set immu_devi for rdip");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* rdip is a PCIE device. set context for it only */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde context_set(immu, domain, immu->immu_ctx_root, r_bus,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_WARN, rdip, "Driver bug: Devices 0x%lx and "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* rdip is a PCIE device. set context for it only */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde context_set(immu, domain, immu->immu_ctx_root, r_bus,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde } else if (d_pcib_type == IMMU_PCIB_PCIE_PCI) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * ddip is a PCIE_PCI bridge. Set context for ddip's
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * secondary bus. If rdip is on ddip's secondary
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * bus, set context for rdip. Else, set context
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * for rdip's PCI bridge on ddip's secondary bus.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (immu_walk_ancestor(rdip, ddip, find_top_pcib,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde " bridge for PCI device");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde context_set(immu, domain, immu->immu_ctx_root, d_bus,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde } else if (d_pcib_type == IMMU_PCIB_ENDPOINT) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * ddip is a PCIE device which has a non-PCI device under it
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * i.e. it is a PCI-nonPCI bridge. Example: pciicde-ata
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde context_set(immu, domain, immu->immu_ctx_root, d_bus,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_PANIC, rdip, "unknown device type. Cannot "
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "set iommu context.");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* XXX do we need a membar_producer() here */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* ##################### END CONTEXT CODE ################################## */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* ##################### MAPPING CODE ################################## */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram HegdePDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* The PDTE must be set i.e. present bit is set */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Just assert to check most significant system software field
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * (PDTE_SW4) as it is same as present bit and we
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * checked that above
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * TM field should be clear if not reserved.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * non-leaf is always reserved
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde if (next == NULL && immu->immu_TM_reserved == B_FALSE) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * The SW3 field is not used and must be clear
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * PFN (for PTE) or next level pgtable-paddr (for PDE) must be set
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "PTE paddr mismatch: %lx != %lx",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "PDE paddr mismatch: %lx != %lx",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * SNP field should be clear if not reserved.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * non-leaf is always reserved
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde if (next == NULL && immu->immu_SNP_reserved == B_FALSE) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* second field available for system software should be clear */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Super pages field should be clear */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * least significant field available for
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * system software should be clear
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if ((immu_flags & IMMU_FLAGS_READ) && !PDTE_READ(pdte)) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if ((immu_flags & IMMU_FLAGS_WRITE) && !PDTE_WRITE(pdte)) {
e03dceed3deb85ad561202c77277e701f763fa13Vikram HegdePTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde uint64_t *dvma_ptr, uint64_t *npages_ptr, dev_info_t *rdip)
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * since a caller gets a unique dvma for a physical address,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * no other concurrent thread will be writing to the same
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * PTE even if it has the same paddr. So no locks needed.
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde for (; npages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenxlate_setup(uint64_t dvma, xlate_t *xlate, int nlevels)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Skip the first 12 bits which is the offset into
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * 4K PFN (phys page frame based on IMMU_PAGESIZE)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* skip to level 1 i.e. leaf PTE */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde for (level = 1, xlate++; level <= nlevels; level++, xlate++) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde xlate->xlt_idx = (offbits & IMMU_PGTABLE_LEVEL_MASK);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Read the pgtables
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der LindenPDE_lookup(domain_t *domain, xlate_t *xlate, int nlevels)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* start with highest level pgtable i.e. root */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Lock the pgtable in read mode */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * since we are unmapping, the pgtable should
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * already point to a leafier pgtable.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_fault_walk(void *arg, void *base, size_t len)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (dvma >= start && dvma < (start + len)) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "faulting DVMA address is in vmem arena "
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_print_fault_info(uint_t sid, uint64_t dvma)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (void *)(uintptr_t)sid, (void *)&domain) != 0) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "no domain for faulting SID %08x", sid);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden vmem_walk(domain->dom_dvma_arena, VMEM_ALLOC, immu_fault_walk,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "faulting DVMA address is not in vmem arena");
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (!PDE_lookup(domain, xlate, nlevels)) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "pte not found in domid %d for faulting addr %" PRIx64,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (xlatep->xlt_pgtable->hwpg_vaddr) + xlatep->xlt_idx);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "domid %d pte: %" PRIx64 "(paddr %" PRIx64 ")", domain->dom_did,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (unsigned long long)pte, (unsigned long long)PDTE_PADDR(pte));
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram HegdePTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_MODE, rdip, "PTE paddr %lx != paddr %lx",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* clear TM field if not reserved */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Clear 3rd field for system software - not used */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Set paddr */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* clear SNP field if not reserved. */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Clear SW2 field available for software */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* SP is don't care for PTEs. Clear it for cleanliness */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Clear SW1 field available for software */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Now that we are done writing the PTE
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * set the "present" flag. Note this present
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * flag is a bit in the PDE/PTE that the
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * spec says is available for system software.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * This is an implementation detail of Solaris
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * bare-metal Intel IOMMU.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * The present field in a PDE/PTE is not defined
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * by the Vt-d spec
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden#endif /* DEBUG */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden#endif /* BUGGY_DRIVERS */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram HegdePTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden uint64_t *dvma_ptr, uint64_t *nvpages_ptr, immu_dcookie_t *dcookies,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde int dcount, dev_info_t *rdip, immu_flags_t immu_flags)
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * since a caller gets a unique dvma for a physical address,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * no other concurrent thread will be writing to the same
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * PTE even if it has the same paddr. So no locks needed.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde (dcookies[j].dck_npages - nppages) * IMMU_PAGESIZE;
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde for (; nvpages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde PTE_set_one(immu, hwp, paddr, rdip, immu_flags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram HegdePDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* if PDE is already set, make sure it is correct */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Dont touch SW4, it is the present bit */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* don't touch TM field it is reserved for PDEs */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* 3rd field available for system software is not used */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Set next level pgtable-paddr for PDE */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* don't touch SNP field it is reserved for PDEs */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Clear second field available for system software */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* No super pages for PDEs */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Clear SW1 for software */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Now that we are done writing the PDE
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * set the "present" flag. Note this present
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * flag is a bit in the PDE/PTE that the
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * spec says is available for system software.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * This is an implementation detail of Solaris
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * base-metal Intel IOMMU.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * The present field in a PDE/PTE is not defined
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * by the Vt-d spec
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Used to set PDEs
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram HegdePDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* start with highest level pgtable i.e. root */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde for (level = nlevels; level > 1; level--, xlate--) {
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /* Lock the pgtable in READ mode first */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde hwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * check if leafier level already has a pgtable
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * if yes, verify
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden IMMU_DPROBE2(immu__pdp__alloc, dev_info_t *,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "pgtable alloc err");
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /* Change to a write lock */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde PDE_set_one(immu, hwp, next, rdip, immu_flags);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * If buggy driver we already set permission
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * READ+WRITE so nothing to do for that case
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * XXX Check that read writer perms change before
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * actually setting perms. Also need to hold lock
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * dvma_map()
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * map a contiguous range of DVMA pages
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * immu: IOMMU unit for which we are generating DVMA cookies
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * domain: domain
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * sdvma: Starting dvma
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * spaddr: Starting paddr
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * npages: Number of pages
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * rdip: requesting device
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * immu_flags: flags
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindendvma_map(domain_t *domain, uint64_t sdvma, uint64_t snvpages,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde while (n > 0) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Lookup or allocate PGDIRs and PGTABLEs if necessary */
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* set all matching ptes that fit into this leaf pgtable */
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * dvma_unmap()
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * unmap a range of DVMAs
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * immu: IOMMU unit state
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * domain: domain for requesting device
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * ddip: domain-dip
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * dvma: starting DVMA
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * npages: Number of IMMU pages to be unmapped
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * rdip: requesting device
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindendvma_unmap(domain_t *domain, uint64_t sdvma, uint64_t snpages,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde while (n > 0) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* setup the xlate array */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* just lookup existing pgtables. Should never fail */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (!PDE_lookup(domain, xlate, nlevels))
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (unsigned long long)dvma);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /* clear all matching ptes that fit into this leaf pgtable */
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /* No need to flush IOTLB after unmap */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindendvma_alloc(domain_t *domain, ddi_dma_attr_t *dma_attr, uint_t npages, int kmf)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* parameters */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* handle the rollover cases */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * allocate from vmem arena.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde xsize, align, 0, 0, (void *)(uintptr_t)minaddr,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindendvma_prealloc(dev_info_t *rdip, immu_hdl_priv_t *ihp, ddi_dma_attr_t *dma_attr)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0}, *xlp;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden /* parameters */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (dma_attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden xsize, align, 0, dma_attr->dma_attr_seg + 1,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (void *)(uintptr_t)minaddr, (void *)(uintptr_t)maxaddr, VM_NOSLEEP);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * Set up a mapping at address 0, just so that all PDPs get allocated
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * now. Although this initial mapping should never be used,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * explicitly set it to read-only, just to be safe.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden while (n > 0) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (void) PDE_set_all(immu, domain, xlate, nlevels, rdip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden shwp = (hw_pdte_t *)(xlp->xlt_pgtable->hwpg_vaddr)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindendvma_prefree(dev_info_t *rdip, immu_hdl_priv_t *ihp)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dvma_unmap(domain, ihp->ihp_predvma, IMMU_NPREPTES, rdip);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindendvma_free(domain_t *domain, uint64_t dvma, uint64_t npages)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden uint64_t size = npages * IMMU_PAGESIZE;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (domain->dom_maptype != IMMU_MAPTYPE_XLATE)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden vmem_free(domain->dom_dvma_arena, (void *)(uintptr_t)dvma, size);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_map_dvmaseg(dev_info_t *rdip, ddi_dma_handle_t handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_hdl_priv_t *ihp, struct ddi_dma_req *dmareq,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden uint64_t offset, paddr, dvma, sdvma, rwmask;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_flags = dma_to_immu_flags(dmareq);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden attrp = &((ddi_dma_impl_t *)handle)->dmai_attr;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde pparray = dmar_object->dmao_obj.virt_obj.v_priv;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden IMMU_DPROBE3(immu__map__dvma, dev_info_t *, rdip, ddi_dma_atyp_t,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* retrieve paddr, psize, offset from dmareq */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde offset = dmar_object->dmao_obj.pp_obj.pp_offset &
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden vas = dmar_object->dmao_obj.virt_obj.v_as;
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde paddr = pfn_to_pa(pparray[pcnt]->p_pagenum) + offset;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden paddr = pfn_to_pa(hat_getpfnum(vas->a_hat,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (npgalloc <= IMMU_NPREPTES && ihp->ihp_predvma != 0) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden rwmask = PDTE_MASK_R | PDTE_MASK_W | immu->immu_ptemask;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden PDTE_PADDR(paddr & ~MMU_PAGEOFFSET) | rwmask;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden sdvma = dvma_alloc(domain, attrp, npgalloc,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dcookies[0].dck_paddr = (paddr & ~MMU_PAGEOFFSET);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden IMMU_DPROBE3(immu__dvma__alloc, dev_info_t *, rdip, uint64_t, npgalloc,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde while (size > 0) {
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* get the size for this page (i.e. partial or full page) */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* get the paddr from the page_t */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* index into the array of page_t's to get the paddr */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* call into the VM to get the paddr */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden paddr = pfn_to_pa(hat_getpfnum(vas->a_hat, vaddr));
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden } else if (IMMU_CONTIG_PADDR(dcookies[dmax], paddr)) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden /* No, we need a new dcookie */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * Ran out of dcookies. Map them now.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * Finish up, mapping all, or all of the remaining,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * physical memory ranges.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (ihp->ihp_npremapped == 0 && npages > 0) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden IMMU_DPROBE4(immu__dvmamap__late, dev_info_t *, rdip, \
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden uint64_t, dvma, uint_t, npages, uint_t, dmax+1);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (dvma_map(domain, dvma, npages, dcookies,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden /* Invalidate the IOTLB */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden pde_set > 0 ? TLB_IVA_WHOLE : TLB_IVA_LEAF,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ihp->ihp_dvseg[0].dvs_len = dmar_object->dmao_size;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dma_out->dmao_size = dmar_object->dmao_size;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dma_out->dmao_obj.dvma_obj.dv_off = offset & IMMU_PAGEOFFSET;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dma_out->dmao_obj.dvma_obj.dv_nseg = 1;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dma_out->dmao_obj.dvma_obj.dv_seg = &ihp->ihp_dvseg[0];
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_unmap_dvmaseg(dev_info_t *rdip, ddi_dma_obj_t *dmao)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden npages = IMMU_BTOPR(dvs[0].dvs_len + dmao->dmao_obj.dvma_obj.dv_off);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden /* Unmap only in DEBUG mode */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dvma_unmap(domain, dvma, npages, rdip);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden IMMU_DPROBE3(immu__dvma__free, dev_info_t *, rdip, uint_t, npages,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * In the DEBUG case, the unmap was actually done,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * but an IOTLB flush was not done. So, an explicit
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * write back flush is needed.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde/* ############################# Functions exported ######################## */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * setup the DVMA subsystem
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * this code runs only for the first IOMMU unit
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde mutex_init(&immu_domain_lock, NULL, MUTEX_DEFAULT, NULL);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Create lists */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde list_create(&immu_unity_domain_list, sizeof (domain_t),
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde list_create(&immu_xlate_domain_list, sizeof (domain_t),
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* Setup BDF domain hash */
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde bdf_domain_hash = mod_hash_create_extended("BDF-DOMAIN_HASH",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde nchains, mod_hash_null_keydtor, mod_hash_null_valdtor,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde mod_hash_byid, (void *)(uintptr_t)kval, mod_hash_idkey_cmp,
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Startup up one DVMA unit
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * DVMA will start once IOMMU is "running"
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * immu_dvma_physmem_update()
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * called when the installed memory on a
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * system increases, to expand domain DVMA
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * for domains with UNITY mapping
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegdeimmu_dvma_physmem_update(uint64_t addr, uint64_t size)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Just walk the system-wide list of domains with
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * UNITY mapping. Both the list of *all* domains
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * and *UNITY* domains is protected by the same
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * single lock
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde for (; domain; domain = list_next(&immu_unity_domain_list, domain)) {
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden * Nothing to do if the IOMMU supports passthrough.
be56ae3626e635d164d03331df68da0fea8a8862Frank Van Der Linden if (IMMU_ECAP_GET_PT(domain->dom_immu->immu_regs_excap))
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde /* There is no vmem_arena for unity domains. Just map it */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden "iommu: unity-domain: Adding map "
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde "[0x%" PRIx64 " - 0x%" PRIx64 "]", addr, addr + size);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde npages = (IMMU_ROUNDUP(size) / IMMU_PAGESIZE) + 1;
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags)
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * possible that there is no IOMMU unit for this device
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * - BIOS bugs are one example.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_err(DER_WARN, rdip, "No iommu unit found for device");
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * redirect isa devices attached under lpc to lpc dip
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde if (strcmp(ddi_node_name(ddi_get_parent(rdip)), "isa") == 0) {
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde rdip = get_lpc_devinfo(immu, rdip, immu_flags);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_err(DER_PANIC, rdip, "iommu redirect failed");
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /*NOTREACHED*/
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /* Reset immu, as redirection can change IMMU */
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde * for gart, redirect to the real graphic devinfo
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde if (strcmp(ddi_node_name(rdip), "agpgart") == 0) {
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ddi_err(DER_PANIC, rdip, "iommu redirect failed");
e03dceed3deb85ad561202c77277e701f763fa13Vikram Hegde /*NOTREACHED*/
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Setup DVMA domain for the device. This does
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * work only the first time we do DVMA for a
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde domain = device_domain(rdip, &ddip, immu_flags);
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_MODE, rdip, "Intel IOMMU setup failed for device");
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * If a domain is found, we must also have a domain dip
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * which is the topmost ancestor dip of rdip that shares
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * the same domain with rdip.
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_MODE, rdip, "domain did 0(%d) or ddip NULL(%p)",
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde * Update the root and context entries
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde if (immu_context_update(immu, domain, ddip, rdip, immu_flags)
3a634bfc9a31448c742688c603d3e76b83b041a0Vikram Hegde ddi_err(DER_MODE, rdip, "DVMA map: context update failed");
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_map_memrange(dev_info_t *rdip, memrng_t *mrng)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dcookies[0].dck_paddr = mrng->mrng_start;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dcookies[0].dck_npages = mrng->mrng_npages;
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden pde_set = dvma_map(domain, mrng->mrng_start,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_init_inv_wait(&iw, "memrange", B_TRUE);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden volatile uintptr_t *vptr = (uintptr_t *)&(DEVI(rdip)->devi_iommu);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden /* Just want atomic reads. No need for lock */
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_devi = (immu_devi_t *)(uintptr_t)atomic_or_64_nv((uint64_t *)vptr,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_hdl_priv_ctor(void *buf, void *arg, int kmf)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_init_inv_wait(&ihp->ihp_inv_wait, "dmahandle", B_FALSE);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * iommulib interface functions
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_probe(iommulib_handle_t handle, dev_info_t *dip)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * Make sure the device has all the IOMMU structures
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * initialized. If this device goes through an IOMMU
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * unit (e.g. this probe function returns success),
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * this will be called at most N times, with N being
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * the number of IOMMUs in the system.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * After that, when iommulib_nex_open succeeds,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * we can always assume that this device has all
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * the structures initialized. IOMMU_USED(dip) will
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * be true. There is no need to find the controlling
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ret = immu_dvma_device_setup(dip, IMMU_FLAGS_NOSLEEP);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * For unity domains, there is no need to call in to
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden * the IOMMU code.
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (immu_devi->imd_domain->dom_did == IMMU_UNITY_DID)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden if (immu_devi->imd_immu->immu_dip == iommulib_iommu_getdip(handle))
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ret = iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ihp = kmem_cache_alloc(immu->immu_hdl_cache,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden (void) iommulib_iommu_dma_freehdl(dip, rdip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ret = iommulib_iommu_dmahdl_setprivate(dip, rdip, *dma_handlep,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden kmem_cache_free(IMMU_DEVI(rdip)->imd_immu->immu_hdl_cache, ihp);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden struct ddi_dma_req *dma_req, ddi_dma_cookie_t *cookiep,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ret = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden immu_flush_wait(IMMU_DEVI(rdip)->imd_immu, &ihp->ihp_inv_wait);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_unbindhdl(iommulib_handle_t handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden return (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle));
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_sync(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden return (iommulib_iommu_dma_sync(dip, rdip, dma_handle, off, len,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_win(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden return (iommulib_iommu_dma_win(dip, rdip, dma_handle, win, offp,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden return (immu_map_dvmaseg(rdip, dma_handle, ihp, dmareq, dmao));
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenimmu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);