7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * CDDL HEADER START
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * The contents of this file are subject to the terms of the
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Common Development and Distribution License (the "License").
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * You may not use this file except in compliance with the License.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * See the License for the specific language governing permissions
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * and limitations under the License.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * When distributing Covered Code, include this CDDL HEADER in each
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * If applicable, add the following below this CDDL HEADER, with the
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * fields enclosed by brackets "[]" replaced with your own identifying
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * information: Portions Copyright [yyyy] [name of copyright owner]
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * CDDL HEADER END
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
cd21e7c548ae2a3b5e522244bf798f2a6b4ba02dGarrett D'Amore * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
ac48dfe87039078897ed719af26744eca776508cVikram Hegdestatic int amd_iommu_fini(amd_iommu_t *iommu, int type);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void amd_iommu_teardown_interrupts(amd_iommu_t *iommu);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int amd_iommu_unbindhdl(iommulib_handle_t handle,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int amd_iommu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenstatic int amd_iommu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpextern void *device_arena_alloc(size_t size, int vm_flag);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpextern void device_arena_free(void * vaddr, size_t size);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 0U, /* dma_attr_addr_lo */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 0 /* dma_attr_flags */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "AMD IOMMU Vers. 1",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_register";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommulib_ops = kmem_zalloc(sizeof (iommulib_ops_t), KM_SLEEP);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_register(dip, iommulib_ops, &handle)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "failed idx=%d", f, driver, instance, iommu->aiomt_idx);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* we never registered */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_unregister(iommu->aiomt_iommulib_handle)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(iommu->aiomt_iommulib_ops, sizeof (iommulib_ops_t));
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Setup passthru mapping for "special" devices
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (gfxp = gfx_devinfo_list; gfxp; gfxp = gfxp->g_next) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_start";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Disable HT tunnel translation.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * XXX use ACPI
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "amd_iommu: using ACPI for CTRL registers");
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * The Device table entry bit 0 (V) controls whether the device
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * table entry is valid for address translation and Device table
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * entry bit 128 (IV) controls whether interrupt remapping is valid.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * By setting both to zero we are essentially doing pass-thru. Since
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * this table is zeroed on allocation, essentially we will have
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * pass-thru when IOMMU is enabled.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Finally enable the IOMMU ... */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_stop";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Disable translation on HT tunnel traffic
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_setup_tables_and_buffers";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * We will put the Device Table, Command Buffer and
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Event Log in contiguous memory. Allocate the maximum
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * size allowed for such structures
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Device Table: 256b * 64K = 32B * 64K
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Command Buffer: 128b * 32K = 16B * 32K
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Event Log: 128b * 32K = 16B * 32K
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_devtbl_sz = (1<<AMD_IOMMU_DEVTBL_SZ) * AMD_IOMMU_DEVENT_SZ;
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_cmdbuf_sz = (1<<AMD_IOMMU_CMDBUF_SZ) * AMD_IOMMU_CMD_SZ;
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dma_bufsz = iommu->aiomt_devtbl_sz + iommu->aiomt_cmdbuf_sz
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Alloc a DMA handle.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: Cannot alloc DMA handle for "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Alloc memory for tables and buffers
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * XXX remove cast to size_t
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err = ddi_dma_mem_alloc(iommu->aiomt_dmahdl, dma_bufsz,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp DDI_DMA_SLEEP, NULL, (caddr_t *)&iommu->aiomt_dma_bufva,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (size_t *)&iommu->aiomt_dma_mem_realsz, &iommu->aiomt_dma_mem_hdl);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: Cannot alloc memory for DMA "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "to AMD IOMMU tables and buffers", f, driver, instance);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * The VA must be 4K aligned and >= table size
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Now bind the handle
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err = ddi_dma_addr_bind_handle(iommu->aiomt_dmahdl, NULL,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp NULL, &iommu->aiomt_buf_dma_cookie, &iommu->aiomt_buf_dma_ncookie);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: Cannot bind memory for DMA "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "to AMD IOMMU tables and buffers. bufrealsz=%p",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * We assume the DMA engine on the IOMMU is capable of handling the
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * whole table buffer in a single cookie. If not and multiple cookies
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * are needed we fail.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "cookies for DMA to AMD IOMMU tables and buffers. "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * The address in the cookie must be 4K aligned and >= table size
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size >= dma_bufsz);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Setup the device table pointers in the iommu struct as
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * well as the IOMMU device table register
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Set V=1 and TV = 0, so any inadvertant pass-thrus cause
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * page faults. Also set SE bit so we aren't swamped with
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * page fault messages
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (i = 0; i <= AMD_IOMMU_MAX_DEVICEID; i++) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*LINTED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(dentry[1]), AMD_IOMMU_DEVTBL_SE, 1);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp addr = (caddr_t)(uintptr_t)iommu->aiomt_buf_dma_cookie.dmac_cookie_addr;
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_DEVTABBASE, ((uint64_t)(uintptr_t)addr) >> 12);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Setup the command buffer pointers
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*LINTED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf;
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Setup the event log pointers
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp bzero(iommu->aiomt_eventlog, iommu->aiomt_eventlog_sz);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_EVENTBASE, ((uint64_t)(uintptr_t)addr) >> 12);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*LINTED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_event_head = (uint32_t *)iommu->aiomt_eventlog;
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_tail_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* dma sync so device sees this init */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s%d: successfully setup AMD IOMMU "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "tables, idx=%d", f, driver, instance, iommu->aiomt_idx);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void
ac48dfe87039078897ed719af26744eca776508cVikram Hegdeamd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu, int type)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_teardown_tables_and_buffers";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
ac48dfe87039078897ed719af26744eca776508cVikram Hegde AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
ac48dfe87039078897ed719af26744eca776508cVikram Hegde AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
ac48dfe87039078897ed719af26744eca776508cVikram Hegde AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
ac48dfe87039078897ed719af26744eca776508cVikram Hegde AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
ac48dfe87039078897ed719af26744eca776508cVikram Hegde if (iommu->aiomt_dmahdl == NULL || type == AMD_IOMMU_QUIESCE)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Unbind the handle */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (ddi_dma_unbind_handle(iommu->aiomt_dmahdl) != DDI_SUCCESS) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Free the table memory allocated for DMA */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Free the DMA handle */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Must be set prior to enabling command buffer */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Must be set prior to enabling event logging */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* No interrupts for completion wait - too heavy weight. use polling */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*LINTED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_intr_handler";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d. In INTR handler",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISPLAY);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISCARD);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_setup_interrupts";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (ddi_intr_get_supported_types(dip, &type) != DDI_SUCCESS) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: ddi_intr_get_supported_types "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "failed: idx=%d", f, driver, instance, iommu->aiomt_idx);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Interrupt types supported = 0x%x", f, driver, instance,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * for now we only support MSI
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "MSI interrupts not supported. Failing init.",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. MSI supported",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &req);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "ddi_intr_get_nintrs failed err = %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "MSI number of interrupts requested: %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (req == 0) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "interrupts requested. Failing init", f,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "ddi_intr_get_navail failed err = %d", f,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "MSI number of interrupts available: %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (avail == 0) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "interrupts available. Failing init", f,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "interrupts: requested (%d) > available (%d). "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Allocate memory for DDI interrupt handles */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_intr_htable_sz = req * sizeof (ddi_intr_handle_t);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_intr_htable = kmem_zalloc(iommu->aiomt_intr_htable_sz,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Convert req to a power of two as required by ddi_intr_alloc */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "MSI power of 2 number of interrupts: %d,%d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err = ddi_intr_alloc(iommu->aiomt_dip, iommu->aiomt_intr_htable,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp DDI_INTR_TYPE_MSI, 0, req, &actual, DDI_INTR_ALLOC_STRICT);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "ddi_intr_alloc failed: err = %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "number of interrupts actually allocated %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "ddi_intr_alloc failed: actual (%d) < req (%d)",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "ddi_intr_add_handler failed: intr = %d, err = %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (j = 0; j < i; j++) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (ddi_intr_get_cap(iommu->aiomt_intr_htable[0], &intrcap0)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_intr_htable[iommu->aiomt_actual_intrs - 1], &intrcapN)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "ddi_intr_get_cap failed or inconsistent cap among "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "interrupts: intrcap0 (%d) < intrcapN (%d)",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, iommu->aiomt_idx, intrcap0, intrcapN);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Need to call block enable */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Need to call block enable",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Need to call individual enable",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "ddi_intr_enable failed: intr = %d", f,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (j = 0; j < i; j++) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Interrupts successfully %s enabled. # of interrupts = %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ENABLED) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_HANDLER) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ALLOCED) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_init";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: capability registers not locked. "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Unable to use IOMMU unit idx=%d - skipping ...", f, driver,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_init(&iommu->aiomt_mutex, NULL, MUTEX_DRIVER, NULL);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_init(&iommu->aiomt_cmdlock, NULL, MUTEX_DRIVER, NULL);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_init(&iommu->aiomt_eventlock, NULL, MUTEX_DRIVER, NULL);
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam if (acpica_get_bdf(iommu->aiomt_dip, &bus, &device, &func)
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam cmn_err(CE_WARN, "%s: %s%d: Failed to get BDF"
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam "Unable to use IOMMU unit idx=%d - skipping ...",
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam iommu->aiomt_bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) |
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Since everything in the capability block is locked and RO at this
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * point, copy everything into the IOMMU struct
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Get cap header */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp caphdr = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_HDR_OFF);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_httun = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_HTTUN);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_captype = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_capid = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Get address of IOMMU control registers
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_reg_pa = ((uint64_t)hi_addr32 << 32 | low_addr32);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Get cap range reg
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp range = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_RANGE_OFF);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Get cap misc reg
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp misc = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_MISC_OFF);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Set up mapping between control registers PA and VA
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_reg_pages = mmu_btopr(AMD_IOMMU_REG_SIZE + pgoffset);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_reg_size = mmu_ptob(iommu->aiomt_reg_pages);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: Failed to alloc VA for IOMMU "
ac48dfe87039078897ed719af26744eca776508cVikram Hegde (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp hat_devload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Setup the various control register's VA
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_reg_cmdbuf_head_va = iommu->aiomt_reg_va +
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_reg_cmdbuf_tail_va = iommu->aiomt_reg_va +
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_reg_eventlog_head_va = iommu->aiomt_reg_va +
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_reg_eventlog_tail_va = iommu->aiomt_reg_va +
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Setup the DEVICE table, CMD buffer, and LOG buffer in
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * memory and setup DMA access to this memory location
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) {
ac48dfe87039078897ed719af26744eca776508cVikram Hegde (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
ac48dfe87039078897ed719af26744eca776508cVikram Hegde (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) {
ac48dfe87039078897ed719af26744eca776508cVikram Hegde (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * need to setup domain table before gfx bypass
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Set pass-thru for special devices like IOAPIC and HPET
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Also, gfx devices don't use DDI for DMA. No need to register
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * before setting up gfx passthru
ac48dfe87039078897ed719af26744eca776508cVikram Hegde (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam /* Initialize device table entries based on ACPI settings */
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam if (amd_iommu_acpi_init_devtbl(iommu) != DDI_SUCCESS) {
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam cmn_err(CE_WARN, "%s: %s%d: Can't initialize device table",
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
ac48dfe87039078897ed719af26744eca776508cVikram Hegde (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* xxx register/start race */
ac48dfe87039078897ed719af26744eca776508cVikram Hegde (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s%d: IOMMU idx=%d inited.", f, driver,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_fini";
ac48dfe87039078897ed719af26744eca776508cVikram Hegde if (amd_iommu_unregister(iommu) != DDI_SUCCESS) {
ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. "
ac48dfe87039078897ed719af26744eca776508cVikram Hegde amd_iommu_teardown_tables_and_buffers(iommu, type);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit complete. idx = %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_setup";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* First setup PCI access to config space */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * The AMD IOMMU is part of an independent PCI function. There may be
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * more than one IOMMU in that PCI function
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp base_class = pci_config_get8(handle, PCI_CONF_BASCLASS);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp sub_class = pci_config_get8(handle, PCI_CONF_SUBCLASS);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp prog_class = pci_config_get8(handle, PCI_CONF_PROGCLASS);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (base_class != PCI_CLASS_PERIPH || sub_class != PCI_PERIPH_IOMMU ||
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "subclass(0x%x)/programming interface(0x%x)", f, driver,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Find and initialize all IOMMU units in this function
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pci_cap_probe(handle, idx, &id, &cap_base) != DDI_SUCCESS)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* check if cap ID is secure device cap id */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "%s: %s%d: skipping IOMMU: idx(0x%x) "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "cap ID (0x%x) != secure dev capid (0x%x)",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* check if cap type is IOMMU cap type */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cap_type = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cap_id = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "cap type (0x%x) != AMD IOMMU CAP (0x%x)", f,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "failed to init IOMMU", f,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s%d: state=%p: setup %d IOMMU units",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, (void *)statep, statep->aioms_nunits);
ac48dfe87039078897ed719af26744eca776508cVikram Hegdeamd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep, int type)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_teardown";
ac48dfe87039078897ed719af26744eca776508cVikram Hegde if (amd_iommu_fini(iommu, type) != DDI_SUCCESS) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s%d: state=%p: toredown %d units. "
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliamamd_iommu_pci_dip(dev_info_t *rdip, const char *path)
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam const char *f = "amd_iommu_pci_dip";
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam /* Hold rdip so it and its parents don't go away */
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam#endif /* DEBUG */
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam "%s: %s%d dip = %p has no PCI parent, path = %s",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/* Interface with IOMMULIB */
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip)
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam const char *f = "amd_iommu_probe";
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam int instance = ddi_get_instance(iommu->aiomt_dip);
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam const char *idriver = ddi_driver_name(iommu->aiomt_dip);
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam if ((pathp = ddi_pathname(rdip, path)) == NULL)
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam "for rdip=%p, path = %s",
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get BDF "
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam "for rdip=%p, path = %s",
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam * See whether device is described by IVRS as being managed
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam * by this IOMMU
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;
ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam if (hinfop && hinfop->ach_IOMMU_deviceid == iommu->aiomt_bdf)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookie_array, uint_t ccount,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(iommu->aiomt_dip);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "map_current_window";
ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmn_err(CE_NOTE, "%s: %s%d: idx=%d Attempting to get cookies "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "from handle for device %s",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (i = 0; i < ccount; i++) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((error = amd_iommu_map_pa2va(iommu, rdip, attrp, dmareq,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_VMEM_MAP, &start_va, km_flags)) != DDI_SUCCESS) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cookie_array[i].dmac_cookie_addr = (uintptr_t)start_va;
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (i != ccount) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot map cookie# %d "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpunmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(iommu->aiomt_dip);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "unmap_current_window";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (i = 0; i < ncookies; i++) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, NULL, 0, 0)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: AMD IOMMU completion wait failed for: %s",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (i != ncookies) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot unmap cookie# %d "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_bindhdl";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dma_error = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dma_error != DDI_DMA_MAPPED && dma_error != DDI_DMA_PARTIAL_MAP)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s got cookie (%p), #cookies: %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((error = iommulib_iommu_dma_get_cookies(dip, dma_handle,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((error = iommulib_iommu_dma_set_cookies(dip, dma_handle,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s remapped cookie (%p), #cookies: %d",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dma_error == DDI_DMA_MAPPED || dma_error == DDI_DMA_PARTIAL_MAP);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_unbindhdl";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: failed to unbindhdl for dip=%p",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: failed to unmap current window "
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_sync";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp error = iommulib_iommu_dma_sync(dip, rdip, dma_handle, off,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp/*ARGSUSED*/
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_win(iommulib_handle_t handle, dev_info_t *dip,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_win";
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: failed switch windows for dip=%p",
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp error = map_current_window(iommu, rdip, attrp, &sdmareq,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (error == DDI_SUCCESS ? DDI_SUCCESS : DDI_FAILURE);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenamd_iommu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao)
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Lindenamd_iommu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
50200e773f0242e336d032a7b43485e1bcfc9bfeFrank Van Der Linden dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_reg_get64_workaround(uint64_t *regp, uint32_t bits)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_reg_set64_workaround(uint64_t *regp, uint32_t bits, uint64_t value)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (s.u64);
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * if "amd-iommu = no/false" boot property is set,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * ignore AMD iommu
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp DDI_PROP_DONTPASS, "amd-iommu", &propval) == DDI_SUCCESS) {
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp * Copy the list of drivers for which IOMMU is disabled by user.
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_disable_list = kmem_alloc(strlen(propval) + 1,
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu", &disable)
7d87efa8fdfb9453670f2832df666fdae8291a84jmcp DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu-disable-list",