7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include <sys/sunddi.h>

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include <sys/sunndi.h>

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include <sys/acpi/acpi.h>

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include <sys/acpica.h>

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include <sys/amd_iommu.h>

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include <sys/bootconf.h>

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include <sys/sysmacros.h>

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include <sys/ddidmareq.h>

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include "amd_iommu_impl.h"

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include "amd_iommu_acpi.h"

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp#include "amd_iommu_page_tables.h"

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpddi_dma_attr_t amd_iommu_pgtable_dma_attr = {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp DMA_ATTR_V0,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 0U, /* dma_attr_addr_lo */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 0xffffffffffffffffULL, /* dma_attr_addr_hi */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 0xffffffffU, /* dma_attr_count_max */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (uint64_t)4096, /* dma_attr_align */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 1, /* dma_attr_burstsizes */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 64, /* dma_attr_minxfer */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 0xffffffffU, /* dma_attr_maxxfer */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 0xffffffffU, /* dma_attr_seg */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 1, /* dma_attr_sgllen, variable */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 64, /* dma_attr_granular */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp 0 /* dma_attr_flags */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp};

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic amd_iommu_domain_t **amd_iommu_domain_table;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic struct {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int f_count;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *f_list;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp} amd_iommu_pgtable_freelist;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpint amd_iommu_no_pgtable_freelist;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_acpi_ivhd_t *hinfop;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp hinfop = amd_iommu_lookup_ivhd(bdf);

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam if (hinfop == NULL) {

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam if (bdf == -1) {

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam *src_bdfp = bdf;

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam } else {

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam cmn_err(CE_WARN, "No IVHD entry for 0x%x", bdf);

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam return (DDI_FAILURE);

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam }

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam } else if (hinfop->ach_src_deviceid == -1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *src_bdfp = bdf;

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *src_bdfp = hinfop->ach_src_deviceid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

ba758cf1b2fe06a606303351c36a766f2f9f6665Jerry Gilliam return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint16_t deviceid, domain_id_t *domainid, const char *path)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_get_domain";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *domainid = AMD_IOMMU_INVALID_DOMAIN;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp switch (deviceid) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp case AMD_IOMMU_INVALID_DOMAIN:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp case AMD_IOMMU_IDENTITY_DOMAIN:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp case AMD_IOMMU_PASSTHRU_DOMAIN:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp case AMD_IOMMU_SYS_DOMAIN:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *domainid = AMD_IOMMU_SYS_DOMAIN;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp break;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp default:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *domainid = deviceid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp break;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: domainid for %s = %d",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, path, *domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic uint16_t

7d87efa8fdfb9453670f2832df666fdae8291a84jmcphash_domain(domain_id_t domainid)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_init_page_tables(amd_iommu_t *iommu)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_domain_table = kmem_zalloc(

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_fini_page_tables(amd_iommu_t *iommu)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_domain_table) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(amd_iommu_domain_table,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_domain_table = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic amd_iommu_domain_t *

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp map_type_t type, int km_flags)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint16_t idx;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_domain_t *dp;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp char name[AMD_IOMMU_VMEM_NAMELEN+1];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(amd_iommu_domain_table);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp idx = hash_domain(domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dp->d_domainid == domainid)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (dp);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(type != AMD_IOMMU_INVALID_MAP);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp = kmem_zalloc(sizeof (*dp), km_flags);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dp == NULL)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_domainid = domainid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_pgtable_root_4K = 0; /* make this explicit */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (type == AMD_IOMMU_VMEM_MAP) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t base;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t size;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_idx, domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp base = MMU_PAGESIZE;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp MMU_PAGESIZE, NULL, NULL, NULL, 0,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dp->d_vmem == NULL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(dp, sizeof (*dp));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_vmem = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_next = amd_iommu_domain_table[idx];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_prev = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_domain_table[idx] = dp;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dp->d_next)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_next->d_prev = dp;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_ref = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (dp);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint16_t idx;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int flags;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_cmdargs_t cmdargs = {0};

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domain_id_t domainid = dp->d_domainid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_teardown_domain";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp->d_ref == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp idx = hash_domain(dp->d_domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dp->d_prev == NULL)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_domain_table[idx] = dp->d_next;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp else

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_prev->d_next = dp->d_next;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dp->d_next)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_next->d_prev = dp->d_prev;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dp->d_vmem != NULL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp vmem_destroy(dp->d_vmem);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_vmem = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(dp, sizeof (*dp));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmdargs.ca_domainid = (uint16_t)domainid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp &cmdargs, flags, 0) != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Failed to invalidate domain in IOMMU HW cache",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, iommu->aiomt_idx, cmdargs.ca_domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int *aliasp, const char *path)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int bus = -1;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int device = -1;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int func = -1;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint16_t bdf;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int32_t src_bdf;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *idip = iommu->aiomt_dip;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(idip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int instance = ddi_get_instance(idip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *pci_dip;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_get_deviceid";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* be conservative. Always assume an alias */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *aliasp = 1;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *deviceid = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Check for special special devices (rdip == NULL) */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (rdip == NULL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "%s: %s%d: idx=%d, failed to get SRC BDF "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "for special-device",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, iommu->aiomt_idx);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NOMAPPING);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *deviceid = src_bdf;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *aliasp = 1;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pci_dip = amd_iommu_pci_dip(rdip, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pci_dip == NULL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "for rdip=%p, path = %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, iommu->aiomt_idx, (void *)rdip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NOMAPPING);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ndi_rele_devi(pci_dip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "PCI dip (%p). rdip path = %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, iommu->aiomt_idx,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void *)pci_dip, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NOMAPPING);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ndi_rele_devi(pci_dip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (bus > UINT8_MAX || bus < 0 ||

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp device > UINT8_MAX || device < 0 ||

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp func > UINT8_MAX || func < 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d, invalid BDF(%d,%d,%d) "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "for PCI dip (%p). rdip path = %s", f, driver, instance,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_idx,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp bus, device, func,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void *)pci_dip, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NOMAPPING);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "for PCI dip (%p) rdip path = %s.",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, iommu->aiomt_idx, (void *)pci_dip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NOMAPPING);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, src_bdf, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *deviceid = src_bdf;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *aliasp = (src_bdf != bdf);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpinit_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_domain_t *dp)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t entry[4] = {0};

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int i;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* If already passthru, don't touch */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 &&

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 &&

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp->d_pgtable_root_4K ==

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_DEVTBL_ROOT_PGTBL));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_DEVTBL_DOMAINID));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* New devtbl entry for this domain. Bump up the domain ref-count */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_ref++;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp entry[3] = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp entry[2] = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (uint16_t)domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_pgtable_root_4K);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_PGTABLE_MAXLEVEL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (i = 1; i < 4; i++) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp devtbl_entry[i] = entry[i];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp devtbl_entry[0] = entry[0];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* we did an actual init */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int32_t deviceid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int alias;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *devtbl_entry;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_cmdargs_t cmdargs = {0};

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp char *path;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int pathfree;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int V;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int TV;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int instance;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_set_passthru";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (rdip) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp driver = ddi_driver_name(rdip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp instance = ddi_get_instance(rdip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp driver = "special-device";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp instance = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (path) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (rdip)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) ddi_pathname(rdip, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp else

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) strcpy(path, "special-device");

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pathfree = 1;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pathfree = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp path = "<path-mem-alloc-failed>";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Failed to get device ID for device %s.", f, driver,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp instance,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_idx, (void *)rdip, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto out;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* No deviceid */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (deviceid == -1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto out;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_devtbl_sz) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "for rdip (%p) exceeds device table size (%u), path=%s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp instance, iommu->aiomt_idx, deviceid, (void *)rdip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_devtbl_sz, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto out;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*LINTED*/

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Already passthru */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (V == 0 && TV == 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto out;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Existing translations */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (V == 1 && TV == 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto out;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Invalid setting */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (V == 0 && TV == 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto out;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmdargs.ca_deviceid = (uint16_t)deviceid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp &cmdargs, 0, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpout:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pathfree)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(path, MAXPATHLEN);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *path)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *devtbl_entry;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_cmdargs_t cmdargs = {0};

ac48dfe87039078897ed719af26744eca776508cVikram Hegde int error, flags;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *idip = iommu->aiomt_dip;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(idip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int instance = ddi_get_instance(idip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_set_devtbl_entry";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {

ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_devtbl_sz) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "for rdip (%p) exceeds device table size (%u), path=%s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp instance, iommu->aiomt_idx, deviceid, (void *)rdip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_devtbl_sz, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NOMAPPING);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*LINTED*/

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {

ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

ac48dfe87039078897ed719af26744eca776508cVikram Hegde /*

ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmdargs.ca_deviceid = deviceid;

ac48dfe87039078897ed719af26744eca776508cVikram Hegde error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,

ac48dfe87039078897ed719af26744eca776508cVikram Hegde &cmdargs, 0, 0);

ac48dfe87039078897ed719af26744eca776508cVikram Hegde if (error != DDI_SUCCESS) {

ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d"

ac48dfe87039078897ed719af26744eca776508cVikram Hegde "Failed to invalidate domain in IOMMU HW cache",

ac48dfe87039078897ed719af26744eca776508cVikram Hegde f, iommu->aiomt_idx, deviceid);

ac48dfe87039078897ed719af26744eca776508cVikram Hegde return (error);

ac48dfe87039078897ed719af26744eca776508cVikram Hegde }

ac48dfe87039078897ed719af26744eca776508cVikram Hegde

ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmdargs.ca_domainid = (uint16_t)domainid;

ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;

ac48dfe87039078897ed719af26744eca776508cVikram Hegde flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |

ac48dfe87039078897ed719af26744eca776508cVikram Hegde AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;

ac48dfe87039078897ed719af26744eca776508cVikram Hegde

ac48dfe87039078897ed719af26744eca776508cVikram Hegde error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,

ac48dfe87039078897ed719af26744eca776508cVikram Hegde &cmdargs, flags, 0);

ac48dfe87039078897ed719af26744eca776508cVikram Hegde if (error != DDI_SUCCESS) {

ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"

ac48dfe87039078897ed719af26744eca776508cVikram Hegde "Failed to invalidate translations in IOMMU HW cache",

ac48dfe87039078897ed719af26744eca776508cVikram Hegde f, iommu->aiomt_idx, cmdargs.ca_domainid);

ac48dfe87039078897ed719af26744eca776508cVikram Hegde return (error);

ac48dfe87039078897ed719af26744eca776508cVikram Hegde }

ac48dfe87039078897ed719af26744eca776508cVikram Hegde

ac48dfe87039078897ed719af26744eca776508cVikram Hegde /* Initialize device table entry */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (init_devtbl(iommu, devtbl_entry, domainid, dp)) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmdargs.ca_deviceid = deviceid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp &cmdargs, 0, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (error);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int *domain_freed, char *path)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *devtbl_entry;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int error = DDI_SUCCESS;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_cmdargs_t cmdargs = {0};

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(iommu->aiomt_dip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int instance = ddi_get_instance(iommu->aiomt_dip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_clear_devtbl_entry";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "domainid = %d, deviceid = %u, path = %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, domainid, deviceid, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_devtbl_sz) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "for rdip (%p) exceeds device table size (%u), path = %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_idx, deviceid, (void *)rdip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_devtbl_sz, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_FAILURE);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*LINTED*/

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {

ac48dfe87039078897ed719af26744eca776508cVikram Hegde cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Nothing to do */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_DEVTBL_ROOT_PGTBL));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_DEVTBL_DOMAINID));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp SYNC_FORDEV(iommu->aiomt_dmahdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_ref--;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp->d_ref >= 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dp->d_ref == 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *domain_freed = 1;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmdargs.ca_deviceid = deviceid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp &cmdargs, 0, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (error != DDI_SUCCESS)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp error = DDI_FAILURE;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (error);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) *

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(ampt->ampt_hash,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ampt->ampt_hash = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic uint32_t

7d87efa8fdfb9453670f2832df666fdae8291a84jmcppt_hashfn(uint64_t pa_4K)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint32_t idx = pt_hashfn(pa_4K);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_enter(&amd_iommu_page_table_hash.ampt_lock);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_prev = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_hash.ampt_hash[idx] = pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt->pt_next)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_next->pt_prev = pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_exit(&amd_iommu_page_table_hash.ampt_lock);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint32_t idx = pt_hashfn(pa_4K);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_enter(&amd_iommu_page_table_hash.ampt_lock);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt->pt_next)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_next->pt_prev = pt->pt_prev;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt->pt_prev)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_prev->pt_next = pt->pt_next;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp else

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_next = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_prev = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_exit(&amd_iommu_page_table_hash.ampt_lock);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic amd_iommu_page_table_t *

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint32_t idx = pt_hashfn(pgtable_pa_4K);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_enter(&amd_iommu_page_table_hash.ampt_lock);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt = amd_iommu_page_table_hash.ampt_hash[idx];

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (; pt; pt = pt->pt_next) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (domainid != pt->pt_domainid)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp continue;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT((pt->pt_cookie.dmac_cookie_addr &

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_PGTABLE_ALIGN) == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp break;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp mutex_exit(&amd_iommu_page_table_hash.ampt_lock);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (pt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic amd_iommu_page_table_t *

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_domain_t *dp, int level, uint16_t index)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *pdtep;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t pgtable_pa_4K;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(ppt == NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(index == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pgtable_pa_4K = dp->d_pgtable_root_4K;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(ppt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pdtep = &(ppt->pt_pgtblva[index]);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%"

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp PRIx64, *pdtep);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic amd_iommu_page_table_t *

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_alloc_from_freelist(void)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int i;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *pte_array;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_no_pgtable_freelist == 1)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_pgtable_freelist.f_count == 0)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt = amd_iommu_pgtable_freelist.f_list;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_pgtable_freelist.f_list = pt->pt_next;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_pgtable_freelist.f_count--;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pte_array = pt->pt_pgtblva;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_pte_ref[i] == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_PTDE_PR) == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (pt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *path, amd_iommu_page_table_t **ptp, int km_flags)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int err;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint_t ncookies;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *idip = iommu->aiomt_dip;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(idip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int instance = ddi_get_instance(idip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_alloc_pgtable";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *ptp = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt = amd_iommu_alloc_from_freelist();

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto init_pgtable;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt == NULL)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NORESOURCES);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp NULL, &pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (err != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Cannot alloc DMA handle for IO Page Table",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, domainid, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(pt, sizeof (amd_iommu_page_table_t));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp NULL, (caddr_t *)&pt->pt_pgtblva,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (err != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Cannot allocate DMA memory for IO Page table",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, domainid, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_free_handle(&pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(pt, sizeof (amd_iommu_page_table_t));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NORESOURCES);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp DDI_DMA_READ | DDI_DMA_CONSISTENT,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp NULL, &pt->pt_cookie, &ncookies);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (err != DDI_DMA_MAPPED) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Cannot bind memory for DMA to IO Page Tables. "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "bufrealsz=%p",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, domainid, path,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void *)(uintptr_t)pt->pt_mem_realsz);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_mem_free(&pt->pt_mem_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_free_handle(&pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(pt, sizeof (amd_iommu_page_table_t));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING :

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp err);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (ncookies != 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Cannot handle multiple "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "cookies for DMA to IO page Table, #cookies=%u",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, domainid, path, ncookies);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void) ddi_dma_unbind_handle(pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_mem_free(&pt->pt_mem_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_free_handle(&pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(pt, sizeof (amd_iommu_page_table_t));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_DMA_NOMAPPING);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpinit_pgtable:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_cookie.dmac_cookie_addr != NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_pgtblva);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_level = 0x7;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_index = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_ref = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_next = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_prev = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_parent = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp bzero(pt->pt_pgtblva, pt->pt_mem_realsz);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp SYNC_FORDEV(pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_insert_pgtable_hash(pt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *ptp = pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_move_to_freelist(amd_iommu_page_table_t *pt)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_no_pgtable_freelist == 1)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_FAILURE);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_pgtable_freelist.f_count ==

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_PGTABLE_FREELIST_MAX)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_FAILURE);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_next = amd_iommu_pgtable_freelist.f_list;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_pgtable_freelist.f_list = pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_pgtable_freelist.f_count++;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int i;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *pte_array;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dev_info_t *dip = iommu->aiomt_dip;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int instance = ddi_get_instance(dip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(dip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_free_pgtable";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_ref == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_remove_pgtable_hash(pt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pte_array = pt->pt_pgtblva;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_pte_ref[i] == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_PTDE_PR) == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Unbind the handle */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Failed to unbind handle: %p for IOMMU Page Table",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, iommu->aiomt_idx, pt->pt_domainid,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void *)pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Free the table memory allocated for DMA */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_mem_free(&pt->pt_mem_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Free the DMA handle */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ddi_dma_free_handle(&pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp kmem_free(pt, sizeof (amd_iommu_page_table_t));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpinit_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* nothing to set. PDE is already set */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(PT_REF_VALID(ppt));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(PT_REF_VALID(pt));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp == next_pgtable_pa_4K);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ppt->pt_ref++;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(PT_REF_VALID(ppt));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Page Directories are always RW */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp next_pgtable_pa_4K);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_parent = ppt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_level);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ppt->pt_pte_ref[pt->pt_index] = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp SYNC_FORDEV(ppt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpinit_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp struct ddi_dma_req *dmareq)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *ptep = &(pt->pt_pgtblva[index]);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t pa_4K = pa >> 12;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int R;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int W;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* nothing to set if PTE is already set */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /*

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_no_RW_perms == 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) ||

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (dmareq->dmar_flags & DDI_DMA_RDWR))) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (W == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) ||

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (dmareq->dmar_flags & DDI_DMA_RDWR))) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(PT_REF_VALID(pt));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_pte_ref[index]++;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp == pa_4K);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_ref++;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(PT_REF_VALID(pt));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* see comment above about inverting sense of RD/WR */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_no_RW_perms == 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dmareq->dmar_flags & DDI_DMA_RDWR) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dmareq->dmar_flags & DDI_DMA_WRITE) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (dmareq->dmar_flags & DDI_DMA_READ) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* TODO what is correct for FC and U */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_pte_ref[index] == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_pte_ref[index] = 1;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp SYNC_FORDEV(pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic void

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpinit_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int level, uint16_t index)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_domainid = dp->d_domainid;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_level = level;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_index = index;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp struct ddi_dma_req *dmareq,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domain_id_t domainid, amd_iommu_domain_t *dp,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *ppt,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint16_t index, int level, uint64_t va, uint64_t pa,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t **ptp, uint16_t *next_idxp, const char *path,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int km_flags)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int error;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(rdip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int instance = ddi_get_instance(rdip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_setup_1_pgtable";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *ptp = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *next_idxp = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp error = DDI_SUCCESS;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(ppt == NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(index == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(ppt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* Check if page table is already allocated */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_domainid == domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_level == level);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_index == index);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto out;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp km_flags)) != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "path = %s", f, driver, instance, iommu->aiomt_idx,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domainid, (void *)(uintptr_t)va, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (error);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(dp->d_domainid == domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp init_pt(pt, dp, level, index);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpout:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp error = init_pde(ppt, pt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (level == 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(error == DDI_SUCCESS);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *next_idxp = AMD_IOMMU_VA_BITS(va, level);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp *ptp = pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (error);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcptypedef enum {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp PDTE_NOT_TORN = 0x1,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp PDTE_TORN_DOWN = 0x2,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp PGTABLE_TORN_DOWN = 0x4

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp} pdte_tear_t;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic pdte_tear_t

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_teardown_pdte(amd_iommu_t *iommu,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *pt, int index)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint8_t next_level;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pdte_tear_t retval;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t *ptdep = &(pt->pt_pgtblva[index]);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp next_level = AMD_IOMMU_REG_GET64(ptdep,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_PTDE_NXT_LVL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt->pt_level == 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(next_level == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* PTE */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_pte_ref[index]--;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt->pt_pte_ref[index] != 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (PDTE_NOT_TORN);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(next_level != 0 && next_level != 7);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_pte_ref[index] == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(PT_REF_VALID(pt));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp SYNC_FORDEV(pt->pt_dma_hdl);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(AMD_IOMMU_REG_GET64(ptdep,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp AMD_IOMMU_PTDE_PR) == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp pt->pt_ref--;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(PT_REF_VALID(pt));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp retval = PDTE_TORN_DOWN;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_pte_ref[index] == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(PT_REF_VALID(pt));

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp retval = PDTE_NOT_TORN;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (pt->pt_ref == 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_free_pgtable(iommu, pt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (PGTABLE_TORN_DOWN);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (retval);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp}

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpstatic int

7d87efa8fdfb9453670f2832df666fdae8291a84jmcpamd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp struct ddi_dma_req *dmareq, uint64_t va,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint64_t pa, uint16_t deviceid, domain_id_t domainid,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_domain_t *dp, const char *path, int km_flags)

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp{

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int level;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint16_t index;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp uint16_t next_idx;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp amd_iommu_page_table_t *ppt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int error;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *driver = ddi_driver_name(rdip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp int instance = ddi_get_instance(rdip);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp const char *f = "amd_iommu_create_pgtables";

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "deviceid = %u, va = %p, pa = %p, path = %s",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_idx, domainid, deviceid,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void *)(uintptr_t)va,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void *)(uintptr_t)pa, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp /* No need for pagetables. Just set up device table entry */

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp goto passthru;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp index = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ppt = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp level--, pt = NULL, next_idx = 0) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domainid, dp, ppt, index, level, va, pa, &pt,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp &next_idx, path, km_flags)) != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "deviceid=%u, va= %p, pa = %p, Failed to setup "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "page table(s) at level = %d, path = %s.",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance, iommu->aiomt_idx,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp domainid, deviceid, (void *)(uintptr_t)va,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp (void *)(uintptr_t)pa, level, path);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp return (error);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if (level > 1) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt->pt_domainid == domainid);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ppt = pt;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp index = next_idx;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp } else {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(level == 1);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(pt == NULL);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ASSERT(next_idx == 0);

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp ppt = NULL;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp index = 0;

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp }

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp

7d87efa8fdfb9453670f2832df666fdae8291a84jmcppassthru:

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp dp, path)) != DDI_SUCCESS) {

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, "

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "domainid=%d."

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp "Failed to set device table entry for path %s.",

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp f, driver, instance,

7d87efa8fdfb9453670f2832df666fdae8291a84jmcp iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path);