Lines Matching defs:immu
47 #include <sys/immu.h>
74 static domain_t *domain_create(immu_t *immu, dev_info_t *ddip,
248 immu_devi_set_spclist(dev_info_t *dip, immu_t *immu)
255 spclist = &(immu->immu_dvma_gfx_list);
257 spclist = &(immu->immu_dvma_lpc_list);
261 mutex_enter(&(immu->immu_lock));
263 mutex_exit(&(immu->immu_lock));
321 get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)
324 dvarg.dva_list = &(immu->immu_dvma_lpc_list);
347 immu_t *immu;
356 immu = list_head(&immu_list);
357 for (; immu; immu = list_next(&immu_list, immu)) {
358 list_gfx = &(immu->immu_dvma_gfx_list);
426 immu_t *immu = arg;
444 if (!immu->immu_dvma_coherent)
505 pgtable_alloc(immu_t *immu, immu_flags_t immu_flags)
512 pgtable = kmem_cache_alloc(immu->immu_pgtable_cache, kmflags);
527 pgtable_free(immu_t *immu, pgtable_t *pgtable)
529 kmem_cache_free(immu->immu_pgtable_cache, pgtable);
608 * get the immu unit structure for a dev_info node
614 immu_t *immu;
617 * check if immu unit was already found earlier.
636 immu = immu_devi->imd_immu;
638 return (immu);
642 immu = immu_dmar_get_immu(dip);
643 if (immu == NULL) {
650 * Check if some other thread found immu
664 immu_devi->imd_immu = immu;
665 immu_devi_set_spclist(dip, immu);
668 * if some other thread got immu before
671 if (immu_devi->imd_immu != immu) {
673 "immu units found for device. Expected (%p), "
674 "actual (%p)", (void *)immu,
682 return (immu);
805 did_alloc(immu_t *immu, dev_info_t *rdip,
810 did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1,
815 " domain-device: %s%d. immu unit is %s. Using "
818 immu->immu_name, immu->immu_unity_domain->dom_did);
819 did = immu->immu_unity_domain->dom_did;
831 immu_t *immu;
852 immu = immu_devi->imd_immu;
853 if (immu == NULL)
854 immu = immu_dvma_get_immu(pdip, dvp->dva_flags);
880 dvp->dva_domain = immu->immu_unity_domain;
1019 create_xlate_arena(immu_t *immu, domain_t *domain,
1036 "%s-domain-%d-xlate-DVMA-arena", immu->immu_name,
1042 mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);
1174 immu_t *immu;
1192 immu = immu_dvma_get_immu(rdip, immu_flags);
1193 if (immu == NULL) {
1246 domain = domain_create(immu, ddip, rdip, immu_flags);
1266 create_unity_domain(immu_t *immu)
1278 domain->dom_immu = immu;
1279 immu->immu_unity_domain = domain;
1285 domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1293 if (!IMMU_ECAP_GET_PT(immu->immu_regs_excap))
1312 domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,
1330 did = did_alloc(immu, rdip, ddip, immu_flags);
1333 ASSERT(immu->immu_unity_domain);
1334 return (immu->immu_unity_domain);
1341 "structure for device. IOMMU unit: %s", immu->immu_name);
1348 "immu%s-domain%d-pava-hash", immu->immu_name, did);
1351 domain->dom_immu = immu;
1358 create_xlate_arena(immu, domain, rdip, immu_flags);
1363 domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);
1367 domain->dom_did, immu->immu_name);
1373 * Since this is a immu unit-specific domain, put it on
1374 * the per-immu domain list.
1376 mutex_enter(&(immu->immu_lock));
1377 list_insert_head(&immu->immu_domain_list, domain);
1378 mutex_exit(&(immu->immu_lock));
1418 did_init(immu_t *immu)
1420 (void) snprintf(immu->immu_did_arena_name,
1421 sizeof (immu->immu_did_arena_name),
1422 "%s_domainid_arena", immu->immu_name);
1424 ddi_err(DER_VERB, immu->immu_dip, "creating domainid arena %s",
1425 immu->immu_did_arena_name);
1427 immu->immu_did_arena = vmem_create(
1428 immu->immu_did_arena_name,
1430 immu->immu_max_domains - IMMU_UNITY_DID,
1439 if (immu->immu_did_arena == NULL) {
1441 "IOMMU domainid allocator: %s", immu->immu_name,
1442 immu->immu_did_arena_name);
1449 context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,
1472 rw_enter(&(immu->immu_ctx_rwlock), RW_READER);
1478 rw_exit(&(immu->immu_ctx_rwlock));
1489 rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) {
1490 rw_exit(&(immu->immu_ctx_rwlock));
1491 rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
1500 immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t));
1509 immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
1512 immu_flush_context_fsi(immu, 0, sid, domain->dom_did,
1513 &immu->immu_ctx_inv_wait);
1517 CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
1520 IMMU_ECAP_GET_PT(immu->immu_regs_excap))
1526 if (IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {
1531 immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
1533 rw_exit(&(immu->immu_ctx_rwlock));
1537 context_create(immu_t *immu)
1549 root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1559 context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1567 immu->immu_unity_domain->dom_pgtable_root;
1569 immu->immu_unity_domain->dom_did);
1570 CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
1572 if (IMMU_ECAP_GET_PT(immu->immu_regs_excap))
1580 immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE);
1591 context_init(immu_t *immu)
1593 rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL);
1595 immu_init_inv_wait(&immu->immu_ctx_inv_wait, "ctxglobal", B_TRUE);
1597 immu_regs_wbf_flush(immu);
1599 immu->immu_ctx_root = context_create(immu);
1601 immu_regs_set_root_table(immu);
1603 rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
1604 immu_flush_context_gbl(immu, &immu->immu_ctx_inv_wait);
1605 immu_flush_iotlb_gbl(immu, &immu->immu_ctx_inv_wait);
1606 rw_exit(&(immu->immu_ctx_rwlock));
1629 immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,
1677 context_set(immu, domain, immu->immu_ctx_root, r_bus,
1686 context_set(immu, domain, immu->immu_ctx_root, r_bus,
1696 context_set(immu, domain, immu->immu_ctx_root,
1699 context_set(immu, domain, immu->immu_ctx_root,
1709 context_set(immu, domain, immu->immu_ctx_root,
1718 context_set(immu, domain, immu->immu_ctx_root, d_bus,
1725 context_set(immu, domain, immu->immu_ctx_root, d_bus,
1743 PDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr,
1763 if (next == NULL && immu->immu_TM_reserved == B_FALSE) {
1802 if (next == NULL && immu->immu_SNP_reserved == B_FALSE) {
1846 PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
1971 immu_t *immu;
1981 immu = domain->dom_immu;
1990 nlevels = immu->immu_dvma_nlevels;
2011 PTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr,
2017 pte = immu->immu_ptemask;
2035 if (immu->immu_TM_reserved == B_FALSE) {
2048 if (immu->immu_SNP_reserved == B_FALSE) {
2075 pte |= immu->immu_ptemask;
2094 PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
2135 PTE_set_one(immu, hwp, paddr, rdip, immu_flags);
2138 ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)
2171 PDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next,
2241 PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,
2279 new = pgtable_alloc(immu, immu_flags);
2299 PDE_set_one(immu, hwp, next, rdip, immu_flags);
2323 ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)
2331 pgtable_free(immu, new);
2341 * immu: IOMMU unit for which we are generating DVMA cookies
2356 immu_t *immu = domain->dom_immu;
2357 int nlevels = immu->immu_dvma_nlevels;
2368 if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)
2374 PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,
2385 * immu: IOMMU unit state
2396 immu_t *immu = domain->dom_immu;
2397 int nlevels = immu->immu_dvma_nlevels;
2416 PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);
2460 immu_t *immu;
2465 immu = domain->dom_immu;
2466 nlevels = immu->immu_dvma_nlevels;
2499 (void) PDE_set_all(immu, domain, xlate, nlevels, rdip,
2507 PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie,
2555 immu_t *immu;
2571 immu = domain->dom_immu;
2619 rwmask = PDTE_MASK_R | PDTE_MASK_W | immu->immu_ptemask;
2621 rwmask = immu->immu_ptemask;
2716 immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc,
2776 immu_t *immu;
2798 immu = list_head(listp);
2799 for (; immu; immu = list_next(listp, immu)) {
2800 create_unity_domain(immu);
2801 did_init(immu);
2802 context_init(immu);
2803 immu->immu_dvma_setup = B_TRUE;
2811 immu_dvma_startup(immu_t *immu)
2814 immu->immu_dvma_gfx_only == B_TRUE) {
2821 immu->immu_dvma_running = B_TRUE;
2876 immu_t *immu;
2881 immu = immu_dvma_get_immu(rdip, immu_flags);
2882 if (immu == NULL) {
2895 rdip = get_lpc_devinfo(immu, rdip, immu_flags);
2902 /* Reset immu, as redirection can change IMMU */
2903 immu = NULL;
2928 immu = domain->dom_immu;
2947 if (immu_context_update(immu, domain, ddip, rdip, immu_flags)
2961 immu_t *immu;
2969 immu = domain->dom_immu;
2977 immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start,
3060 immu_t *immu;
3065 immu = IMMU_DEVI(rdip)->imd_immu;
3067 ihp = kmem_cache_alloc(immu->immu_hdl_cache,