Lines Matching defs:iommu
39 static int amd_iommu_fini(amd_iommu_t *iommu, int type);
40 static void amd_iommu_teardown_interrupts(amd_iommu_t *iommu);
41 static void amd_iommu_stop(amd_iommu_t *iommu);
68 static int unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
114 amd_iommu_register(amd_iommu_t *iommu)
116 dev_info_t *dip = iommu->aiomt_dip;
127 iommulib_ops->ilops_data = (void *)iommu;
128 iommu->aiomt_iommulib_ops = iommulib_ops;
133 "failed idx=%d", f, driver, instance, iommu->aiomt_idx);
138 iommu->aiomt_iommulib_handle = handle;
144 amd_iommu_unregister(amd_iommu_t *iommu)
146 if (iommu->aiomt_iommulib_handle == NULL) {
151 if (iommulib_iommu_unregister(iommu->aiomt_iommulib_handle)
156 kmem_free(iommu->aiomt_iommulib_ops, sizeof (iommulib_ops_t));
157 iommu->aiomt_iommulib_ops = NULL;
158 iommu->aiomt_iommulib_handle = NULL;
164 amd_iommu_setup_passthru(amd_iommu_t *iommu)
172 amd_iommu_set_passthru(iommu, NULL);
178 amd_iommu_set_passthru(iommu, dip);
187 amd_iommu_start(amd_iommu_t *iommu)
189 dev_info_t *dip = iommu->aiomt_dip;
201 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
209 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
211 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
213 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
217 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
231 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
237 iommu->aiomt_idx);
240 instance, iommu->aiomt_idx);
246 amd_iommu_stop(amd_iommu_t *iommu)
248 dev_info_t *dip = iommu->aiomt_dip;
253 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
255 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
257 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
259 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
265 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
267 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
272 iommu->aiomt_idx);
276 amd_iommu_setup_tables_and_buffers(amd_iommu_t *iommu)
278 dev_info_t *dip = iommu->aiomt_dip;
298 iommu->aiomt_devtbl_sz = (1<<AMD_IOMMU_DEVTBL_SZ) * AMD_IOMMU_DEVENT_SZ;
299 iommu->aiomt_cmdbuf_sz = (1<<AMD_IOMMU_CMDBUF_SZ) * AMD_IOMMU_CMD_SZ;
300 iommu->aiomt_eventlog_sz =
303 dma_bufsz = iommu->aiomt_devtbl_sz + iommu->aiomt_cmdbuf_sz
304 + iommu->aiomt_eventlog_sz;
310 DDI_DMA_SLEEP, NULL, &iommu->aiomt_dmahdl);
321 err = ddi_dma_mem_alloc(iommu->aiomt_dmahdl, dma_bufsz,
323 DDI_DMA_SLEEP, NULL, (caddr_t *)&iommu->aiomt_dma_bufva,
324 (size_t *)&iommu->aiomt_dma_mem_realsz, &iommu->aiomt_dma_mem_hdl);
328 iommu->aiomt_dma_bufva = NULL;
329 iommu->aiomt_dma_mem_realsz = 0;
330 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
331 iommu->aiomt_dmahdl = NULL;
338 ASSERT(((uintptr_t)iommu->aiomt_dma_bufva &
340 ASSERT(iommu->aiomt_dma_mem_realsz >= dma_bufsz);
345 err = ddi_dma_addr_bind_handle(iommu->aiomt_dmahdl, NULL,
346 iommu->aiomt_dma_bufva, iommu->aiomt_dma_mem_realsz,
348 NULL, &iommu->aiomt_buf_dma_cookie, &iommu->aiomt_buf_dma_ncookie);
353 (void *)(uintptr_t)iommu->aiomt_dma_mem_realsz);
354 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
355 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
356 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
357 iommu->aiomt_buf_dma_ncookie = 0;
358 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
359 iommu->aiomt_dma_mem_hdl = NULL;
360 iommu->aiomt_dma_bufva = NULL;
361 iommu->aiomt_dma_mem_realsz = 0;
362 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
363 iommu->aiomt_dmahdl = NULL;
372 if (iommu->aiomt_buf_dma_ncookie != 1) {
376 iommu->aiomt_buf_dma_ncookie);
377 (void) ddi_dma_unbind_handle(iommu->aiomt_dmahdl);
378 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
379 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
380 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
381 iommu->aiomt_buf_dma_ncookie = 0;
382 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
383 iommu->aiomt_dma_mem_hdl = NULL;
384 iommu->aiomt_dma_bufva = NULL;
385 iommu->aiomt_dma_mem_realsz = 0;
386 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
387 iommu->aiomt_dmahdl = NULL;
394 ASSERT((iommu->aiomt_buf_dma_cookie.dmac_cookie_addr
396 ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size
397 <= iommu->aiomt_dma_mem_realsz);
398 ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size >= dma_bufsz);
401 * Setup the device table pointers in the iommu struct as
404 iommu->aiomt_devtbl = iommu->aiomt_dma_bufva;
405 bzero(iommu->aiomt_devtbl, iommu->aiomt_devtbl_sz);
414 dentry = (uint64_t *)&iommu->aiomt_devtbl
420 addr = (caddr_t)(uintptr_t)iommu->aiomt_buf_dma_cookie.dmac_cookie_addr;
421 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
423 sz = (iommu->aiomt_devtbl_sz >> 12) - 1;
425 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
431 iommu->aiomt_cmdbuf = iommu->aiomt_devtbl +
432 iommu->aiomt_devtbl_sz;
433 bzero(iommu->aiomt_cmdbuf, iommu->aiomt_cmdbuf_sz);
434 addr += iommu->aiomt_devtbl_sz;
435 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
441 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
444 iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf;
445 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
447 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va),
453 iommu->aiomt_eventlog = iommu->aiomt_cmdbuf +
454 iommu->aiomt_eventlog_sz;
455 bzero(iommu->aiomt_eventlog, iommu->aiomt_eventlog_sz);
456 addr += iommu->aiomt_cmdbuf_sz;
457 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
462 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
465 iommu->aiomt_event_head = (uint32_t *)iommu->aiomt_eventlog;
466 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
468 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_tail_va),
472 SYNC_FORDEV(iommu->aiomt_dmahdl);
476 "tables, idx=%d", f, driver, instance, iommu->aiomt_idx);
483 amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu, int type)
485 dev_info_t *dip = iommu->aiomt_dip;
490 iommu->aiomt_eventlog = NULL;
491 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
493 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
495 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
497 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
501 iommu->aiomt_cmdbuf = NULL;
502 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
504 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
506 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
508 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
512 iommu->aiomt_devtbl = NULL;
513 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
515 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
518 if (iommu->aiomt_dmahdl == NULL || type == AMD_IOMMU_QUIESCE)
522 if (ddi_dma_unbind_handle(iommu->aiomt_dmahdl) != DDI_SUCCESS) {
525 (void *)iommu->aiomt_dmahdl, iommu->aiomt_idx);
527 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
528 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
529 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
530 iommu->aiomt_buf_dma_ncookie = 0;
533 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
534 iommu->aiomt_dma_mem_hdl = NULL;
535 iommu->aiomt_dma_bufva = NULL;
536 iommu->aiomt_dma_mem_realsz = 0;
539 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
540 iommu->aiomt_dmahdl = NULL;
544 amd_iommu_enable_interrupts(amd_iommu_t *iommu)
546 ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
548 ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
553 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
556 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
558 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
560 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
565 amd_iommu_setup_exclusion(amd_iommu_t *iommu)
573 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
576 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
578 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
580 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
587 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
589 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
591 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
593 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
601 amd_iommu_teardown_exclusion(amd_iommu_t *iommu)
603 (void) amd_iommu_setup_exclusion(iommu);
610 amd_iommu_t *iommu = (amd_iommu_t *)arg1;
611 dev_info_t *dip = iommu->aiomt_dip;
621 f, driver, instance, iommu->aiomt_idx);
624 if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
629 iommu->aiomt_idx);
631 (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISPLAY);
633 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
638 if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
642 iommu->aiomt_idx);
643 (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISCARD);
644 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
646 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
656 amd_iommu_setup_interrupts(amd_iommu_t *iommu)
658 dev_info_t *dip = iommu->aiomt_dip;
675 "failed: idx=%d", f, driver, instance, iommu->aiomt_idx);
682 iommu->aiomt_idx, type);
691 f, driver, instance, iommu->aiomt_idx);
697 f, driver, instance, iommu->aiomt_idx);
704 f, driver, instance, iommu->aiomt_idx, err);
711 f, driver, instance, iommu->aiomt_idx, req);
717 driver, instance, iommu->aiomt_idx);
725 driver, instance, iommu->aiomt_idx, err);
732 f, driver, instance, iommu->aiomt_idx, avail);
738 driver, instance, iommu->aiomt_idx);
745 "Failing init", f, driver, instance, iommu->aiomt_idx,
751 iommu->aiomt_intr_htable_sz = req * sizeof (ddi_intr_handle_t);
752 iommu->aiomt_intr_htable = kmem_zalloc(iommu->aiomt_intr_htable_sz,
755 iommu->aiomt_intr_state = AMD_IOMMU_INTR_TABLE;
767 f, driver, instance, iommu->aiomt_idx, p2req, req);
770 err = ddi_intr_alloc(iommu->aiomt_dip, iommu->aiomt_intr_htable,
775 f, driver, instance, iommu->aiomt_idx, err);
776 amd_iommu_teardown_interrupts(iommu);
780 iommu->aiomt_actual_intrs = actual;
781 iommu->aiomt_intr_state = AMD_IOMMU_INTR_ALLOCED;
786 f, driver, instance, iommu->aiomt_idx, actual);
789 if (iommu->aiomt_actual_intrs < req) {
792 f, driver, instance, iommu->aiomt_idx,
793 iommu->aiomt_actual_intrs, req);
794 amd_iommu_teardown_interrupts(iommu);
798 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
799 if (ddi_intr_add_handler(iommu->aiomt_intr_htable[i],
800 amd_iommu_intr_handler, (void *)iommu, NULL)
804 f, driver, instance, iommu->aiomt_idx, i, err);
807 iommu->aiomt_intr_htable[j]);
809 amd_iommu_teardown_interrupts(iommu);
813 iommu->aiomt_intr_state = AMD_IOMMU_INTR_HANDLER;
816 if (ddi_intr_get_cap(iommu->aiomt_intr_htable[0], &intrcap0)
819 iommu->aiomt_intr_htable[iommu->aiomt_actual_intrs - 1], &intrcapN)
824 f, driver, instance, iommu->aiomt_idx, intrcap0, intrcapN);
825 amd_iommu_teardown_interrupts(iommu);
828 iommu->aiomt_intr_cap = intrcap0;
835 f, driver, instance, iommu->aiomt_idx);
837 if (ddi_intr_block_enable(iommu->aiomt_intr_htable,
838 iommu->aiomt_actual_intrs) != DDI_SUCCESS) {
841 instance, iommu->aiomt_idx);
842 (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
843 iommu->aiomt_actual_intrs);
844 amd_iommu_teardown_interrupts(iommu);
851 f, driver, instance, iommu->aiomt_idx);
853 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
854 if (ddi_intr_enable(iommu->aiomt_intr_htable[i])
858 driver, instance, iommu->aiomt_idx, i);
861 iommu->aiomt_intr_htable[j]);
863 amd_iommu_teardown_interrupts(iommu);
868 iommu->aiomt_intr_state = AMD_IOMMU_INTR_ENABLED;
873 f, driver, instance, iommu->aiomt_idx,
875 "(individually)", iommu->aiomt_actual_intrs);
882 amd_iommu_teardown_interrupts(amd_iommu_t *iommu)
886 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ENABLED) {
887 if (iommu->aiomt_intr_cap & DDI_INTR_FLAG_BLOCK) {
888 (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
889 iommu->aiomt_actual_intrs);
891 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
893 iommu->aiomt_intr_htable[i]);
898 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_HANDLER) {
899 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
901 iommu->aiomt_intr_htable[i]);
905 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ALLOCED) {
906 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
907 (void) ddi_intr_free(iommu->aiomt_intr_htable[i]);
910 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_TABLE) {
911 kmem_free(iommu->aiomt_intr_htable,
912 iommu->aiomt_intr_htable_sz);
914 iommu->aiomt_intr_htable = NULL;
915 iommu->aiomt_intr_htable_sz = 0;
916 iommu->aiomt_intr_state = AMD_IOMMU_INTR_INVALID;
923 amd_iommu_t *iommu;
946 iommu = kmem_zalloc(sizeof (amd_iommu_t), KM_SLEEP);
947 mutex_init(&iommu->aiomt_mutex, NULL, MUTEX_DRIVER, NULL);
948 mutex_enter(&iommu->aiomt_mutex);
950 mutex_init(&iommu->aiomt_cmdlock, NULL, MUTEX_DRIVER, NULL);
951 mutex_init(&iommu->aiomt_eventlock, NULL, MUTEX_DRIVER, NULL);
953 iommu->aiomt_dip = dip;
954 iommu->aiomt_idx = idx;
956 if (acpica_get_bdf(iommu->aiomt_dip, &bus, &device, &func)
964 iommu->aiomt_bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) |
974 iommu->aiomt_cap_hdr = caphdr;
975 iommu->aiomt_npcache = AMD_IOMMU_REG_GET32(&caphdr,
977 iommu->aiomt_httun = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_HTTUN);
980 hinfop = amd_iommu_lookup_any_ivhd(iommu);
983 iommu->aiomt_iotlb = hinfop->ach_IotlbSup;
985 iommu->aiomt_iotlb =
988 iommu->aiomt_captype = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
989 iommu->aiomt_capid = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
996 iommu->aiomt_low_addr32 = low_addr32;
997 iommu->aiomt_hi_addr32 = hi_addr32;
1001 iommu->aiomt_reg_pa = hinfop->ach_IOMMU_reg_base;
1004 iommu->aiomt_reg_pa = ((uint64_t)hi_addr32 << 32 | low_addr32);
1011 iommu->aiomt_range = range;
1012 iommu->aiomt_rng_valid = AMD_IOMMU_REG_GET32(&range,
1014 if (iommu->aiomt_rng_valid) {
1015 iommu->aiomt_rng_bus = AMD_IOMMU_REG_GET32(&range,
1017 iommu->aiomt_first_devfn = AMD_IOMMU_REG_GET32(&range,
1019 iommu->aiomt_last_devfn = AMD_IOMMU_REG_GET32(&range,
1022 iommu->aiomt_rng_bus = 0;
1023 iommu->aiomt_first_devfn = 0;
1024 iommu->aiomt_last_devfn = 0;
1028 iommu->aiomt_ht_unitid = hinfop->ach_IOMMU_UnitID;
1030 iommu->aiomt_ht_unitid = AMD_IOMMU_REG_GET32(&range,
1037 iommu->aiomt_misc = misc;
1040 iommu->aiomt_htatsresv = global->acg_HtAtsResv;
1041 iommu->aiomt_vasize = global->acg_VAsize;
1042 iommu->aiomt_pasize = global->acg_PAsize;
1044 iommu->aiomt_htatsresv = AMD_IOMMU_REG_GET32(&misc,
1046 iommu->aiomt_vasize = AMD_IOMMU_REG_GET32(&misc,
1048 iommu->aiomt_pasize = AMD_IOMMU_REG_GET32(&misc,
1053 iommu->aiomt_msinum = hinfop->ach_IOMMU_MSInum;
1055 iommu->aiomt_msinum =
1062 pgoffset = iommu->aiomt_reg_pa & MMU_PAGEOFFSET;
1064 iommu->aiomt_reg_pages = mmu_btopr(AMD_IOMMU_REG_SIZE + pgoffset);
1065 iommu->aiomt_reg_size = mmu_ptob(iommu->aiomt_reg_pages);
1067 iommu->aiomt_va = (uintptr_t)device_arena_alloc(
1068 ptob(iommu->aiomt_reg_pages), VM_SLEEP);
1069 if (iommu->aiomt_va == 0) {
1073 mutex_exit(&iommu->aiomt_mutex);
1074 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1078 hat_devload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
1079 iommu->aiomt_reg_size,
1080 mmu_btop(iommu->aiomt_reg_pa), PROT_READ | PROT_WRITE
1083 iommu->aiomt_reg_va = iommu->aiomt_va + pgoffset;
1088 iommu->aiomt_reg_devtbl_va = iommu->aiomt_reg_va +
1090 iommu->aiomt_reg_cmdbuf_va = iommu->aiomt_reg_va +
1092 iommu->aiomt_reg_eventlog_va = iommu->aiomt_reg_va +
1094 iommu->aiomt_reg_ctrl_va = iommu->aiomt_reg_va +
1096 iommu->aiomt_reg_excl_base_va = iommu->aiomt_reg_va +
1098 iommu->aiomt_reg_excl_lim_va = iommu->aiomt_reg_va +
1100 iommu->aiomt_reg_cmdbuf_head_va = iommu->aiomt_reg_va +
1102 iommu->aiomt_reg_cmdbuf_tail_va = iommu->aiomt_reg_va +
1104 iommu->aiomt_reg_eventlog_head_va = iommu->aiomt_reg_va +
1106 iommu->aiomt_reg_eventlog_tail_va = iommu->aiomt_reg_va +
1108 iommu->aiomt_reg_status_va = iommu->aiomt_reg_va +
1116 if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) {
1117 mutex_exit(&iommu->aiomt_mutex);
1118 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1122 if (amd_iommu_setup_exclusion(iommu) != DDI_SUCCESS) {
1123 mutex_exit(&iommu->aiomt_mutex);
1124 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1128 amd_iommu_enable_interrupts(iommu);
1130 if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) {
1131 mutex_exit(&iommu->aiomt_mutex);
1132 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1139 amd_iommu_init_page_tables(iommu);
1147 if (amd_iommu_setup_passthru(iommu) != DDI_SUCCESS) {
1148 mutex_exit(&iommu->aiomt_mutex);
1149 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1154 if (amd_iommu_acpi_init_devtbl(iommu) != DDI_SUCCESS) {
1157 mutex_exit(&iommu->aiomt_mutex);
1158 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1162 if (amd_iommu_start(iommu) != DDI_SUCCESS) {
1163 mutex_exit(&iommu->aiomt_mutex);
1164 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1169 if (amd_iommu_register(iommu) != DDI_SUCCESS) {
1170 mutex_exit(&iommu->aiomt_mutex);
1171 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1180 return (iommu);
1184 amd_iommu_fini(amd_iommu_t *iommu, int type)
1186 int idx = iommu->aiomt_idx;
1187 dev_info_t *dip = iommu->aiomt_dip;
1193 mutex_enter(&iommu->aiomt_mutex);
1194 if (amd_iommu_unregister(iommu) != DDI_SUCCESS) {
1201 amd_iommu_stop(iommu);
1204 amd_iommu_fini_page_tables(iommu);
1205 amd_iommu_teardown_interrupts(iommu);
1206 amd_iommu_teardown_exclusion(iommu);
1209 amd_iommu_teardown_tables_and_buffers(iommu, type);
1214 if (iommu->aiomt_va != NULL) {
1215 hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
1216 iommu->aiomt_reg_size, HAT_UNLOAD_UNLOCK);
1217 device_arena_free((void *)(uintptr_t)iommu->aiomt_va,
1218 ptob(iommu->aiomt_reg_pages));
1219 iommu->aiomt_va = NULL;
1220 iommu->aiomt_reg_va = NULL;
1222 mutex_destroy(&iommu->aiomt_eventlock);
1223 mutex_destroy(&iommu->aiomt_cmdlock);
1224 mutex_exit(&iommu->aiomt_mutex);
1225 mutex_destroy(&iommu->aiomt_mutex);
1226 kmem_free(iommu, sizeof (amd_iommu_t));
1249 amd_iommu_t *iommu;
1314 iommu = amd_iommu_init(dip, handle, idx, cap_base);
1315 if (iommu == NULL) {
1323 statep->aioms_iommu_start = iommu;
1325 statep->aioms_iommu_end->aiomt_next = iommu;
1327 statep->aioms_iommu_end = iommu;
1347 amd_iommu_t *iommu, *next_iommu;
1353 for (iommu = statep->aioms_iommu_start; iommu;
1354 iommu = next_iommu) {
1356 next_iommu = iommu->aiomt_next;
1357 if (amd_iommu_fini(iommu, type) != DDI_SUCCESS) {
1419 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1421 int instance = ddi_get_instance(iommu->aiomt_dip);
1422 const char *idriver = ddi_driver_name(iommu->aiomt_dip);
1432 amd_iommu_set_passthru(iommu, rdip);
1446 f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
1455 f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
1468 if (hinfop && hinfop->ach_IOMMU_deviceid == iommu->aiomt_bdf)
1494 map_current_window(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
1498 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1499 int instance = ddi_get_instance(iommu->aiomt_dip);
1500 int idx = iommu->aiomt_idx;
1523 if ((error = amd_iommu_map_pa2va(iommu, rdip, attrp, dmareq,
1536 (void) unmap_current_window(iommu, rdip, cookie_array,
1554 unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
1557 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1558 int instance = ddi_get_instance(iommu->aiomt_dip);
1559 int idx = iommu->aiomt_idx;
1582 if (amd_iommu_unmap_va(iommu, rdip,
1590 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, NULL, 0, 0)
1628 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1669 error = map_current_window(iommu, rdip, attrp, dmareq,
1707 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1740 if (unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0)
1810 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1848 (void) unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0);
1870 error = map_current_window(iommu, rdip, attrp, &sdmareq,
1944 * if "amd-iommu = no/false" boot property is set,
1945 * ignore AMD iommu
1948 DDI_PROP_DONTPASS, "amd-iommu", &propval) == DDI_SUCCESS) {
1960 DDI_PROP_DONTPASS, "amd-iommu-disable-list", &propval)
1976 DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu", &disable)
1985 DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu-disable-list",