Lines Matching defs:immu
35 #include <sys/immu.h>
38 #define get_reg32(immu, offset) ddi_get32((immu)->immu_regs_handle, \
39 (uint32_t *)(immu->immu_regs_addr + (offset)))
40 #define get_reg64(immu, offset) ddi_get64((immu)->immu_regs_handle, \
41 (uint64_t *)(immu->immu_regs_addr + (offset)))
42 #define put_reg32(immu, offset, val) ddi_put32\
43 ((immu)->immu_regs_handle, \
44 (uint32_t *)(immu->immu_regs_addr + (offset)), val)
45 #define put_reg64(immu, offset, val) ddi_put64\
46 ((immu)->immu_regs_handle, \
47 (uint64_t *)(immu->immu_regs_addr + (offset)), val)
65 #define wait_completion(immu, offset, getf, completion, status) \
71 status = getf(immu, offset); \
78 "immu wait completion time out"); \
97 iotlb_flush(immu_t *immu, uint_t domain_id,
105 iva_offset = IMMU_ECAP_GET_IRO(immu->immu_regs_excap);
111 if (IMMU_CAP_GET_DWD(immu->immu_regs_cap)) {
115 if (IMMU_CAP_GET_DRD(immu->immu_regs_cap)) {
138 immu->immu_name);
143 put_reg64(immu, iva_offset, iva);
144 put_reg64(immu, iotlb_offset, command);
145 wait_completion(immu, iotlb_offset, get_reg64,
155 immu_regs_iotlb_psi(immu_t *immu, uint_t did, uint64_t dvma, uint_t snpages,
167 if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap)) {
168 immu_regs_iotlb_dsi(immu, did, iwp);
172 max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap);
174 mutex_enter(&(immu->immu_regs_lock));
196 iotlb_flush(immu, did, dvma, am, hint, IOTLB_PSI);
204 iotlb_flush(immu, did, 0, 0, 0, IOTLB_DSI);
206 mutex_exit(&(immu->immu_regs_lock));
215 immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
217 mutex_enter(&(immu->immu_regs_lock));
218 iotlb_flush(immu, domain_id, 0, 0, 0, IOTLB_DSI);
219 mutex_exit(&(immu->immu_regs_lock));
228 immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp)
230 mutex_enter(&(immu->immu_regs_lock));
231 iotlb_flush(immu, 0, 0, 0, 0, IOTLB_GLOBAL);
232 mutex_exit(&(immu->immu_regs_lock));
259 set_agaw(immu_t *immu)
273 mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);
274 sagaw_mask = IMMU_CAP_SAGAW(immu->immu_regs_cap);
293 immu->immu_name, sagaw_mask, max_sagaw_mask);
325 "and magaw", immu->immu_name, agaw, magaw);
332 immu->immu_name, nlevels);
339 immu->immu_dvma_nlevels = nlevels;
340 immu->immu_dvma_agaw = agaw;
346 setup_regs(immu_t *immu)
353 mutex_init(&(immu->immu_regs_lock), NULL, MUTEX_DRIVER,
359 error = ddi_regs_map_setup(immu->immu_dip, 0,
360 (caddr_t *)&(immu->immu_regs_addr), (offset_t)0,
362 &(immu->immu_regs_handle));
366 immu->immu_name);
367 mutex_destroy(&(immu->immu_regs_lock));
374 immu->immu_regs_cap = get_reg64(immu, IMMU_REG_CAP);
375 immu->immu_regs_excap = get_reg64(immu, IMMU_REG_EXCAP);
380 if (IMMU_ECAP_GET_C(immu->immu_regs_excap)) {
381 immu->immu_dvma_coherent = B_TRUE;
383 immu->immu_dvma_coherent = B_FALSE;
386 "immu unit %s can't be enabled due to "
387 "missing clflush functionality", immu->immu_name);
388 ddi_regs_map_free(&(immu->immu_regs_handle));
389 mutex_destroy(&(immu->immu_regs_lock));
395 immu->immu_SNP_reserved = immu_regs_is_SNP_reserved(immu);
396 immu->immu_TM_reserved = immu_regs_is_TM_reserved(immu);
398 if (IMMU_ECAP_GET_CH(immu->immu_regs_excap) && immu_use_tm)
399 immu->immu_ptemask = PDTE_MASK_TM;
401 immu->immu_ptemask = 0;
407 !IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) {
411 IMMU_CAP_SET_RWBF(immu->immu_regs_cap);
417 immu->immu_max_domains = IMMU_CAP_ND(immu->immu_regs_cap);
422 if (set_agaw(immu) != DDI_SUCCESS) {
423 ddi_regs_map_free(&(immu->immu_regs_handle));
424 mutex_destroy(&(immu->immu_regs_lock));
427 immu->immu_regs_cmdval = 0;
429 immu->immu_flushops = &immu_regs_flushops;
445 immu_t *immu;
448 immu = list_head(listp);
449 for (; immu; immu = list_next(listp, immu)) {
451 if (setup_regs(immu) != DDI_SUCCESS) {
452 immu->immu_regs_setup = B_FALSE;
454 immu->immu_regs_setup = B_TRUE;
464 immu_regs_resume(immu_t *immu)
471 error = ddi_regs_map_setup(immu->immu_dip, 0,
472 (caddr_t *)&(immu->immu_regs_addr), (offset_t)0,
474 &(immu->immu_regs_handle));
479 immu_regs_set_root_table(immu);
481 immu_regs_intr_enable(immu, immu->immu_regs_intr_msi_addr,
482 immu->immu_regs_intr_msi_data, immu->immu_regs_intr_uaddr);
484 (void) immu_intr_handler(immu);
486 immu_regs_intrmap_enable(immu, immu->immu_intrmap_irta_reg);
488 immu_regs_qinv_enable(immu, immu->immu_qinv_reg_value);
498 immu_regs_suspend(immu_t *immu)
501 immu->immu_intrmap_running = B_FALSE;
504 ddi_regs_map_free(&(immu->immu_regs_handle));
512 immu_regs_startup(immu_t *immu)
516 if (immu->immu_regs_setup == B_FALSE) {
520 mutex_enter(&(immu->immu_regs_lock));
521 put_reg32(immu, IMMU_REG_GLOBAL_CMD,
522 immu->immu_regs_cmdval | IMMU_GCMD_TE);
523 wait_completion(immu, IMMU_REG_GLOBAL_STS,
525 immu->immu_regs_cmdval |= IMMU_GCMD_TE;
526 immu->immu_regs_running = B_TRUE;
527 mutex_exit(&(immu->immu_regs_lock));
529 ddi_err(DER_NOTE, NULL, "%s running", immu->immu_name);
537 immu_regs_shutdown(immu_t *immu)
541 if (immu->immu_regs_running == B_FALSE) {
545 mutex_enter(&(immu->immu_regs_lock));
546 immu->immu_regs_cmdval &= ~IMMU_GCMD_TE;
547 put_reg32(immu, IMMU_REG_GLOBAL_CMD,
548 immu->immu_regs_cmdval);
549 wait_completion(immu, IMMU_REG_GLOBAL_STS,
551 immu->immu_regs_running = B_FALSE;
552 mutex_exit(&(immu->immu_regs_lock));
554 ddi_err(DER_NOTE, NULL, "IOMMU %s stopped", immu->immu_name);
563 immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
566 mutex_enter(&(immu->immu_regs_lock));
567 immu->immu_regs_intr_msi_addr = msi_addr;
568 immu->immu_regs_intr_uaddr = uaddr;
569 immu->immu_regs_intr_msi_data = msi_data;
570 put_reg32(immu, IMMU_REG_FEVNT_ADDR, msi_addr);
571 put_reg32(immu, IMMU_REG_FEVNT_UADDR, uaddr);
572 put_reg32(immu, IMMU_REG_FEVNT_DATA, msi_data);
573 put_reg32(immu, IMMU_REG_FEVNT_CON, 0);
574 mutex_exit(&(immu->immu_regs_lock));
582 immu_regs_passthru_supported(immu_t *immu)
584 if (IMMU_ECAP_GET_PT(immu->immu_regs_excap)) {
597 immu_regs_is_TM_reserved(immu_t *immu)
599 if (IMMU_ECAP_GET_DI(immu->immu_regs_excap) ||
600 IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {
611 immu_regs_is_SNP_reserved(immu_t *immu)
614 return (IMMU_ECAP_GET_SC(immu->immu_regs_excap) ? B_FALSE : B_TRUE);
623 immu_regs_wbf_flush(immu_t *immu)
627 if (!IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) {
631 mutex_enter(&(immu->immu_regs_lock));
632 put_reg32(immu, IMMU_REG_GLOBAL_CMD,
633 immu->immu_regs_cmdval | IMMU_GCMD_WBF);
634 wait_completion(immu, IMMU_REG_GLOBAL_STS,
636 mutex_exit(&(immu->immu_regs_lock));
645 immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size)
649 if (immu->immu_dvma_coherent == B_TRUE)
668 context_flush(immu_t *immu, uint8_t function_mask,
693 immu->immu_name);
697 mutex_enter(&(immu->immu_regs_lock));
698 put_reg64(immu, IMMU_REG_CONTEXT_CMD, command);
699 wait_completion(immu, IMMU_REG_CONTEXT_CMD, get_reg64,
701 mutex_exit(&(immu->immu_regs_lock));
706 immu_regs_context_fsi(immu_t *immu, uint8_t function_mask,
709 context_flush(immu, function_mask, source_id, domain_id, CONTEXT_FSI);
714 immu_regs_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
716 context_flush(immu, 0, 0, domain_id, CONTEXT_DSI);
721 immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp)
723 context_flush(immu, 0, 0, 0, CONTEXT_GLOBAL);
736 immu_regs_set_root_table(immu_t *immu)
740 mutex_enter(&(immu->immu_regs_lock));
741 put_reg64(immu, IMMU_REG_ROOTENTRY,
742 immu->immu_ctx_root->hwpg_paddr);
743 put_reg32(immu, IMMU_REG_GLOBAL_CMD,
744 immu->immu_regs_cmdval | IMMU_GCMD_SRTP);
745 wait_completion(immu, IMMU_REG_GLOBAL_STS,
747 mutex_exit(&(immu->immu_regs_lock));
753 immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value)
760 mutex_enter(&immu->immu_regs_lock);
761 immu->immu_qinv_reg_value = qinv_reg_value;
763 put_reg64(immu, IMMU_REG_INVAL_QT, 0);
766 put_reg64(immu, IMMU_REG_INVAL_QAR, qinv_reg_value);
769 put_reg32(immu, IMMU_REG_GLOBAL_CMD,
770 immu->immu_regs_cmdval | IMMU_GCMD_QIE);
771 wait_completion(immu, IMMU_REG_GLOBAL_STS,
773 mutex_exit(&immu->immu_regs_lock);
775 immu->immu_regs_cmdval |= IMMU_GCMD_QIE;
776 immu->immu_qinv_running = B_TRUE;
782 immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg)
790 mutex_enter(&(immu->immu_regs_lock));
791 immu->immu_intrmap_irta_reg = irta_reg;
792 put_reg64(immu, IMMU_REG_IRTAR, irta_reg);
793 put_reg32(immu, IMMU_REG_GLOBAL_CMD,
794 immu->immu_regs_cmdval | IMMU_GCMD_SIRTP);
795 wait_completion(immu, IMMU_REG_GLOBAL_STS,
797 mutex_exit(&(immu->immu_regs_lock));
800 immu_qinv_intr_global(immu, &immu->immu_intrmap_inv_wait);
803 mutex_enter(&(immu->immu_regs_lock));
804 put_reg32(immu, IMMU_REG_GLOBAL_CMD,
805 immu->immu_regs_cmdval | IMMU_GCMD_IRE);
806 wait_completion(immu, IMMU_REG_GLOBAL_STS,
809 immu->immu_regs_cmdval |= IMMU_GCMD_IRE;
812 put_reg32(immu, IMMU_REG_GLOBAL_CMD,
813 immu->immu_regs_cmdval | IMMU_GCMD_CFI);
814 wait_completion(immu, IMMU_REG_GLOBAL_STS,
817 immu->immu_regs_cmdval |= IMMU_GCMD_CFI;
818 mutex_exit(&(immu->immu_regs_lock));
820 immu->immu_intrmap_running = B_TRUE;
824 immu_regs_get64(immu_t *immu, uint_t reg)
826 return (get_reg64(immu, reg));
830 immu_regs_get32(immu_t *immu, uint_t reg)
832 return (get_reg32(immu, reg));
836 immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val)
838 put_reg64(immu, reg, val);
842 immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val)
844 put_reg32(immu, reg, val);