immu.h revision d2256d265bf2bcad0d811b81411de3802a4b97c6
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Portions Copyright (c) 2010, Oracle and/or its affiliates.
* All rights reserved.
*/
/*
* Copyright (c) 2008, Intel Corporation.
* All rights reserved.
*/
#ifndef _SYS_INTEL_IOMMU_H
#define _SYS_INTEL_IOMMU_H
/*
* Intel IOMMU implementation specific state
*/
#ifdef __cplusplus
extern "C" {
#endif
#include <sys/types.h>
#include <sys/bitset.h>
#include <sys/kstat.h>
#include <sys/vmem.h>
#include <sys/rootnex.h>
/*
* Some ON drivers have bugs. Keep this define until all such drivers
* have been fixed
*/
#define BUGGY_DRIVERS 1
/* PD(T)E entries */
typedef uint64_t hw_pdte_t;
#define IMMU_MAXNAMELEN (64)
#define IMMU_MAXSEG (1)
#define IMMU_REGSZ (1UL << 12)
#define IMMU_PAGESIZE (4096)
#define IMMU_PAGESHIFT (12)
#define IMMU_PAGEOFFSET (IMMU_PAGESIZE - 1)
#define IMMU_PAGEMASK (~IMMU_PAGEOFFSET)
#define IMMU_BTOP(b) (((uint64_t)b) >> IMMU_PAGESHIFT)
#define IMMU_PTOB(p) (((uint64_t)p) << IMMU_PAGESHIFT)
#define IMMU_PGTABLE_MAX_LEVELS (6)
#define IMMU_ROUNDUP(size) (((size) + IMMU_PAGEOFFSET) & ~IMMU_PAGEOFFSET)
#define IMMU_ROUNDOWN(addr) ((addr) & ~IMMU_PAGEOFFSET)
#define IMMU_PGTABLE_LEVEL_STRIDE (9)
#define IMMU_PGTABLE_LEVEL_MASK ((1<<IMMU_PGTABLE_LEVEL_STRIDE) - 1)
#define IMMU_PGTABLE_OFFSHIFT (IMMU_PAGESHIFT - IMMU_PGTABLE_LEVEL_STRIDE)
#define IMMU_PGTABLE_MAXIDX ((IMMU_PAGESIZE / sizeof (hw_pdte_t)) - 1)
#define IMMU_ROUNDUP(size) (((size) + IMMU_PAGEOFFSET) & ~IMMU_PAGEOFFSET)
#define IMMU_ROUNDOWN(addr) ((addr) & ~IMMU_PAGEOFFSET)
/*
* DMAR global defines
*/
#define DMAR_TABLE "dmar-table"
#define DMAR_INTRMAP_SUPPORT (0x01)
/* DMAR unit types */
#define DMAR_DRHD 0
#define DMAR_RMRR 1
#define DMAR_ATSR 2
#define DMAR_RHSA 3
/* DRHD flag values */
#define DMAR_INCLUDE_ALL (0x01)
/* Device scope types */
#define DMAR_ENDPOINT 1
#define DMAR_SUBTREE 2
#define DMAR_IOAPIC 3
#define DMAR_HPET 4
/* Forward declarations for IOMMU state structure and DVMA domain struct */
struct immu;
struct domain;
/*
* The following structure describes the formate of DMAR ACPI table format.
* They are used to parse DMAR ACPI table. Read the spec for the meaning
* of each member.
*/
/* lengths of various strings */
#define DMAR_SIG_LEN (4) /* table signature */
#define DMAR_OEMID_LEN (6) /* OEM ID */
#define DMAR_TBLID_LEN (8) /* OEM table ID */
#define DMAR_ASL_LEN (4) /* ASL len */
typedef struct dmar_table {
kmutex_t tbl_lock;
uint8_t tbl_haw;
boolean_t tbl_intrmap;
list_t tbl_drhd_list[IMMU_MAXSEG];
list_t tbl_rmrr_list[IMMU_MAXSEG];
char *tbl_oem_id;
char *tbl_oem_tblid;
uint32_t tbl_oem_rev;
caddr_t tbl_raw;
int tbl_rawlen;
} dmar_table_t;
typedef struct drhd {
kmutex_t dr_lock; /* protects the dmar field */
struct immu *dr_immu;
dev_info_t *dr_dip;
uint16_t dr_seg;
uint64_t dr_regs;
boolean_t dr_include_all;
list_t dr_scope_list;
list_node_t dr_node;
} drhd_t;
typedef struct rmrr {
kmutex_t rm_lock;
uint16_t rm_seg;
uint64_t rm_base;
uint64_t rm_limit;
list_t rm_scope_list;
list_node_t rm_node;
} rmrr_t;
/*
* Macros based on PCI spec
*/
#define IMMU_PCI_DEV(devfunc) ((uint64_t)devfunc >> 3) /* from devfunc */
#define IMMU_PCI_FUNC(devfunc) (devfunc & 7) /* get func from devfunc */
#define IMMU_PCI_DEVFUNC(d, f) (((d) << 3) | (f)) /* create devfunc */
typedef struct scope {
uint8_t scp_type;
uint8_t scp_enumid;
uint8_t scp_bus;
uint8_t scp_dev;
uint8_t scp_func;
list_node_t scp_node;
} scope_t;
/*
* interrupt source id and drhd info for ioapic
*/
typedef struct ioapic_drhd {
uchar_t ioapic_ioapicid;
uint16_t ioapic_sid; /* ioapic source id */
drhd_t *ioapic_drhd;
list_node_t ioapic_node;
} ioapic_drhd_t;
typedef struct memrng {
uint64_t mrng_start;
uint64_t mrng_npages;
} memrng_t;
typedef enum immu_flags {
IMMU_FLAGS_NONE = 0x1,
IMMU_FLAGS_SLEEP = 0x1,
IMMU_FLAGS_NOSLEEP = 0x2,
IMMU_FLAGS_READ = 0x4,
IMMU_FLAGS_WRITE = 0x8,
IMMU_FLAGS_DONTPASS = 0x10,
IMMU_FLAGS_ALLOC = 0x20,
IMMU_FLAGS_MUST_MATCH = 0x40,
IMMU_FLAGS_PAGE1 = 0x80,
IMMU_FLAGS_UNITY = 0x100,
IMMU_FLAGS_DMAHDL = 0x200,
IMMU_FLAGS_MEMRNG = 0x400
} immu_flags_t;
typedef enum cont_avail {
IMMU_CONT_BAD = 0x0,
IMMU_CONT_UNINITED = 0x1,
IMMU_CONT_INITED = 0x2
} cont_avail_t;
/* Size of root and context tables and their entries */
#define IMMU_ROOT_TBLSZ (4096)
#define IMMU_CONT_TBLSZ (4096)
#define IMMU_ROOT_NUM (256)
#define IMMU_CONT_NUM (256)
/* register offset */
#define IMMU_REG_VERSION (0x00) /* Version Rigister, 32 bit */
#define IMMU_REG_CAP (0x08) /* Capability Register, 64 bit */
#define IMMU_REG_EXCAP (0x10) /* Extended Capability Reg, 64 bit */
#define IMMU_REG_GLOBAL_CMD (0x18) /* Global Command Register, 32 bit */
#define IMMU_REG_GLOBAL_STS (0x1C) /* Global Status Register, 32 bit */
#define IMMU_REG_ROOTENTRY (0x20) /* Root-Entry Table Addr Reg, 64 bit */
#define IMMU_REG_CONTEXT_CMD (0x28) /* Context Comand Register, 64 bit */
#define IMMU_REG_FAULT_STS (0x34) /* Fault Status Register, 32 bit */
#define IMMU_REG_FEVNT_CON (0x38) /* Fault Event Control Reg, 32 bit */
#define IMMU_REG_FEVNT_DATA (0x3C) /* Fault Event Data Register, 32 bit */
#define IMMU_REG_FEVNT_ADDR (0x40) /* Fault Event Address Reg, 32 bit */
#define IMMU_REG_FEVNT_UADDR (0x44) /* Fault Event Upper Addr Reg, 32 bit */
#define IMMU_REG_AFAULT_LOG (0x58) /* Advanced Fault Log Reg, 64 bit */
#define IMMU_REG_PMER (0x64) /* Protected Memory Enble Reg, 32 bit */
#define IMMU_REG_PLMBR (0x68) /* Protected Low Mem Base Reg, 32 bit */
#define IMMU_REG_PLMLR (0x6C) /* Protected Low Mem Lim Reg, 32 bit */
#define IMMU_REG_PHMBR (0X70) /* Protectd High Mem Base Reg, 64 bit */
#define IMMU_REG_PHMLR (0x78) /* Protected High Mem Lim Reg, 64 bit */
#define IMMU_REG_INVAL_QH (0x80) /* Invalidation Queue Head, 64 bit */
#define IMMU_REG_INVAL_QT (0x88) /* Invalidation Queue Tail, 64 bit */
#define IMMU_REG_INVAL_QAR (0x90) /* Invalidtion Queue Addr Reg, 64 bit */
#define IMMU_REG_INVAL_CSR (0x9C) /* Inval Compl Status Reg, 32 bit */
#define IMMU_REG_INVAL_CECR (0xA0) /* Inval Compl Evnt Ctrl Reg, 32 bit */
#define IMMU_REG_INVAL_CEDR (0xA4) /* Inval Compl Evnt Data Reg, 32 bit */
#define IMMU_REG_INVAL_CEAR (0xA8) /* Inval Compl Event Addr Reg, 32 bit */
#define IMMU_REG_INVAL_CEUAR (0xAC) /* Inval Comp Evnt Up Addr reg, 32bit */
#define IMMU_REG_IRTAR (0xB8) /* INTR Remap Tbl Addr Reg, 64 bit */
/* ioapic memory region */
#define IOAPIC_REGION_START (0xfee00000)
#define IOAPIC_REGION_END (0xfeefffff)
/* fault register */
#define IMMU_FAULT_STS_PPF (2)
#define IMMU_FAULT_STS_PFO (1)
#define IMMU_FAULT_STS_ITE (1 << 6)
#define IMMU_FAULT_STS_ICE (1 << 5)
#define IMMU_FAULT_STS_IQE (1 << 4)
#define IMMU_FAULT_GET_INDEX(x) ((((uint64_t)x) >> 8) & 0xff)
#define IMMU_FRR_GET_F(x) (((uint64_t)x) >> 63)
#define IMMU_FRR_GET_FR(x) ((((uint64_t)x) >> 32) & 0xff)
#define IMMU_FRR_GET_FT(x) ((((uint64_t)x) >> 62) & 0x1)
#define IMMU_FRR_GET_SID(x) ((x) & 0xffff)
/* (ex)capability register */
#define IMMU_CAP_GET_NFR(x) (((((uint64_t)x) >> 40) & 0xff) + 1)
#define IMMU_CAP_GET_DWD(x) ((((uint64_t)x) >> 54) & 1)
#define IMMU_CAP_GET_DRD(x) ((((uint64_t)x) >> 55) & 1)
#define IMMU_CAP_GET_PSI(x) ((((uint64_t)x) >> 39) & 1)
#define IMMU_CAP_GET_SPS(x) ((((uint64_t)x) >> 34) & 0xf)
#define IMMU_CAP_GET_ISOCH(x) ((((uint64_t)x) >> 23) & 1)
#define IMMU_CAP_GET_ZLR(x) ((((uint64_t)x) >> 22) & 1)
#define IMMU_CAP_GET_MAMV(x) ((((uint64_t)x) >> 48) & 0x3f)
#define IMMU_CAP_GET_CM(x) ((((uint64_t)x) >> 7) & 1)
#define IMMU_CAP_GET_PHMR(x) ((((uint64_t)x) >> 6) & 1)
#define IMMU_CAP_GET_PLMR(x) ((((uint64_t)x) >> 5) & 1)
#define IMMU_CAP_GET_RWBF(x) ((((uint64_t)x) >> 4) & 1)
#define IMMU_CAP_GET_AFL(x) ((((uint64_t)x) >> 3) & 1)
#define IMMU_CAP_GET_FRO(x) (((((uint64_t)x) >> 24) & 0x3ff) * 16)
#define IMMU_CAP_MGAW(x) (((((uint64_t)x) >> 16) & 0x3f) + 1)
#define IMMU_CAP_SAGAW(x) ((((uint64_t)x) >> 8) & 0x1f)
#define IMMU_CAP_ND(x) (1 << (((x) & 0x7) *2 + 4)) -1
#define IMMU_ECAP_GET_IRO(x) (((((uint64_t)x) >> 8) & 0x3ff) << 4)
#define IMMU_ECAP_GET_MHMV(x) (((uint64_t)x >> 20) & 0xf)
#define IMMU_ECAP_GET_SC(x) ((x) & 0x80)
#define IMMU_ECAP_GET_PT(x) ((x) & 0x40)
#define IMMU_ECAP_GET_CH(x) ((x) & 0x20)
#define IMMU_ECAP_GET_EIM(x) ((x) & 0x10)
#define IMMU_ECAP_GET_IR(x) ((x) & 0x8)
#define IMMU_ECAP_GET_DI(x) ((x) & 0x4)
#define IMMU_ECAP_GET_QI(x) ((x) & 0x2)
#define IMMU_ECAP_GET_C(x) ((x) & 0x1)
#define IMMU_CAP_SET_RWBF(x) ((x) |= (1 << 4))
/* iotlb invalidation */
#define TLB_INV_GLOBAL (((uint64_t)1) << 60)
#define TLB_INV_DOMAIN (((uint64_t)2) << 60)
#define TLB_INV_PAGE (((uint64_t)3) << 60)
#define TLB_INV_GET_IAIG(x) ((((uint64_t)x) >> 57) & 7)
#define TLB_INV_DRAIN_READ (((uint64_t)1) << 49)
#define TLB_INV_DRAIN_WRITE (((uint64_t)1) << 48)
#define TLB_INV_DID(x) (((uint64_t)((x) & 0xffff)) << 32)
#define TLB_INV_IVT (((uint64_t)1) << 63)
#define TLB_IVA_HINT(x) (((x) & 0x1) << 6)
#define TLB_IVA_LEAF 1
#define TLB_IVA_WHOLE 0
/* dont use value 0 for enums - to catch unit 8 */
typedef enum iotlb_inv {
IOTLB_PSI = 1,
IOTLB_DSI,
IOTLB_GLOBAL
} immu_iotlb_inv_t;
typedef enum context_inv {
CONTEXT_FSI = 1,
CONTEXT_DSI,
CONTEXT_GLOBAL
} immu_context_inv_t;
/* context invalidation */
#define CCMD_INV_ICC (((uint64_t)1) << 63)
#define CCMD_INV_GLOBAL (((uint64_t)1) << 61)
#define CCMD_INV_DOMAIN (((uint64_t)2) << 61)
#define CCMD_INV_DEVICE (((uint64_t)3) << 61)
#define CCMD_INV_DID(x) ((uint64_t)((x) & 0xffff))
#define CCMD_INV_SID(x) (((uint64_t)((x) & 0xffff)) << 16)
#define CCMD_INV_FM(x) (((uint64_t)((x) & 0x3)) << 32)
/* global command register */
#define IMMU_GCMD_TE (((uint32_t)1) << 31)
#define IMMU_GCMD_SRTP (((uint32_t)1) << 30)
#define IMMU_GCMD_SFL (((uint32_t)1) << 29)
#define IMMU_GCMD_EAFL (((uint32_t)1) << 28)
#define IMMU_GCMD_WBF (((uint32_t)1) << 27)
#define IMMU_GCMD_QIE (((uint32_t)1) << 26)
#define IMMU_GCMD_IRE (((uint32_t)1) << 25)
#define IMMU_GCMD_SIRTP (((uint32_t)1) << 24)
#define IMMU_GCMD_CFI (((uint32_t)1) << 23)
/* global status register */
#define IMMU_GSTS_TES (((uint32_t)1) << 31)
#define IMMU_GSTS_RTPS (((uint32_t)1) << 30)
#define IMMU_GSTS_FLS (((uint32_t)1) << 29)
#define IMMU_GSTS_AFLS (((uint32_t)1) << 28)
#define IMMU_GSTS_WBFS (((uint32_t)1) << 27)
#define IMMU_GSTS_QIES (((uint32_t)1) << 26)
#define IMMU_GSTS_IRES (((uint32_t)1) << 25)
#define IMMU_GSTS_IRTPS (((uint32_t)1) << 24)
#define IMMU_GSTS_CFIS (((uint32_t)1) << 23)
/* psi address mask */
#define ADDR_AM_MAX(m) (((uint_t)1) << (m))
#define ADDR_AM_OFFSET(n, m) ((n) & (ADDR_AM_MAX(m) - 1))
/* dmar fault event */
#define IMMU_INTR_IPL (4)
#define IMMU_REG_FEVNT_CON_IM_SHIFT (31)
#define IMMU_ALLOC_RESOURCE_DELAY (drv_usectohz(5000))
/* max value of Size field of Interrupt Remapping Table Address Register */
#define INTRMAP_MAX_IRTA_SIZE 0xf
/* interrupt remapping table entry size */
#define INTRMAP_RTE_SIZE 0x10
/* ioapic redirection table entry related shift of remappable interrupt */
#define INTRMAP_IOAPIC_IDX_SHIFT 17
#define INTRMAP_IOAPIC_FORMAT_SHIFT 16
#define INTRMAP_IOAPIC_TM_SHIFT 15
#define INTRMAP_IOAPIC_POL_SHIFT 13
#define INTRMAP_IOAPIC_IDX15_SHIFT 11
/* msi intr entry related shift of remappable interrupt */
#define INTRMAP_MSI_IDX_SHIFT 5
#define INTRMAP_MSI_FORMAT_SHIFT 4
#define INTRMAP_MSI_SHV_SHIFT 3
#define INTRMAP_MSI_IDX15_SHIFT 2
#define INTRMAP_IDX_FULL (uint_t)-1
#define RDT_DLM(rdt) BITX((rdt), 10, 8)
#define RDT_DM(rdt) BT_TEST(&(rdt), 11)
#define RDT_POL(rdt) BT_TEST(&(rdt), 13)
#define RDT_TM(rdt) BT_TEST(&(rdt), 15)
#define INTRMAP_DISABLE (void *)-1
/*
* invalidation granularity
*/
typedef enum {
TLB_INV_G_GLOBAL = 1,
TLB_INV_G_DOMAIN,
TLB_INV_G_PAGE
} tlb_inv_g_t;
typedef enum {
CTT_INV_G_GLOBAL = 1,
CTT_INV_G_DOMAIN,
CTT_INV_G_DEVICE
} ctt_inv_g_t;
typedef enum {
IEC_INV_GLOBAL = 0,
IEC_INV_INDEX
} iec_inv_g_t;
struct inv_queue_state;
struct intrmap_tbl_state;
/* A software page table structure */
typedef struct pgtable {
krwlock_t swpg_rwlock;
caddr_t hwpg_vaddr; /* HW pgtable VA */
paddr_t hwpg_paddr; /* HW pgtable PA */
ddi_dma_handle_t hwpg_dmahdl;
ddi_acc_handle_t hwpg_memhdl;
struct pgtable **swpg_next_array;
list_node_t swpg_domain_node; /* domain list of pgtables */
} pgtable_t;
/* interrupt remapping table state info */
typedef struct intrmap {
kmutex_t intrmap_lock;
ddi_dma_handle_t intrmap_dma_hdl;
ddi_acc_handle_t intrmap_acc_hdl;
caddr_t intrmap_vaddr;
paddr_t intrmap_paddr;
uint_t intrmap_size;
bitset_t intrmap_map;
uint_t intrmap_free;
} intrmap_t;
typedef struct hw_rce {
uint64_t lo;
uint64_t hi;
} hw_rce_t;
#define ROOT_GET_P(hrent) ((hrent)->lo & 0x1)
#define ROOT_SET_P(hrent) ((hrent)->lo |= 0x1)
#define ROOT_GET_CONT(hrent) ((hrent)->lo & ~(0xFFF))
#define ROOT_SET_CONT(hrent, paddr) ((hrent)->lo |= (paddr & (~0xFFF)))
#define TTYPE_XLATE_ONLY (0x0)
#define TTYPE_XLATE_IOTLB (0x1)
#define TTYPE_PASSTHRU (0x2)
#define TTYPE_RESERVED (0x3)
#define CONT_GET_DID(hcent) ((((uint64_t)(hcent)->hi) >> 8) & 0xFFFF)
#define CONT_SET_DID(hcent, did) ((hcent)->hi |= ((0xFFFF & (did)) << 8))
#define CONT_GET_AVAIL(hcent) ((((uint64_t)((hcent)->hi)) >> 0x3) & 0xF)
#define CONT_SET_AVAIL(hcent, av) ((hcent)->hi |= ((0xF & (av)) << 0x3))
#define CONT_GET_LO_AW(hcent) (30 + 9 *((hcent)->hi & 0x7))
#define CONT_GET_AW(hcent) \
((CONT_GET_LO_AW(hcent) == 66) ? 64 : CONT_GET_LO_AW(hcent))
#define CONT_SET_AW(hcent, aw) \
((hcent)->hi |= (((((aw) + 2) - 30) / 9) & 0x7))
#define CONT_GET_ASR(hcent) ((hcent)->lo & ~(0xFFF))
#define CONT_SET_ASR(hcent, paddr) ((hcent)->lo |= (paddr & (~0xFFF)))
#define CONT_GET_TTYPE(hcent) ((((uint64_t)(hcent)->lo) >> 0x2) & 0x3)
#define CONT_SET_TTYPE(hcent, ttype) ((hcent)->lo |= (((ttype) & 0x3) << 0x2))
#define CONT_GET_P(hcent) ((hcent)->lo & 0x1)
#define CONT_SET_P(hcent) ((hcent)->lo |= 0x1)
/* we use the bit 63 (available for system SW) as a present bit */
#define PDTE_SW4(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63))
#define PDTE_CLEAR_SW4(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63))
#define PDTE_P(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63))
#define PDTE_CLEAR_P(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63))
#define PDTE_SET_P(hw_pdte) ((hw_pdte) |= ((uint64_t)1<<63))
#define PDTE_TM(hw_pdte) ((hw_pdte) & ((uint64_t)1<<62))
#define PDTE_CLEAR_TM(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<62))
#define PDTE_SW3(hw_pdte) \
(((hw_pdte) & ~(((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) >> 52)
#define PDTE_SW3_OVERFLOW(hw_pdte) \
(PDTE_SW3(hw_pdte) == 0x3FF)
#define PDTE_CLEAR_SW3(hw_pdte) \
((hw_pdte) &= (((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1)))
#define PDTE_SET_SW3(hw_pdte, ref) \
((hw_pdte) |= ((((uint64_t)(ref)) & 0x3FF) << 52))
#define PDTE_PADDR(hw_pdte) ((hw_pdte) & ~(((uint64_t)0xFFF<<52)|((1<<12)-1)))
#define PDTE_CLEAR_PADDR(hw_pdte) \
((hw_pdte) &= (((uint64_t)0xFFF<<52)|((1<<12)-1)))
#define PDTE_SET_PADDR(hw_pdte, paddr) ((hw_pdte) |= PDTE_PADDR(paddr))
#define PDTE_SNP(hw_pdte) ((hw_pdte) & (1<<11))
#define PDTE_CLEAR_SNP(hw_pdte) ((hw_pdte) &= ~(1<<11))
#define PDTE_SET_SNP(hw_pdte) ((hw_pdte) |= (1<<11))
#define PDTE_SW2(hw_pdte) ((hw_pdte) & (0x700))
#define PDTE_CLEAR_SW2(hw_pdte) ((hw_pdte) &= ~(0x700))
#define PDTE_SP(hw_pdte) ((hw_pdte) & (0x80))
#define PDTE_CLEAR_SP(hw_pdte) ((hw_pdte) &= ~(0x80))
#define PDTE_SW1(hw_pdte) ((hw_pdte) & (0x7C))
#define PDTE_CLEAR_SW1(hw_pdte) ((hw_pdte) &= ~(0x7C))
#define PDTE_WRITE(hw_pdte) ((hw_pdte) & (0x2))
#define PDTE_CLEAR_WRITE(hw_pdte) ((hw_pdte) &= ~(0x2))
#define PDTE_SET_WRITE(hw_pdte) ((hw_pdte) |= (0x2))
#define PDTE_READ(hw_pdte) ((hw_pdte) & (0x1))
#define PDTE_CLEAR_READ(hw_pdte) ((hw_pdte) &= ~(0x1))
#define PDTE_SET_READ(hw_pdte) ((hw_pdte) |= (0x1))
struct immu_flushops;
typedef struct immu {
kmutex_t immu_lock;
char *immu_name;
/* lock grabbed by interrupt handler */
kmutex_t immu_intr_lock;
/* ACPI/DMAR table related */
void *immu_dmar_unit;
dev_info_t *immu_dip;
struct domain *immu_unity_domain;
/* IOMMU register related */
kmutex_t immu_regs_lock;
kcondvar_t immu_regs_cv;
boolean_t immu_regs_busy;
boolean_t immu_regs_setup;
boolean_t immu_regs_running;
boolean_t immu_regs_quiesced;
ddi_acc_handle_t immu_regs_handle;
caddr_t immu_regs_addr;
uint64_t immu_regs_cap;
uint64_t immu_regs_excap;
uint32_t immu_regs_cmdval;
uint32_t immu_regs_intr_msi_addr;
uint32_t immu_regs_intr_msi_data;
uint32_t immu_regs_intr_uaddr;
/* DVMA related */
kmutex_t immu_dvma_lock;
boolean_t immu_dvma_setup;
boolean_t immu_dvma_running;
int immu_dvma_gaw;
int immu_dvma_agaw;
int immu_dvma_nlevels;
boolean_t immu_dvma_coherent;
boolean_t immu_TM_reserved;
boolean_t immu_SNP_reserved;
/* DVMA context related */
krwlock_t immu_ctx_rwlock;
pgtable_t *immu_ctx_root;
/* DVMA domain related */
int immu_max_domains;
vmem_t *immu_did_arena;
char immu_did_arena_name[IMMU_MAXNAMELEN];
list_t immu_domain_list;
/* DVMA special devices */
boolean_t immu_dvma_gfx_only;
list_t immu_dvma_lpc_list;
list_t immu_dvma_gfx_list;
/* interrupt remapping related */
kmutex_t immu_intrmap_lock;
boolean_t immu_intrmap_setup;
boolean_t immu_intrmap_running;
intrmap_t *immu_intrmap;
uint64_t immu_intrmap_irta_reg;
/* queued invalidation related */
kmutex_t immu_qinv_lock;
boolean_t immu_qinv_setup;
boolean_t immu_qinv_running;
boolean_t immu_qinv_enabled;
void *immu_qinv;
uint64_t immu_qinv_reg_value;
/* list_node for system-wide list of DMAR units */
list_node_t immu_node;
struct immu_flushops *immu_flushops;
} immu_t;
/* properties that control DVMA */
#define DDI_DVMA_MAPTYPE_ROOTNEX_PROP "immu-dvma-mapping"
#define DDI_DVMA_MAPTYPE_UNITY "unity"
#define DDI_DVMA_MAPTYPE_XLATE "xlate"
typedef enum immu_maptype {
IMMU_MAPTYPE_BAD = 0, /* 0 is always bad */
IMMU_MAPTYPE_UNITY = 1,
IMMU_MAPTYPE_XLATE
} immu_maptype_t;
#define IMMU_COOKIE_HASHSZ (512)
/*
* domain_t
*
*/
typedef struct domain {
/* the basics */
uint_t dom_did;
immu_t *dom_immu;
/* mapping related */
immu_maptype_t dom_maptype;
vmem_t *dom_dvma_arena;
char dom_dvma_arena_name[IMMU_MAXNAMELEN];
/* pgtables */
pgtable_t *dom_pgtable_root;
krwlock_t dom_pgtable_rwlock;
/* list node for list of domains (unity or xlate) */
list_node_t dom_maptype_node;
/* list node for list of domains off immu */
list_node_t dom_immu_node;
mod_hash_t *dom_cookie_hash;
} domain_t;
typedef enum immu_pcib {
IMMU_PCIB_BAD = 0,
IMMU_PCIB_NOBDF,
IMMU_PCIB_PCIE_PCIE,
IMMU_PCIB_PCIE_PCI,
IMMU_PCIB_PCI_PCI,
IMMU_PCIB_ENDPOINT
} immu_pcib_t;
/*
* immu_devi_t
* Intel IOMMU in devinfo node
*/
typedef struct immu_devi {
/* pci seg, bus, dev, func */
int imd_seg;
int imd_bus;
int imd_devfunc;
/* ppb information */
immu_pcib_t imd_pcib_type;
int imd_sec;
int imd_sub;
/* identifier for special devices */
boolean_t imd_display;
boolean_t imd_lpc;
/* dmar unit to which this dip belongs */
immu_t *imd_immu;
immu_flags_t imd_dvma_flags;
/* domain ptr */
domain_t *imd_domain;
dev_info_t *imd_ddip;
/* my devinfo */
dev_info_t *imd_dip;
/*
* if we are a "special" devinfo
* the node for the special linked list
* off the DMAR unit structure
*/
list_node_t imd_spc_node;
} immu_devi_t;
#define IMMU_DEVI(dip) ((immu_devi_t *)(DEVI(dip)->devi_iommu))
#define IMMU_DEVI_SET(dip, imd) (DEVI(dip)->devi_iommu = (void *)imd)
/*
* struct dmar_arg
*/
typedef struct immu_arg {
int ima_seg;
int ima_bus;
int ima_devfunc;
dev_info_t *ima_rdip;
dev_info_t *ima_ddip;
} immu_arg_t;
/*
* Invalidation operation function pointers for context and IOTLB.
* These will be set to either the register or the queue invalidation
* interface functions, since the hardware does not allow using them
* both at the same time.
*/
struct immu_flushops {
void (*imf_context_fsi)(immu_t *, uint8_t, uint16_t, uint_t);
void (*imf_context_dsi)(immu_t *, uint_t);
void (*imf_context_gbl)(immu_t *);
void (*imf_iotlb_psi)(immu_t *, uint_t, uint64_t, uint_t, uint_t);
void (*imf_iotlb_dsi)(immu_t *, uint_t);
void (*imf_iotlb_gbl)(immu_t *);
};
#define immu_flush_context_fsi(i, f, s, d) \
(i)->immu_flushops->imf_context_fsi(i, f, s, d)
#define immu_flush_context_dsi(i, d) \
(i)->immu_flushops->imf_context_dsi(i, d)
#define immu_flush_context_gbl(i) \
(i)->immu_flushops->imf_context_gbl(i)
#define immu_flush_iotlb_psi(i, d, v, c, h) \
(i)->immu_flushops->imf_iotlb_psi(i, d, v, c, h)
#define immu_flush_iotlb_dsi(i, d) \
(i)->immu_flushops->imf_iotlb_dsi(i, d)
#define immu_flush_iotlb_gbl(i) \
(i)->immu_flushops->imf_iotlb_gbl(i)
/*
* Globals used by IOMMU code
*/
/* shared between IOMMU files */
extern dev_info_t *root_devinfo;
extern kmutex_t immu_lock;
extern list_t immu_list;
extern void *immu_pgtable_cache;
extern boolean_t immu_setup;
extern boolean_t immu_running;
extern kmutex_t ioapic_drhd_lock;
extern list_t ioapic_drhd_list;
/* switches */
/* Various features */
extern boolean_t immu_enable;
extern boolean_t immu_gfxdvma_enable;
extern boolean_t immu_intrmap_enable;
extern boolean_t immu_qinv_enable;
/* various quirks that need working around */
extern boolean_t immu_quirk_usbpage0;
extern boolean_t immu_quirk_usbfullpa;
extern boolean_t immu_quirk_usbrmrr;
extern boolean_t immu_quirk_mobile4;
/* debug messages */
extern boolean_t immu_dmar_print;
/* tunables */
extern int64_t immu_flush_gran;
extern immu_flags_t immu_global_dvma_flags;
/* ################### Interfaces exported outside IOMMU code ############## */
void immu_init(void);
void immu_startup(void);
void immu_shutdown(void);
void immu_destroy(void);
int immu_map_sgl(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
int prealloc_count, dev_info_t *rdip);
int immu_unmap_sgl(ddi_dma_impl_t *hp, dev_info_t *rdip);
void immu_device_tree_changed(void);
void immu_physmem_update(uint64_t addr, uint64_t size);
int immu_quiesce(void);
int immu_unquiesce(void);
/* ######################################################################### */
/* ################# Interfaces used within IOMMU code #################### */
/* functions in rootnex.c */
int rootnex_dvcookies_alloc(ddi_dma_impl_t *hp,
struct ddi_dma_req *dmareq, dev_info_t *rdip, void *arg);
void rootnex_dvcookies_free(dvcookie_t *dvcookies, void *arg);
/* immu_dmar.c interfaces */
int immu_dmar_setup(void);
int immu_dmar_parse(void);
void immu_dmar_startup(void);
void immu_dmar_shutdown(void);
void immu_dmar_destroy(void);
boolean_t immu_dmar_blacklisted(char **strings_array, uint_t nstrings);
immu_t *immu_dmar_get_immu(dev_info_t *rdip);
char *immu_dmar_unit_name(void *dmar_unit);
dev_info_t *immu_dmar_unit_dip(void *dmar_unit);
void immu_dmar_set_immu(void *dmar_unit, immu_t *immu);
void *immu_dmar_walk_units(int seg, void *dmar_unit);
boolean_t immu_dmar_intrmap_supported(void);
uint16_t immu_dmar_ioapic_sid(int ioapicid);
immu_t *immu_dmar_ioapic_immu(int ioapicid);
void immu_dmar_rmrr_map(void);
/* immu.c interfaces */
int immu_walk_ancestor(dev_info_t *rdip, dev_info_t *ddip,
int (*func)(dev_info_t *, void *arg), void *arg,
int *level, immu_flags_t immu_flags);
/* immu_regs.c interfaces */
void immu_regs_setup(list_t *immu_list);
void immu_regs_startup(immu_t *immu);
int immu_regs_resume(immu_t *immu);
void immu_regs_suspend(immu_t *immu);
void immu_regs_shutdown(immu_t *immu);
void immu_regs_destroy(list_t *immu_list);
void immu_regs_intr(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
uint32_t uaddr);
boolean_t immu_regs_passthru_supported(immu_t *immu);
boolean_t immu_regs_is_TM_reserved(immu_t *immu);
boolean_t immu_regs_is_SNP_reserved(immu_t *immu);
void immu_regs_wbf_flush(immu_t *immu);
void immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size);
void immu_regs_context_fsi(immu_t *immu, uint8_t function_mask,
uint16_t source_id, uint_t domain_id);
void immu_regs_context_dsi(immu_t *immu, uint_t domain_id);
void immu_regs_context_gbl(immu_t *immu);
void immu_regs_iotlb_psi(immu_t *immu, uint_t domain_id,
uint64_t dvma, uint_t count, uint_t hint);
void immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id);
void immu_regs_iotlb_gbl(immu_t *immu);
void immu_regs_set_root_table(immu_t *immu);
void immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value);
void immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
uint32_t uaddr);
void immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg);
uint64_t immu_regs_get64(immu_t *immu, uint_t reg);
void immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val);
uint32_t immu_regs_get32(immu_t *immu, uint_t reg);
void immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val);
/* immu_dvma.c interfaces */
void immu_dvma_setup(list_t *immu_list);
void immu_dvma_startup(immu_t *immu);
void immu_dvma_shutdown(immu_t *immu);
void immu_dvma_destroy(list_t *immu_list);
void immu_dvma_physmem_update(uint64_t addr, uint64_t size);
int immu_dvma_map(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, memrng_t *,
uint_t prealloc_count, dev_info_t *rdip, immu_flags_t immu_flags);
int immu_dvma_unmap(ddi_dma_impl_t *hp, dev_info_t *rdip);
int immu_dvma_alloc(dvcookie_t *first_dvcookie, void *arg);
void immu_dvma_free(dvcookie_t *first_dvcookie, void *arg);
int immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags);
immu_devi_t *immu_devi_get(dev_info_t *dip);
immu_t *immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags);
int pgtable_ctor(void *buf, void *arg, int kmflag);
void pgtable_dtor(void *buf, void *arg);
/* immu_intrmap.c interfaces */
void immu_intrmap_setup(list_t *immu_list);
void immu_intrmap_startup(immu_t *immu);
void immu_intrmap_shutdown(immu_t *immu);
void immu_intrmap_destroy(list_t *immu_list);
/* registers interrupt handler for IOMMU unit */
void immu_intr_register(immu_t *immu);
int immu_intr_handler(immu_t *immu);
/* immu_qinv.c interfaces */
int immu_qinv_setup(list_t *immu_list);
void immu_qinv_startup(immu_t *immu);
void immu_qinv_shutdown(immu_t *immu);
void immu_qinv_destroy(list_t *immu_list);
void immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask,
uint16_t source_id, uint_t domain_id);
void immu_qinv_context_dsi(immu_t *immu, uint_t domain_id);
void immu_qinv_context_gbl(immu_t *immu);
void immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id,
uint64_t dvma, uint_t count, uint_t hint);
void immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id);
void immu_qinv_iotlb_gbl(immu_t *immu);
void immu_qinv_intr_global(immu_t *immu);
void immu_qinv_intr_one_cache(immu_t *immu, uint_t idx);
void immu_qinv_intr_caches(immu_t *immu, uint_t idx, uint_t cnt);
void immu_qinv_report_fault(immu_t *immu);
#ifdef __cplusplus
}
#endif
#endif /* _SYS_INTEL_IOMMU_H */