/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
*/
/*
* PCI iommu initialization and configuration
*/
#include <sys/types.h>
#include <sys/kmem.h>
#include <sys/async.h>
#include <sys/sysmacros.h>
#include <sys/sunddi.h>
#include <sys/ddi_impldefs.h>
#include <sys/vmem.h>
#include <sys/machsystm.h> /* lddphys() */
#include <sys/iommutsb.h>
#include <sys/pci/pci_obj.h>
/*LINTLIBRARY*/
static void iommu_tlb_flushall(iommu_t *iommu_p);
static void iommu_preserve_tsb(iommu_t *iommu_p);
void
iommu_create(pci_t *pci_p)
{
dev_info_t *dip = pci_p->pci_dip;
iommu_t *iommu_p;
uintptr_t a;
size_t cache_size;
uint32_t tsb_entries;
char map_name[32];
extern uint64_t va_to_pa(void *);
pci_dvma_range_prop_t pci_dvma_range;
/*
* Allocate iommu state structure and link it to the
* pci state structure.
*/
iommu_p = (iommu_t *)kmem_zalloc(sizeof (iommu_t), KM_SLEEP);
pci_p->pci_iommu_p = iommu_p;
iommu_p->iommu_pci_p = pci_p;
iommu_p->iommu_inst = ddi_get_instance(dip);
/*
* chip specific dvma_end, tsb_size & context support
*/
iommu_p->iommu_dvma_end = pci_iommu_dvma_end;
a = pci_iommu_setup(iommu_p);
/*
* Determine the virtual address of iommu registers.
*/
iommu_p->iommu_ctrl_reg =
(uint64_t *)(a + COMMON_IOMMU_CTRL_REG_OFFSET);
iommu_p->iommu_tsb_base_addr_reg =
(uint64_t *)(a + COMMON_IOMMU_TSB_BASE_ADDR_REG_OFFSET);
iommu_p->iommu_flush_page_reg =
(uint64_t *)(a + COMMON_IOMMU_FLUSH_PAGE_REG_OFFSET);
/*
* Configure the rest of the iommu parameters according to:
* tsb_size and dvma_end
*/
iommu_p->iommu_tsb_vaddr = /* retrieve TSB VA reserved by system */
iommu_tsb_cookie_to_va(pci_p->pci_tsb_cookie);
iommu_p->iommu_tsb_entries = tsb_entries =
IOMMU_TSBSIZE_TO_TSBENTRIES(iommu_p->iommu_tsb_size);
iommu_p->iommu_tsb_paddr = va_to_pa((caddr_t)iommu_p->iommu_tsb_vaddr);
iommu_p->iommu_dvma_cache_locks =
kmem_zalloc(pci_dvma_page_cache_entries, KM_SLEEP);
iommu_p->iommu_dvma_base = iommu_p->iommu_dvma_end + 1
- (tsb_entries * IOMMU_PAGE_SIZE);
iommu_p->dvma_base_pg = IOMMU_BTOP(iommu_p->iommu_dvma_base);
iommu_p->iommu_dvma_reserve = tsb_entries >> 1;
iommu_p->dvma_end_pg = IOMMU_BTOP(iommu_p->iommu_dvma_end);
iommu_p->iommu_dma_bypass_base = COMMON_IOMMU_BYPASS_BASE;
iommu_p->iommu_dma_bypass_end = pci_iommu_bypass_end_configure();
/*
* export "virtual-dma" software property to support
* child devices needing to know DVMA range
*/
pci_dvma_range.dvma_base = (uint32_t)iommu_p->iommu_dvma_base;
pci_dvma_range.dvma_len = (uint32_t)
iommu_p->iommu_dvma_end - iommu_p->iommu_dvma_base + 1;
(void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
"virtual-dma", (caddr_t)&pci_dvma_range,
sizeof (pci_dvma_range));
DEBUG2(DBG_ATTACH, dip, "iommu_create: ctrl=%p, tsb=%p\n",
iommu_p->iommu_ctrl_reg, iommu_p->iommu_tsb_base_addr_reg);
DEBUG2(DBG_ATTACH, dip, "iommu_create: page_flush=%p, ctx_flush=%p\n",
iommu_p->iommu_flush_page_reg, iommu_p->iommu_flush_ctx_reg);
DEBUG2(DBG_ATTACH, dip, "iommu_create: tsb vaddr=%p tsb_paddr=%p\n",
iommu_p->iommu_tsb_vaddr, iommu_p->iommu_tsb_paddr);
DEBUG1(DBG_ATTACH, dip, "iommu_create: allocated size=%x\n",
iommu_tsb_cookie_to_size(pci_p->pci_tsb_cookie));
DEBUG2(DBG_ATTACH, dip, "iommu_create: fast tsb tte addr: %x + %x\n",
iommu_p->iommu_tsb_vaddr,
pci_dvma_page_cache_entries * pci_dvma_page_cache_clustsz);
DEBUG3(DBG_ATTACH, dip,
"iommu_create: tsb size=%x, tsb entries=%x, dvma base=%x\n",
iommu_p->iommu_tsb_size, iommu_p->iommu_tsb_entries,
iommu_p->iommu_dvma_base);
DEBUG2(DBG_ATTACH, dip,
"iommu_create: dvma_cache_locks=%x cache_entries=%x\n",
iommu_p->iommu_dvma_cache_locks, pci_dvma_page_cache_entries);
/*
* zero out the area to be used for iommu tsb
*/
bzero(iommu_p->iommu_tsb_vaddr, tsb_entries << 3);
/*
* Create a virtual memory map for dvma address space.
* Reserve 'size' bytes of low dvma space for fast track cache.
*/
(void) snprintf(map_name, sizeof (map_name), "%s%d_dvma",
ddi_driver_name(dip), ddi_get_instance(dip));
cache_size = IOMMU_PTOB(pci_dvma_page_cache_entries *
pci_dvma_page_cache_clustsz);
iommu_p->iommu_dvma_fast_end = iommu_p->iommu_dvma_base +
cache_size - 1;
iommu_p->iommu_dvma_map = vmem_create(map_name,
(void *)(iommu_p->iommu_dvma_fast_end + 1),
IOMMU_PTOB(tsb_entries) - cache_size, IOMMU_PAGE_SIZE,
NULL, NULL, NULL, IOMMU_PAGE_SIZE, VM_SLEEP);
mutex_init(&iommu_p->dvma_debug_lock, NULL, MUTEX_DRIVER, NULL);
/*
* On detach, the TSB Base Address Register gets set to zero,
* so if its zero here, there is no need to preserve TTEs.
*/
if (pci_preserve_iommu_tsb && *iommu_p->iommu_tsb_base_addr_reg)
iommu_preserve_tsb(iommu_p);
iommu_configure(iommu_p);
}
void
iommu_destroy(pci_t *pci_p)
{
#ifdef DEBUG
dev_info_t *dip = pci_p->pci_dip;
#endif
iommu_t *iommu_p = pci_p->pci_iommu_p;
volatile uint64_t ctl_val = *iommu_p->iommu_ctrl_reg;
DEBUG0(DBG_DETACH, dip, "iommu_destroy:\n");
/*
* Disable the IOMMU by setting the TSB Base Address to zero
* and the TSB Table size to the smallest possible.
*/
ctl_val = ctl_val & ~(7 << COMMON_IOMMU_CTRL_TSB_SZ_SHIFT);
*iommu_p->iommu_ctrl_reg = ctl_val;
*iommu_p->iommu_tsb_base_addr_reg = 0;
/*
* Return the boot time allocated tsb.
*/
iommu_tsb_free(pci_p->pci_tsb_cookie);
/*
* Teardown any implementation-specific structures set up in
* pci_iommu_setup.
*/
pci_iommu_teardown(iommu_p);
if (DVMA_DBG_ON(iommu_p))
pci_dvma_debug_fini(iommu_p);
mutex_destroy(&iommu_p->dvma_debug_lock);
/*
* Free the dvma resource map.
*/
vmem_destroy(iommu_p->iommu_dvma_map);
kmem_free(iommu_p->iommu_dvma_cache_locks,
pci_dvma_page_cache_entries);
/*
* Free the iommu state structure.
*/
kmem_free(iommu_p, sizeof (iommu_t));
pci_p->pci_iommu_p = NULL;
}
/*
* re-program iommu on the fly while preserving on-going dma
* transactions on the PCI bus.
*/
void
iommu_configure(iommu_t *iommu_p)
{
pci_t *pci_p = iommu_p->iommu_pci_p;
uint64_t cfgpa = pci_get_cfg_pabase(pci_p);
dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
dev_info_t *cdip = NULL;
volatile uint64_t ctl_val = (uint64_t)
((iommu_p->iommu_tsb_size << COMMON_IOMMU_CTRL_TSB_SZ_SHIFT) |
(0 /* 8k page */ << COMMON_IOMMU_CTRL_TBW_SZ_SHIFT) |
COMMON_IOMMU_CTRL_ENABLE | COMMON_IOMMU_CTRL_DIAG_ENABLE |
(pci_lock_tlb ? COMMON_IOMMU_CTRL_LCK_ENABLE : 0));
DEBUG2(DBG_ATTACH, dip, "iommu_configure: iommu_ctl=%08x.%08x\n",
HI32(ctl_val), LO32(ctl_val));
if (!pci_preserve_iommu_tsb || !(*iommu_p->iommu_tsb_base_addr_reg)) {
*iommu_p->iommu_ctrl_reg = COMMON_IOMMU_CTRL_DIAG_ENABLE;
iommu_tlb_flushall(iommu_p);
goto config;
}
cdip = ddi_get_child(dip);
for (; cdip; cdip = ddi_get_next_sibling(cdip)) {
uint32_t *reg_p;
int reg_len;
if (ddi_getlongprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
"reg", (caddr_t)&reg_p, &reg_len) != DDI_PROP_SUCCESS)
continue;
cfgpa += (*reg_p) & (PCI_CONF_ADDR_MASK ^ PCI_REG_REG_M);
kmem_free(reg_p, reg_len);
break;
}
config:
pci_iommu_config(iommu_p, ctl_val, cdip ? cfgpa : 0);
}
void
iommu_map_pages(iommu_t *iommu_p, ddi_dma_impl_t *mp,
dvma_addr_t dvma_pg, size_t npages, size_t pfn_index)
{
int i;
dvma_addr_t pg_index = dvma_pg - iommu_p->dvma_base_pg;
uint64_t *tte_addr = iommu_p->iommu_tsb_vaddr + pg_index;
size_t pfn_last = pfn_index + npages;
uint64_t tte = PCI_GET_MP_TTE(mp->dmai_tte);
#ifdef DEBUG
dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
#endif
ASSERT(pfn_last <= mp->dmai_ndvmapages);
DEBUG5(DBG_MAP_WIN, dip,
"iommu_map_pages:%x+%x=%x npages=0x%x pfn_index=0x%x\n",
(uint_t)iommu_p->dvma_base_pg, (uint_t)pg_index, dvma_pg,
(uint_t)npages, (uint_t)pfn_index);
for (i = pfn_index; i < pfn_last; i++, pg_index++, tte_addr++) {
iopfn_t pfn = PCI_GET_MP_PFN(mp, i);
volatile uint64_t cur_tte = IOMMU_PTOB(pfn) | tte;
DEBUG3(DBG_MAP_WIN, dip, "iommu_map_pages: mp=%p pg[%x]=%x\n",
mp, i, (uint_t)pfn);
DEBUG3(DBG_MAP_WIN, dip,
"iommu_map_pages: pg_index=%x tte=%08x.%08x\n",
pg_index, HI32(cur_tte), LO32(cur_tte));
ASSERT(TTE_IS_INVALID(*tte_addr));
*tte_addr = cur_tte;
#ifdef DEBUG
if (pfn == 0 && pci_warn_pp0)
cmn_err(CE_WARN, "%s%d <%p> doing DMA to pp0\n",
ddi_driver_name(mp->dmai_rdip),
ddi_get_instance(mp->dmai_rdip), mp);
#endif
}
ASSERT(tte_addr == iommu_p->iommu_tsb_vaddr + pg_index);
#ifdef DEBUG
if (HAS_REDZONE(mp)) {
DEBUG1(DBG_MAP_WIN, dip, "iommu_map_pages: redzone pg=%x\n",
pg_index);
ASSERT(TTE_IS_INVALID(iommu_p->iommu_tsb_vaddr[pg_index]));
}
#endif
if (DVMA_DBG_ON(iommu_p))
pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping,
mp->dmai_size, mp);
}
/*
* iommu_map_window - map a dvma window into the iommu
*
* used by: pci_dma_win(), pci_dma_ctlops() - DDI_DMA_MOVWIN
*
* return value: none
*/
/*ARGSUSED*/
void
iommu_map_window(iommu_t *iommu_p, ddi_dma_impl_t *mp, window_t win_no)
{
uint32_t obj_pg0_off = mp->dmai_roffset;
uint32_t win_pg0_off = win_no ? 0 : obj_pg0_off;
size_t win_size = mp->dmai_winsize;
size_t pfn_index = win_size * win_no; /* temp value */
size_t obj_off = win_no ? pfn_index - obj_pg0_off : 0; /* xferred sz */
dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping);
size_t res_size = mp->dmai_object.dmao_size - obj_off + win_pg0_off;
ASSERT(!(win_size & IOMMU_PAGE_OFFSET));
if (win_no >= mp->dmai_nwin)
return;
if (res_size < win_size) /* last window */
win_size = res_size; /* mp->dmai_winsize unchanged */
mp->dmai_mapping = IOMMU_PTOB(dvma_pg) | win_pg0_off;
mp->dmai_size = win_size - win_pg0_off; /* cur win xferrable size */
mp->dmai_offset = obj_off; /* win offset into object */
pfn_index = IOMMU_BTOP(pfn_index); /* index into pfnlist */
iommu_map_pages(iommu_p, mp, dvma_pg, IOMMU_BTOPR(win_size), pfn_index);
}
void
iommu_unmap_pages(iommu_t *iommu_p, dvma_addr_t dvma_pg, uint_t npages)
{
dvma_addr_t pg_index = IOMMU_PAGE_INDEX(iommu_p, dvma_pg);
for (; npages; npages--, dvma_pg++, pg_index++) {
DEBUG1(DBG_UNMAP_WIN|DBG_CONT, 0, " %x", dvma_pg);
IOMMU_UNLOAD_TTE(iommu_p, pg_index);
if (!tm_mtlb_gc)
IOMMU_PAGE_FLUSH(iommu_p, dvma_pg);
}
}
void
iommu_remap_pages(iommu_t *iommu_p, ddi_dma_impl_t *mp, dvma_addr_t dvma_pg,
size_t npages, size_t pfn_index)
{
iommu_unmap_pages(iommu_p, dvma_pg, npages);
iommu_map_pages(iommu_p, mp, dvma_pg, npages, pfn_index);
}
/*
* iommu_unmap_window
*
* This routine is called to break down the iommu mappings to a dvma window.
* Non partial mappings are viewed as single window mapping.
*
* used by: pci_dma_unbindhdl(), pci_dma_window(),
* and pci_dma_ctlops() - DDI_DMA_FREE, DDI_DMA_MOVWIN, DDI_DMA_NEXTWIN
*
* return value: none
*/
/*ARGSUSED*/
void
iommu_unmap_window(iommu_t *iommu_p, ddi_dma_impl_t *mp)
{
dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping);
dvma_addr_t pg_index = IOMMU_PAGE_INDEX(iommu_p, dvma_pg);
uint_t npages = IOMMU_BTOP(mp->dmai_winsize);
#ifdef DEBUG
dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
#endif
/*
* Invalidate each page of the mapping in the tsb and flush
* it from the tlb.
*/
DEBUG2(DBG_UNMAP_WIN, dip, "mp=%p %x pfns:", mp, npages);
if (mp->dmai_flags & DMAI_FLAGS_CONTEXT) {
dvma_context_t ctx = MP2CTX(mp);
for (; npages; npages--, pg_index++) {
DEBUG1(DBG_UNMAP_WIN|DBG_CONT, dip, " %x", pg_index);
IOMMU_UNLOAD_TTE(iommu_p, pg_index);
}
DEBUG1(DBG_UNMAP_WIN|DBG_CONT, dip, " (context %x)", ctx);
*iommu_p->iommu_flush_ctx_reg = ctx;
} else
iommu_unmap_pages(iommu_p, dvma_pg, npages);
DEBUG0(DBG_UNMAP_WIN|DBG_CONT, dip, "\n");
if (DVMA_DBG_ON(iommu_p))
pci_dvma_free_debug(iommu_p, (char *)mp->dmai_mapping,
mp->dmai_size, mp);
}
int
pci_alloc_tsb(pci_t *pci_p)
{
uint16_t tsbc;
if ((tsbc = iommu_tsb_alloc(pci_p->pci_id)) == IOMMU_TSB_COOKIE_NONE) {
cmn_err(CE_WARN, "%s%d: Unable to allocate IOMMU TSB.",
ddi_driver_name(pci_p->pci_dip),
ddi_get_instance(pci_p->pci_dip));
return (DDI_FAILURE);
}
pci_p->pci_tsb_cookie = tsbc;
return (DDI_SUCCESS);
}
void
pci_free_tsb(pci_t *pci_p)
{
iommu_tsb_free(pci_p->pci_tsb_cookie);
}
#if 0
/*
* The following data structure is used to map a tsb size
* to a tsb size configuration parameter in the iommu
* control register.
* This is a hardware table. It is here for reference only.
*/
static int pci_iommu_tsb_sizes[] = {
0x2000, /* 0 - 8 mb */
0x4000, /* 1 - 16 mb */
0x8000, /* 2 - 32 mb */
0x10000, /* 3 - 64 mb */
0x20000, /* 4 - 128 mb */
0x40000, /* 5 - 256 mb */
0x80000, /* 6 - 512 mb */
0x100000 /* 7 - 1 gb */
};
#endif
uint_t
iommu_tsb_size_encode(uint_t tsb_bytes)
{
uint_t i;
for (i = 7; i && (tsb_bytes < (0x2000 << i)); i--)
/* empty */;
return (i);
}
/*
* invalidate IOMMU TLB entries through diagnostic registers.
*/
static void
iommu_tlb_flushall(iommu_t *iommu_p)
{
int i;
uint64_t base = (uint64_t)(iommu_p->iommu_ctrl_reg) -
COMMON_IOMMU_CTRL_REG_OFFSET;
volatile uint64_t *tlb_tag = (volatile uint64_t *)
(base + COMMON_IOMMU_TLB_TAG_DIAG_ACC_OFFSET);
volatile uint64_t *tlb_data = (volatile uint64_t *)
(base + COMMON_IOMMU_TLB_DATA_DIAG_ACC_OFFSET);
for (i = 0; i < IOMMU_TLB_ENTRIES; i++)
tlb_tag[i] = tlb_data[i] = 0ull;
}
static void
iommu_preserve_tsb(iommu_t *iommu_p)
{
#ifdef DEBUG
dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip;
#endif
uint_t i, obp_tsb_entries, obp_tsb_size, base_pg_index;
uint64_t ctl = *iommu_p->iommu_ctrl_reg;
uint64_t obp_tsb_pa = *iommu_p->iommu_tsb_base_addr_reg;
uint64_t *base_tte_addr;
DEBUG3(DBG_ATTACH, dip,
"iommu_tsb_base_addr_reg=0x%08x (0x%08x.0x%08x)\n",
iommu_p->iommu_tsb_base_addr_reg,
(uint32_t)(*iommu_p->iommu_tsb_base_addr_reg >> 32),
(uint32_t)(*iommu_p->iommu_tsb_base_addr_reg & 0xffffffff));
obp_tsb_size = IOMMU_CTL_TO_TSBSIZE(ctl);
obp_tsb_entries = IOMMU_TSBSIZE_TO_TSBENTRIES(obp_tsb_size);
base_pg_index = iommu_p->dvma_end_pg - obp_tsb_entries + 1;
base_tte_addr = iommu_p->iommu_tsb_vaddr +
(iommu_p->iommu_tsb_entries - obp_tsb_entries);
/*
* old darwin prom does not set tsb size correctly, bail out.
*/
if ((obp_tsb_size == IOMMU_DARWIN_BOGUS_TSBSIZE) &&
(CHIP_TYPE(iommu_p->iommu_pci_p) == PCI_CHIP_SABRE))
return;
DEBUG3(DBG_ATTACH, dip, "iommu_preserve_tsb: kernel info\n"
"iommu_tsb_vaddr=%08x copy to base_tte_addr=%08x "
"base_pg_index=%x\n", iommu_p->iommu_tsb_vaddr,
base_tte_addr, base_pg_index);
DEBUG3(DBG_ATTACH | DBG_CONT, dip, "iommu_preserve_tsb: obp info "
"obp_tsb_entries=0x%x obp_tsb_pa=%08x.%08x\n", obp_tsb_entries,
(uint32_t)(obp_tsb_pa >> 32), (uint32_t)obp_tsb_pa);
for (i = 0; i < obp_tsb_entries; i++) {
uint64_t tte = lddphys(obp_tsb_pa + i * 8);
caddr_t va;
if (TTE_IS_INVALID(tte)) {
DEBUG0(DBG_ATTACH | DBG_CONT, dip, ".");
continue;
}
base_tte_addr[i] = tte;
DEBUG3(DBG_ATTACH | DBG_CONT, dip,
"\npreserve_tsb: (%x)=%08x.%08x\n", base_tte_addr + i,
(uint_t)(tte >> 32), (uint_t)(tte & 0xffffffff));
/*
* permanantly reserve this page from dvma address space
* resource map
*/
va = (caddr_t)(IOMMU_PTOB(base_pg_index + i));
(void) vmem_xalloc(iommu_p->iommu_dvma_map, IOMMU_PAGE_SIZE,
IOMMU_PAGE_SIZE, 0, 0, va, va + IOMMU_PAGE_SIZE,
VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
}
}