px_dma.c revision ef2504f26d1ea5859db9838255bb63f488f1b050
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* PCI Express nexus DVMA and DMA core routines:
* dma_map/dma_bind_handle implementation
* bypass and peer-to-peer support
* fast track DVMA space allocation
* runtime DVMA debug
*/
#include <sys/sysmacros.h>
#include <sys/ddi_impldefs.h>
#include "px_obj.h"
/*LINTLIBRARY*/
/*
* px_dma_allocmp - Allocate a pci dma implementation structure
*
* An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl
* to hold unmodified device limits. The ddi_dma_attr inside the
* ddi_dma_impl structure is augumented with system limits to enhance
* DVMA performance at runtime. The unaugumented device limits saved
* right after (accessed through (ddi_dma_attr_t *)(mp + 1)) is used
* strictly for peer-to-peer transfers which do not obey system limits.
*
* return: DDI_SUCCESS DDI_DMA_NORESOURCES
*/
{
register ddi_dma_impl_t *mp;
/* Caution: we don't use zalloc to enhance performance! */
if (waitfp != DDI_DMA_DONTWAIT) {
}
return (mp);
}
mp->dmai_flags = 0;
/*
* kmem_alloc debug: the following fields are not zero-ed
* mp->dmai_mapping = 0;
* mp->dmai_size = 0;
* mp->dmai_offset = 0;
* mp->dmai_minxfer = 0;
* mp->dmai_burstsizes = 0;
* mp->dmai_ndvmapages = 0;
* mp->dmai_rflags = 0;
* mp->dmai_inuse/flags
* mp->dmai_nwin = 0;
* mp->dmai_winsize = 0;
* mp->dmai_nexus_private/tte = 0;
* mp->dmai_iopte/pfnlst
* mp->dmai_minfo/winlst/fdvma
* mp->dmai_rdip
* bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t));
* bzero(&mp->dmai_attr, sizeof (ddi_dma_attr_t));
* mp->dmai_cookie = 0;
*/
mp->dmai_fault = 0;
/*
* The bdf protection value is set to immediate child
* as the code traverses down the fabric topology.
*
* XXX No IOMMU protection for broken devices.
*/
return (mp);
}
void
{
if (mp->dmai_winlst)
}
void
{
if (addr) {
if (npages > 1)
}
mp->dmai_ndvmapages = 0;
}
/*
* px_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits
* and convert dmareq->dmar_limits to mp->dmai_attr
*
* ddi_dma_impl_t member modified input
* ------------------------------------------------------------------------
* mp->dmai_minxfer - dev
* mp->dmai_burstsizes - dev
* mp->dmai_flags - no limit? peer-to-peer only?
*
* ddi_dma_attr member modified input
* ------------------------------------------------------------------------
* mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo
* mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi
* mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction)
* mp->dmai_attr.dma_attr_align - 1 (no alignment restriction)
*
* The dlim_dmaspeed member of dmareq->dmar_limits is ignored.
*/
{
return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING);
}
if (!count_max)
count_max--;
return (NULL);
/* store original dev input at the 2nd ddi_dma_attr */
else {
}
if (PX_DMA_NOCTX(rdip))
/* store augumented dev input to mp->dmai_attr */
return (mp);
}
/*
* Called from px_attach to check for bypass dma support and set
* flags accordingly.
*/
int
{
&baddr) != DDI_ENOTSUP)
/* ignore all other errors */
if (px_p->px_dma_sync_opt != 0)
return (DDI_SUCCESS);
}
/*
* px_dma_attr2hdl
*
* This routine is called from the alloc handle entry point to sanity check the
* dma attribute structure.
*
* use by: px_dma_allochdl()
*
* return value:
*
* DDI_SUCCESS - on success
* DDI_DMA_BADATTR - attribute has invalid version number
* or address limits exclude dvma space
*/
int
{
int ret;
if (!nocross)
nocross--;
/*
* If Bypass DMA is not supported, return error so that
* target driver can fall back to dvma mode of operation
*/
return (DDI_DMA_BADATTR);
if (nocross != UINT64_MAX)
return (DDI_DMA_BADATTR);
return (DDI_DMA_BADATTR);
/* do a range check and get the limits */
if (ret != DDI_SUCCESS)
return (ret);
} else { /* MMU_XLATE or PEER_TO_PEER */
return (DDI_DMA_BADATTR);
}
}
return (DDI_DMA_BADATTR);
}
if (!count_max)
count_max--;
/*
* If this is an IOMMU bypass access, the caller can't use
* the required addresses, so fail it. Otherwise, it's
* peer-to-peer; ensure that the caller has no alignment or
* segment size restrictions.
*/
return (DDI_DMA_BADATTR);
} else /* set practical counter_max value */
else {
}
return (DDI_SUCCESS);
}
/*
* px_dma_type - determine which of the three types DMA (peer-to-peer,
* mmu bypass, or mmu translate) we are asked to do.
* Also checks pfn0 and rejects any non-peer-to-peer
* requests for peer-only devices.
*
* return values:
* DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type
* DDI_SUCCESS
*
* dma handle members affected (set on exit):
* mp->dmai_object - dmareq->dmar_object
* mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags
* mp->dmai_flags - DMA type
* mp->dmai_roffset - initialized to starting MMU page offset
* mp->dmai_ndvmapages - # of total MMU pages of entire object
*/
int
{
case DMA_OTYP_BUFVADDR:
case DMA_OTYP_VADDR: {
if (pplist) { /* shadow list */
} else {
}
}
break;
case DMA_OTYP_PAGES:
break;
case DMA_OTYP_PADDR:
default:
return (DDI_DMA_NOMAPPING);
}
if (pfn0 == PFN_INVALID) {
return (DDI_DMA_NOMAPPING);
}
pec_p->pec_last32_pfn)) {
goto done; /* leave bypass and dvma flag as 0 */
pec_p->pec_last64_pfn)) {
goto done; /* leave bypass and dvma flag as 0 */
}
if (PX_DMA_ISPEERONLY(mp)) {
return (DDI_DMA_NOMAPPING);
}
done:
return (DDI_SUCCESS);
}
/*
* px_dma_pgpfn - set up pfnlst array according to pages
*/
/*ARGSUSED*/
static int
{
int i;
case DMA_OTYP_BUFVADDR:
case DMA_OTYP_VADDR: {
for (i = 1; i < npages; i++) {
}
}
break;
case DMA_OTYP_PAGES: {
}
}
break;
default: /* check is already done by px_dma_type */
ASSERT(0);
break;
}
return (DDI_SUCCESS);
}
/*
* px_dma_vapfn - set up pfnlst array according to VA
* pfn0 is skipped as it is already done.
* In this case, the cached pfn0 is used to fill pfnlst[0]
*/
static int
{
int i;
if (pfn == PFN_INVALID)
goto err_badpfn;
}
return (DDI_SUCCESS);
return (DDI_DMA_NOMAPPING);
}
/*
* px_dma_pfn - Fills pfn list for all pages being DMA-ed.
*
* dependencies:
* mp->dmai_ndvmapages - set to total # of dma pages
*
* return value:
* DDI_SUCCESS
* DDI_DMA_NOMAPPING
*/
int
{
if (npages == 1) {
return (DDI_SUCCESS);
}
/* allocate pfn array */
if (waitfp != DDI_DMA_DONTWAIT)
&px_kmem_clid);
return (DDI_DMA_NORESOURCES);
}
/* fill pfn array */
goto err;
/* skip pfn0, check mixed mode and adjust peer to peer pfn */
for (i = 1; i < npages; i++) {
goto err;
}
if (pfn_adj)
}
return (DDI_SUCCESS);
err:
return (ret);
}
/*
* px_dvma_win() - trim requested DVMA size down to window size
* The 1st window starts from offset and ends at page-aligned boundary.
* From the 2nd window on, each window starts and ends at page-aligned
* boundary except the last window ends at wherever requested.
*
* accesses the following mp-> members:
* mp->dmai_attr.dma_attr_count_max
* mp->dmai_attr.dma_attr_seg
* mp->dmai_roffset - start offset of 1st window
* mp->dmai_rflags (redzone)
* mp->dmai_ndvmapages (for 1 page fast path)
*
* sets the following mp-> members:
* mp->dmai_winsize - window size (no redzone), n * page size (fixed)
* mp->dmai_nwin - # of DMA windows of entire object (fixed)
* mp->dmai_rflags - remove partial flag if nwin == 1 (fixed)
* mp->dmai_winlst - NULL, window objects not used for DVMA (fixed)
*
* fixed - not changed across different DMA windows
*/
/*ARGSUSED*/
int
{
goto done;
}
/* include redzone in nocross check */ {
"nocross too small: "
"%lx(%lx)+%lx+%lx < %llx\n",
return (DDI_DMA_TOOBIG);
}
}
/* check counter max */ {
}
goto done;
}
return (DDI_DMA_TOOBIG);
}
done:
return (DDI_SUCCESS);
}
/*
* fast track cache entry to mmu context, inserts 3 0 bits between
* upper 6-bits and lower 3-bits of the 9-bit cache entry
*/
/*
* px_dvma_map_fast - attempts to map fast trackable DVMA
*/
/*ARGSUSED*/
int
{
int i = mmu_p->mmu_dvma_addr_scan_start;
;
if (i >= entries) {
i = 0;
;
if (i >= entries) {
#ifdef PX_DMA_PROF
#endif /* PX_DMA_PROF */
return (DDI_DMA_NORESOURCES);
}
}
i *= clustsz;
MMU_MAP_PFN) != DDI_SUCCESS) {
"px_lib_iommu_map failed\n");
return (DDI_FAILURE);
}
if (!PX_MAP_BUFZONE(mp))
goto done;
MMU_MAP_PFN) != DDI_SUCCESS) {
"mapping REDZONE page failed\n");
return (DDI_FAILURE);
}
done:
#ifdef PX_DMA_PROF
#endif
mp->dmai_offset = 0;
if (PX_DVMA_DBG_ON(mmu_p))
return (DDI_SUCCESS);
}
/*
* px_dvma_map: map non-fasttrack DMA
* Use quantum cache if single page DMA.
*/
int
{
void *dvma_addr;
int ret = DDI_SUCCESS;
/*
* allocate dvma space resource and map in the first window.
* (vmem_t *vmp, size_t size,
* size_t align, size_t phase, size_t nocross,
* void *minaddr, void *maxaddr, int vmflag)
*/
#ifdef PX_DMA_PROF
#endif /* PX_DMA_PROF */
} else {
0,
sleep);
#ifdef PX_DMA_PROF
#endif /* PX_DMA_PROF */
}
if (dvma_pg == 0)
goto noresource;
mp->dmai_offset = 0;
#ifdef PX_DMA_PROF
#endif /* PX_DMA_PROF */
} else {
#ifdef PX_DMA_PROF
#endif /* PX_DMA_PROF */
}
}
return (ret);
&mmu_p->mmu_dvma_clid);
}
return (DDI_DMA_NORESOURCES);
}
void
{
#ifdef PX_DMA_PROF
#endif /* PX_DMA_PROF */
return;
}
#ifdef PX_DMA_PROF
#endif /* PX_DMA_PROF */
} else {
#ifdef PX_DMA_PROF
#endif /* PX_DMA_PROF */
}
}
/*
* DVMA mappings may have multiple windows, but each window always have
* one segment.
*/
int
{
switch (cmd) {
case DDI_DMA_SYNC:
case DDI_DMA_HTOC: {
int ret;
return (DDI_FAILURE);
}
if (ret)
return (ret);
}
return (DDI_SUCCESS);
case DDI_DMA_REPWIN:
return (DDI_SUCCESS);
case DDI_DMA_MOVWIN: {
return (DDI_FAILURE);
}
case DDI_DMA_NEXTWIN: {
if (offp) {
/* window not active */
return (DDI_DMA_STALE);
}
win++;
} else /* map win 0 */
win = 0;
return (DDI_DMA_DONE);
}
win, 0, 0, 0, 0)) {
return (DDI_FAILURE);
}
}
return (DDI_SUCCESS);
case DDI_DMA_NEXTSEG:
return (DDI_DMA_STALE);
if (lenp) /* only 1 seg allowed */
return (DDI_DMA_DONE);
/* return mp as seg 0 */
return (DDI_SUCCESS);
case DDI_DMA_SEGTOC:
return (DDI_SUCCESS);
case DDI_DMA_COFF: {
return (DDI_FAILURE);
mp->dmai_offset);
}
return (DDI_SUCCESS);
default:
break;
}
return (DDI_FAILURE);
}
void
{
}
}
/*
* px_dma_newwin - create a dma window object and cookies
*
* After the initial scan in px_dma_physwin(), which identifies
* a portion of the pfn array that belongs to a dma window,
* we are called to allocate and initialize representing memory
* resources. We know from the 1st scan the number of cookies
* or dma segment in this window so we can allocate a contiguous
* memory array for the dma cookies (The implementation of
* ddi_dma_nextcookie(9f) dictates dma cookies be contiguous).
*
* A second round scan is done on the pfn array to identify
* each dma segment and initialize its corresponding dma cookie.
* We don't need to do all the safety checking and we know they
* all belong to the same dma window.
*
* Input: cookie_no - # of cookies identified by the 1st scan
* start_idx - subscript of the pfn array for the starting pfn
* end_idx - subscript of the last pfn in dma window
* win_pp - pointer to win_next member of previous window
* Return: DDI_SUCCESS - with **win_pp as newly created window object
* DDI_DMA_NORESROUCE - caller frees all previous window objs
* Note: Each cookie and window size are all initialized on page
* boundary. This is not true for the 1st cookie of the 1st
* window and the last cookie of the last window.
* We fix that later in upper layer which has access to size
* and offset info.
*
*/
/*ARGSUSED*/
static int
{
if (!win_p)
goto noresource;
/* win_p->win_offset is left uninitialized */
start_idx++;
continue;
/* close up the cookie up to (including) prev_pfn */
if (bypass) {
== DDI_SUCCESS)
else
return (DDI_FAILURE);
}
cookie_p++; /* advance to next available cookie cell */
pfn_no = 0;
}
if (bypass) {
== DDI_SUCCESS)
else
return (DDI_FAILURE);
}
#ifdef DEBUG
cookie_p++;
#endif /* DEBUG */
return (DDI_SUCCESS);
if (waitfp != DDI_DMA_DONTWAIT)
return (DDI_DMA_NORESOURCES);
}
/*
* px_dma_adjust - adjust 1st and last cookie and window sizes
* remove initial dma page offset from 1st cookie and window size
* remove last dma page remainder from last cookie and window size
* fill win_offset of each dma window according to just fixed up
* each window sizes
* px_dma_win_t members modified:
* win_p->win_offset - this window's offset within entire DMA object
* win_p->win_size - xferrable size (in bytes) for this window
*
* ddi_dma_impl_t members modified:
* mp->dmai_size - 1st window xferrable size
* mp->dmai_offset - 0, which is the dma offset of the 1st window
*
* ddi_dma_cookie_t members modified:
* cookie_p->dmac_size - 1st and last cookie remove offset or remainder
* cookie_p->dmac_laddress - 1st cookie add page offset
*/
static void
{
size_t win_offset = 0;
mp->dmai_offset = 0;
if (pg_offset)
}
/* last window */
}
/*
* px_dma_physwin() - carve up dma windows using physical addresses.
* Called to handle mmu bypass and pci peer-to-peer transfers.
* Calls px_dma_newwin() to allocate window objects.
*
* Dependency: mp->dmai_pfnlst points to an array of pfns
*
* 1. Each dma window is represented by a px_dma_win_t object.
* The object will be casted to ddi_dma_win_t and returned
* to leaf driver through the DDI interface.
* 2. Each dma window can have several dma segments with each
* segment representing a physically contiguous either memory
* space (if we are doing an mmu bypass transfer) or pci address
* space (if we are doing a peer-to-peer transfer).
* 3. Each segment has a DMA cookie to program the DMA engine.
* The cookies within each DMA window must be located in a
* contiguous array per ddi_dma_nextcookie(9f).
* 4. The number of DMA segments within each DMA window cannot exceed
* mp->dmai_attr.dma_attr_sgllen. If the transfer size is
* too large to fit in the sgllen, the rest needs to be
* relocated to the next dma window.
* 5. Peer-to-peer DMA segment follows device hi, lo, count_max,
* and nocross restrictions while bypass DMA follows the set of
* restrictions with system limits factored in.
*
* Return:
* mp->dmai_winlst - points to a link list of px_dma_win_t objects.
* Each px_dma_win_t object on the link list contains
* infomation such as its window size (# of pages),
* starting offset (also see Restriction), an array of
* DMA cookies, and # of cookies in the array.
* mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory.
* mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst.
* mp->dmai_mapping - starting cookie address
* mp->dmai_rflags - consistent, nosync, no redzone
* mp->dmai_cookie - start of cookie table of the 1st DMA window
*
* Restriction:
* Each px_dma_win_t object can theoratically start from any offset
* since the mmu is not involved. However, this implementation
* always make windows start from page aligned offset (except
* the 1st window, which follows the requested offset) due to the
* fact that we are handed a pfn list. This does require device's
* count_max and attr_seg to be at least MMU_PAGE_SIZE aligned.
*/
int
{
return (DDI_DMA_NOMAPPING);
return (DDI_DMA_NOMAPPING);
return (DDI_DMA_NOMAPPING);
/*
* the following count_max trim is not done because we are
* making sure pfn_lo <= pfn <= pfn_hi inside the loop
* count_max=MIN(count_max, MMU_PTOB(pfn_hi - pfn_lo + 1)-1);
*/
"bypass cookie failure %lx\n", pfn);
return (DDI_DMA_NOMAPPING);
}
}
/* pfn: absolute (bypass mode) or relative (p2p mode) */
if (bypass_addr) {
&bypass_addr) != DDI_SUCCESS) {
goto err;
}
}
continue;
goto err;
}
cookie_no++;
pfn_no = 0;
continue;
goto err;
win_no++;
win_pfn0_index = i;
cookie_no = 0;
}
goto err;
}
cookie_no++;
goto err;
win_no++;
return (DDI_DMA_MAPPED);
err:
return (ret);
}
int
{
switch (cmd) {
case DDI_DMA_SYNC:
return (DDI_SUCCESS);
case DDI_DMA_HTOC: {
return (DDI_FAILURE);
/* locate window */
"HTOC: cookie - dmac_laddress=%p dmac_size=%x\n",
}
return (DDI_SUCCESS);
case DDI_DMA_REPWIN:
return (DDI_SUCCESS);
case DDI_DMA_MOVWIN: {
return (DDI_FAILURE);
/* locate window */
win_p->win_curseg = 0;
"HTOC: cookie - dmac_laddress=%p dmac_size=%x\n",
}
return (DDI_SUCCESS);
case DDI_DMA_NEXTWIN: {
if (!win_p) {
return (DDI_SUCCESS);
}
return (DDI_DMA_STALE);
return (DDI_DMA_DONE);
win_p->win_curseg = 0;
}
return (DDI_SUCCESS);
case DDI_DMA_NEXTSEG: {
return (DDI_DMA_STALE);
return (DDI_DMA_DONE);
w_p->win_curseg++;
}
return (DDI_SUCCESS);
case DDI_DMA_SEGTOC: {
int i;
/* locate active window */
;
}
return (DDI_SUCCESS);
case DDI_DMA_COFF: {
int i;
win_off = 0;
goto found;
}
}
return (DDI_FAILURE);
return (DDI_SUCCESS);
}
default:
break;
}
return (DDI_FAILURE);
}
static void
{
mmu_p->dvma_alloc_rec_index = 0;
mmu_p->dvma_free_rec_index = 0;
mmu_p->dvma_active_count = 0;
}
void
{
if (mmu_p->dvma_alloc_rec) {
}
if (mmu_p->dvma_free_rec) {
}
if (!prev)
return;
mmu_p->dvma_alloc_rec_index = 0;
mmu_p->dvma_free_rec_index = 0;
mmu_p->dvma_active_count = 0;
px_dvma_debug_on &= mask;
}
void
{
struct px_dvma_rec *ptr;
if (!mmu_p->dvma_alloc_rec)
if (PX_DVMA_DBG_OFF(mmu_p)) {
goto done;
}
mmu_p->dvma_alloc_rec_index = 0;
done:
}
void
{
if (!mmu_p->dvma_alloc_rec)
if (PX_DVMA_DBG_OFF(mmu_p)) {
goto done;
}
mmu_p->dvma_free_rec_index = 0;
break;
}
if (!ptr) {
goto done;
}
else
done:
}
#ifdef DEBUG
void
{
hp->dmai_cookie);
}
#endif /* DEBUG */