rootnex.c revision 86a9c507121ec77a92601d8844a5ca82373cd4aa
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2010 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* x86 root nexus driver
*/
#include <sys/sysmacros.h>
#include <sys/conf.h>
#include <sys/autoconf.h>
#include <sys/sysmacros.h>
#include <sys/debug.h>
#include <sys/psw.h>
#include <sys/ddidmareq.h>
#include <sys/promif.h>
#include <sys/devops.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <vm/seg.h>
#include <vm/seg_kmem.h>
#include <vm/seg_dev.h>
#include <sys/vmem.h>
#include <sys/mman.h>
#include <vm/hat.h>
#include <vm/as.h>
#include <vm/page.h>
#include <sys/avintr.h>
#include <sys/errno.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/mach_intr.h>
#include <sys/psm.h>
#include <sys/ontrap.h>
#include <sys/atomic.h>
#include <sys/sdt.h>
#include <sys/rootnex.h>
#include <vm/hat_i86.h>
#include <sys/ddifm.h>
#include <sys/ddi_isa.h>
#ifdef __xpv
#include <sys/bootinfo.h>
#include <sys/hypervisor.h>
#include <sys/bootconf.h>
#include <vm/kboot_mmu.h>
#else
#include <sys/intel_iommu.h>
#endif
/*
* enable/disable extra checking of function parameters. Useful for debugging
* drivers.
*/
#ifdef DEBUG
int rootnex_alloc_check_parms = 1;
int rootnex_bind_check_parms = 1;
int rootnex_bind_check_inuse = 1;
int rootnex_unbind_verify_buffer = 0;
int rootnex_sync_check_parms = 1;
#else
int rootnex_alloc_check_parms = 0;
int rootnex_bind_check_parms = 0;
int rootnex_bind_check_inuse = 0;
int rootnex_unbind_verify_buffer = 0;
int rootnex_sync_check_parms = 0;
#endif
/* Master Abort and Target Abort panic flag */
int rootnex_fm_ma_ta_panic_flag = 0;
/* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
int rootnex_bind_fail = 1;
int rootnex_bind_warn = 1;
uint8_t *rootnex_warn_list;
/* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
#define ROOTNEX_BIND_WARNING (0x1 << 0)
/*
* revert back to old broken behavior of always sync'ing entire copy buffer.
* This is useful if be have a buggy driver which doesn't correctly pass in
* the offset and size into ddi_dma_sync().
*/
int rootnex_sync_ignore_params = 0;
/*
* For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
* page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
* 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
* kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
* sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
* (< 8K). We will still need to allocate the copy buffer during bind though
* (if we need one). These can only be modified in /etc/system before rootnex
* attach.
*/
#if defined(__amd64)
int rootnex_prealloc_cookies = 65;
int rootnex_prealloc_windows = 4;
int rootnex_prealloc_copybuf = 2;
#else
int rootnex_prealloc_cookies = 33;
int rootnex_prealloc_windows = 4;
int rootnex_prealloc_copybuf = 2;
#endif
/* driver global state */
static rootnex_state_t *rootnex_state;
/* shortcut to rootnex counters */
static uint64_t *rootnex_cnt;
/*
* XXX - does x86 even need these or are they left over from the SPARC days?
*/
/* statically defined integer/boolean properties for the root node */
static rootnex_intprop_t rootnex_intprp[] = {
{ "PAGESIZE", PAGESIZE },
{ "MMU_PAGESIZE", MMU_PAGESIZE },
{ "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
{ DDI_RELATIVE_ADDRESSING, 1 },
};
#define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
#ifdef __xpv
typedef maddr_t rootnex_addr_t;
#define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \
(DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa))
#else
typedef paddr_t rootnex_addr_t;
#endif
#if !defined(__xpv)
char _depends_on[] = "mach/pcplusmp misc/iommulib misc/acpica";
#endif
static struct cb_ops rootnex_cb_ops = {
nodev, /* open */
nodev, /* close */
nodev, /* strategy */
nodev, /* print */
nodev, /* dump */
nodev, /* read */
nodev, /* write */
nodev, /* ioctl */
nodev, /* devmap */
nodev, /* mmap */
nodev, /* segmap */
nochpoll, /* chpoll */
ddi_prop_op, /* cb_prop_op */
NULL, /* struct streamtab */
D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
CB_REV, /* Rev */
nodev, /* cb_aread */
nodev /* cb_awrite */
};
static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
off_t offset, off_t len, caddr_t *vaddrp);
static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
struct hat *hat, struct seg *seg, caddr_t addr,
struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip,
struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep);
static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
ddi_dma_handle_t *handlep);
static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle);
static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
ddi_dma_cookie_t *cookiep, uint_t *ccountp);
static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle);
static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
ddi_dma_cookie_t *cookiep, uint_t *ccountp);
static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
ddi_ctl_enum_t ctlop, void *arg, void *result);
static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
ddi_iblock_cookie_t *ibc);
static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
ddi_dma_handle_t *handlep);
static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle);
static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
ddi_dma_cookie_t *cookiep, uint_t *ccountp);
static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle);
#if !defined(__xpv)
static void rootnex_coredma_reset_cookies(dev_info_t *dip,
ddi_dma_handle_t handle);
static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
ddi_dma_cookie_t *cookiep, uint_t ccount);
static int rootnex_coredma_clear_cookies(dev_info_t *dip,
ddi_dma_handle_t handle);
static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
#endif
static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
ddi_dma_cookie_t *cookiep, uint_t *ccountp);
static struct bus_ops rootnex_bus_ops = {
BUSO_REV,
rootnex_map,
NULL,
NULL,
NULL,
rootnex_map_fault,
rootnex_dma_map,
rootnex_dma_allochdl,
rootnex_dma_freehdl,
rootnex_dma_bindhdl,
rootnex_dma_unbindhdl,
rootnex_dma_sync,
rootnex_dma_win,
rootnex_dma_mctl,
rootnex_ctlops,
ddi_bus_prop_op,
i_ddi_rootnex_get_eventcookie,
i_ddi_rootnex_add_eventcall,
i_ddi_rootnex_remove_eventcall,
i_ddi_rootnex_post_event,
0, /* bus_intr_ctl */
0, /* bus_config */
0, /* bus_unconfig */
rootnex_fm_init, /* bus_fm_init */
NULL, /* bus_fm_fini */
NULL, /* bus_fm_access_enter */
NULL, /* bus_fm_access_exit */
NULL, /* bus_powr */
rootnex_intr_ops /* bus_intr_op */
};
static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
static struct dev_ops rootnex_ops = {
DEVO_REV,
0,
ddi_no_info,
nulldev,
nulldev,
rootnex_attach,
rootnex_detach,
nulldev,
&rootnex_cb_ops,
&rootnex_bus_ops,
NULL,
ddi_quiesce_not_needed, /* quiesce */
};
static struct modldrv rootnex_modldrv = {
&mod_driverops,
"i86pc root nexus",
&rootnex_ops
};
static struct modlinkage rootnex_modlinkage = {
MODREV_1,
(void *)&rootnex_modldrv,
NULL
};
#if !defined(__xpv)
static iommulib_nexops_t iommulib_nexops = {
IOMMU_NEXOPS_VERSION,
"Rootnex IOMMU ops Vers 1.1",
NULL,
rootnex_coredma_allochdl,
rootnex_coredma_freehdl,
rootnex_coredma_bindhdl,
rootnex_coredma_unbindhdl,
rootnex_coredma_reset_cookies,
rootnex_coredma_get_cookies,
rootnex_coredma_set_cookies,
rootnex_coredma_clear_cookies,
rootnex_coredma_get_sleep_flags,
rootnex_coredma_sync,
rootnex_coredma_win,
rootnex_dma_map,
rootnex_dma_mctl
};
#endif
/*
* extern hacks
*/
extern struct seg_ops segdev_ops;
extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
#ifdef DDI_MAP_DEBUG
extern int ddi_map_debug_flag;
#define ddi_map_debug if (ddi_map_debug_flag) prom_printf
#endif
extern void i86_pp_map(page_t *pp, caddr_t kaddr);
extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
psm_intr_op_t, int *);
extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
/*
* Use device arena to use for device control register mappings.
* Various kernel memory walkers (debugger, dtrace) need to know
* to avoid this address range to prevent undesired device activity.
*/
extern void *device_arena_alloc(size_t size, int vm_flag);
extern void device_arena_free(void * vaddr, size_t size);
/*
* Internal functions
*/
static int rootnex_dma_init();
static void rootnex_add_props(dev_info_t *);
static int rootnex_ctl_reportdev(dev_info_t *dip);
static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
static int rootnex_map_handle(ddi_map_req_t *mp);
static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
ddi_dma_attr_t *attr);
static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
rootnex_sglinfo_t *sglinfo);
static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag);
static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
rootnex_dma_t *dma, ddi_dma_attr_t *attr);
static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
ddi_dma_attr_t *attr, int kmflag);
static void rootnex_teardown_windows(rootnex_dma_t *dma);
static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
size_t *copybuf_used, page_t **cur_pp);
static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
ddi_dma_attr_t *attr, off_t cur_offset);
static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
rootnex_dma_t *dma, rootnex_window_t **windowp,
ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
off_t offset, size_t size, uint_t cache_flags);
static int rootnex_verify_buffer(rootnex_dma_t *dma);
static int rootnex_dma_check(dev_info_t *dip, const void *handle,
const void *comp_addr, const void *not_used);
/*
* _init()
*
*/
int
_init(void)
{
rootnex_state = NULL;
return (mod_install(&rootnex_modlinkage));
}
/*
* _info()
*
*/
int
_info(struct modinfo *modinfop)
{
return (mod_info(&rootnex_modlinkage, modinfop));
}
/*
* _fini()
*
*/
int
_fini(void)
{
return (EBUSY);
}
/*
* rootnex_attach()
*
*/
static int
rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
int fmcap;
int e;
switch (cmd) {
case DDI_ATTACH:
break;
case DDI_RESUME:
return (DDI_SUCCESS);
default:
return (DDI_FAILURE);
}
/*
* We should only have one instance of rootnex. Save it away since we
* don't have an easy way to get it back later.
*/
ASSERT(rootnex_state == NULL);
rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
rootnex_state->r_dip = dip;
rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
rootnex_state->r_reserved_msg_printed = B_FALSE;
rootnex_cnt = &rootnex_state->r_counters[0];
rootnex_state->r_intel_iommu_enabled = B_FALSE;
/*
* Set minimum fm capability level for i86pc platforms and then
* initialize error handling. Since we're the rootnex, we don't
* care what's returned in the fmcap field.
*/
ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
fmcap = ddi_system_fmcap;
ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
/* initialize DMA related state */
e = rootnex_dma_init();
if (e != DDI_SUCCESS) {
kmem_free(rootnex_state, sizeof (rootnex_state_t));
return (DDI_FAILURE);
}
/* Add static root node properties */
rootnex_add_props(dip);
/* since we can't call ddi_report_dev() */
cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
/* Initialize rootnex event handle */
i_ddi_rootnex_init_events(dip);
#if !defined(__xpv)
#if defined(__amd64)
/* probe intel iommu */
intel_iommu_probe_and_parse();
/* attach the iommu nodes */
if (intel_iommu_support) {
if (intel_iommu_attach_dmar_nodes() == DDI_SUCCESS) {
rootnex_state->r_intel_iommu_enabled = B_TRUE;
} else {
intel_iommu_release_dmar_info();
}
}
#endif
e = iommulib_nexus_register(dip, &iommulib_nexops,
&rootnex_state->r_iommulib_handle);
ASSERT(e == DDI_SUCCESS);
#endif
return (DDI_SUCCESS);
}
/*
* rootnex_detach()
*
*/
/*ARGSUSED*/
static int
rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
switch (cmd) {
case DDI_SUSPEND:
break;
default:
return (DDI_FAILURE);
}
return (DDI_SUCCESS);
}
/*
* rootnex_dma_init()
*
*/
/*ARGSUSED*/
static int
rootnex_dma_init()
{
size_t bufsize;
/*
* size of our cookie/window/copybuf state needed in dma bind that we
* pre-alloc in dma_alloc_handle
*/
rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
rootnex_state->r_prealloc_size =
(rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
(rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
(rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
/*
* setup DDI DMA handle kmem cache, align each handle on 64 bytes,
* allocate 16 extra bytes for struct pointer alignment
* (p->dmai_private & dma->dp_prealloc_buffer)
*/
bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
rootnex_state->r_prealloc_size + 0x10;
rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
if (rootnex_state->r_dmahdl_cache == NULL) {
return (DDI_FAILURE);
}
/*
* allocate array to track which major numbers we have printed warnings
* for.
*/
rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
KM_SLEEP);
return (DDI_SUCCESS);
}
/*
* rootnex_add_props()
*
*/
static void
rootnex_add_props(dev_info_t *dip)
{
rootnex_intprop_t *rpp;
int i;
/* Add static integer/boolean properties to the root node */
rpp = rootnex_intprp;
for (i = 0; i < NROOT_INTPROPS; i++) {
(void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
rpp[i].prop_name, rpp[i].prop_value);
}
}
/*
* *************************
* ctlops related routines
* *************************
*/
/*
* rootnex_ctlops()
*
*/
/*ARGSUSED*/
static int
rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
void *arg, void *result)
{
int n, *ptr;
struct ddi_parent_private_data *pdp;
switch (ctlop) {
case DDI_CTLOPS_DMAPMAPC:
/*
* Return 'partial' to indicate that dma mapping
* has to be done in the main MMU.
*/
return (DDI_DMA_PARTIAL);
case DDI_CTLOPS_BTOP:
/*
* Convert byte count input to physical page units.
* (byte counts that are not a page-size multiple
* are rounded down)
*/
*(ulong_t *)result = btop(*(ulong_t *)arg);
return (DDI_SUCCESS);
case DDI_CTLOPS_PTOB:
/*
* Convert size in physical pages to bytes
*/
*(ulong_t *)result = ptob(*(ulong_t *)arg);
return (DDI_SUCCESS);
case DDI_CTLOPS_BTOPR:
/*
* Convert byte count input to physical page units
* (byte counts that are not a page-size multiple
* are rounded up)
*/
*(ulong_t *)result = btopr(*(ulong_t *)arg);
return (DDI_SUCCESS);
case DDI_CTLOPS_INITCHILD:
return (impl_ddi_sunbus_initchild(arg));
case DDI_CTLOPS_UNINITCHILD:
impl_ddi_sunbus_removechild(arg);
return (DDI_SUCCESS);
case DDI_CTLOPS_REPORTDEV:
return (rootnex_ctl_reportdev(rdip));
case DDI_CTLOPS_IOMIN:
/*
* Nothing to do here but reflect back..
*/
return (DDI_SUCCESS);
case DDI_CTLOPS_REGSIZE:
case DDI_CTLOPS_NREGS:
break;
case DDI_CTLOPS_SIDDEV:
if (ndi_dev_is_prom_node(rdip))
return (DDI_SUCCESS);
if (ndi_dev_is_persistent_node(rdip))
return (DDI_SUCCESS);
return (DDI_FAILURE);
case DDI_CTLOPS_POWER:
return ((*pm_platform_power)((power_req_t *)arg));
case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
if (!rootnex_state->r_reserved_msg_printed) {
rootnex_state->r_reserved_msg_printed = B_TRUE;
cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
"1 or more reserved/obsolete operations.");
}
return (DDI_FAILURE);
default:
return (DDI_FAILURE);
}
/*
* The rest are for "hardware" properties
*/
if ((pdp = ddi_get_parent_data(rdip)) == NULL)
return (DDI_FAILURE);
if (ctlop == DDI_CTLOPS_NREGS) {
ptr = (int *)result;
*ptr = pdp->par_nreg;
} else {
off_t *size = (off_t *)result;
ptr = (int *)arg;
n = *ptr;
if (n >= pdp->par_nreg) {
return (DDI_FAILURE);
}
*size = (off_t)pdp->par_reg[n].regspec_size;
}
return (DDI_SUCCESS);
}
/*
* rootnex_ctl_reportdev()
*
*/
static int
rootnex_ctl_reportdev(dev_info_t *dev)
{
int i, n, len, f_len = 0;
char *buf;
buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
f_len += snprintf(buf, REPORTDEV_BUFSIZE,
"%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
len = strlen(buf);
for (i = 0; i < sparc_pd_getnreg(dev); i++) {
struct regspec *rp = sparc_pd_getreg(dev, i);
if (i == 0)
f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
": ");
else
f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
" and ");
len = strlen(buf);
switch (rp->regspec_bustype) {
case BTEISA:
f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
"%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
break;
case BTISA:
f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
"%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
break;
default:
f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
"space %x offset %x",
rp->regspec_bustype, rp->regspec_addr);
break;
}
len = strlen(buf);
}
for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
int pri;
if (i != 0) {
f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
",");
len = strlen(buf);
}
pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
" sparc ipl %d", pri);
len = strlen(buf);
}
#ifdef DEBUG
if (f_len + 1 >= REPORTDEV_BUFSIZE) {
cmn_err(CE_NOTE, "next message is truncated: "
"printed length 1024, real length %d", f_len);
}
#endif /* DEBUG */
cmn_err(CE_CONT, "?%s\n", buf);
kmem_free(buf, REPORTDEV_BUFSIZE);
return (DDI_SUCCESS);
}
/*
* ******************
* map related code
* ******************
*/
/*
* rootnex_map()
*
*/
static int
rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
off_t len, caddr_t *vaddrp)
{
struct regspec *rp, tmp_reg;
ddi_map_req_t mr = *mp; /* Get private copy of request */
int error;
mp = &mr;
switch (mp->map_op) {
case DDI_MO_MAP_LOCKED:
case DDI_MO_UNMAP:
case DDI_MO_MAP_HANDLE:
break;
default:
#ifdef DDI_MAP_DEBUG
cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
mp->map_op);
#endif /* DDI_MAP_DEBUG */
return (DDI_ME_UNIMPLEMENTED);
}
if (mp->map_flags & DDI_MF_USER_MAPPING) {
#ifdef DDI_MAP_DEBUG
cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
#endif /* DDI_MAP_DEBUG */
return (DDI_ME_UNIMPLEMENTED);
}
/*
* First, if given an rnumber, convert it to a regspec...
* (Presumably, this is on behalf of a child of the root node?)
*/
if (mp->map_type == DDI_MT_RNUMBER) {
int rnumber = mp->map_obj.rnumber;
#ifdef DDI_MAP_DEBUG
static char *out_of_range =
"rootnex_map: Out of range rnumber <%d>, device <%s>";
#endif /* DDI_MAP_DEBUG */
rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
if (rp == NULL) {
#ifdef DDI_MAP_DEBUG
cmn_err(CE_WARN, out_of_range, rnumber,
ddi_get_name(rdip));
#endif /* DDI_MAP_DEBUG */
return (DDI_ME_RNUMBER_RANGE);
}
/*
* Convert the given ddi_map_req_t from rnumber to regspec...
*/
mp->map_type = DDI_MT_REGSPEC;
mp->map_obj.rp = rp;
}
/*
* Adjust offset and length correspnding to called values...
* XXX: A non-zero length means override the one in the regspec
* XXX: (regardless of what's in the parent's range?)
*/
tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */
rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */
#ifdef DDI_MAP_DEBUG
cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
"handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
len, mp->map_handlep);
#endif /* DDI_MAP_DEBUG */
/*
* I/O or memory mapping:
*
* <bustype=0, addr=x, len=x>: memory
* <bustype=1, addr=x, len=x>: i/o
* <bustype>1, addr=0, len=x>: x86-compatibility i/o
*/
if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
cmn_err(CE_WARN, "<%s,%s> invalid register spec"
" <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
ddi_get_name(rdip), rp->regspec_bustype,
rp->regspec_addr, rp->regspec_size);
return (DDI_ME_INVAL);
}
if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
/*
* compatibility i/o mapping
*/
rp->regspec_bustype += (uint_t)offset;
} else {
/*
* Normal memory or i/o mapping
*/
rp->regspec_addr += (uint_t)offset;
}
if (len != 0)
rp->regspec_size = (uint_t)len;
#ifdef DDI_MAP_DEBUG
cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
"len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
offset, len, mp->map_handlep);
#endif /* DDI_MAP_DEBUG */
/*
* Apply any parent ranges at this level, if applicable.
* (This is where nexus specific regspec translation takes place.
* Use of this function is implicit agreement that translation is
* provided via ddi_apply_range.)
*/
#ifdef DDI_MAP_DEBUG
ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
ddi_get_name(dip), ddi_get_name(rdip));
#endif /* DDI_MAP_DEBUG */
if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
return (error);
switch (mp->map_op) {
case DDI_MO_MAP_LOCKED:
/*
* Set up the locked down kernel mapping to the regspec...
*/
return (rootnex_map_regspec(mp, vaddrp));
case DDI_MO_UNMAP:
/*
* Release mapping...
*/
return (rootnex_unmap_regspec(mp, vaddrp));
case DDI_MO_MAP_HANDLE:
return (rootnex_map_handle(mp));
default:
return (DDI_ME_UNIMPLEMENTED);
}
}
/*
* rootnex_map_fault()
*
* fault in mappings for requestors
*/
/*ARGSUSED*/
static int
rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
uint_t lock)
{
#ifdef DDI_MAP_DEBUG
ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
ddi_map_debug(" Seg <%s>\n",
seg->s_ops == &segdev_ops ? "segdev" :
seg == &kvseg ? "segkmem" : "NONE!");
#endif /* DDI_MAP_DEBUG */
/*
* This is all terribly broken, but it is a start
*
* XXX Note that this test means that segdev_ops
* must be exported from seg_dev.c.
* XXX What about devices with their own segment drivers?
*/
if (seg->s_ops == &segdev_ops) {
struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
if (hat == NULL) {
/*
* This is one plausible interpretation of
* a null hat i.e. use the first hat on the
* address space hat list which by convention is
* the hat of the system MMU. At alternative
* would be to panic .. this might well be better ..
*/
ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock));
hat = seg->s_as->a_hat;
cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
}
hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
(lock ? HAT_LOAD_LOCK : HAT_LOAD));
} else if (seg == &kvseg && dp == NULL) {
hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
HAT_LOAD_LOCK);
} else
return (DDI_FAILURE);
return (DDI_SUCCESS);
}
/*
* rootnex_map_regspec()
* we don't support mapping of I/O cards above 4Gb
*/
static int
rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
{
rootnex_addr_t rbase;
void *cvaddr;
uint_t npages, pgoffset;
struct regspec *rp;
ddi_acc_hdl_t *hp;
ddi_acc_impl_t *ap;
uint_t hat_acc_flags;
paddr_t pbase;
rp = mp->map_obj.rp;
hp = mp->map_handlep;
#ifdef DDI_MAP_DEBUG
ddi_map_debug(
"rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
rp->regspec_bustype, rp->regspec_addr,
rp->regspec_size, mp->map_handlep);
#endif /* DDI_MAP_DEBUG */
/*
* I/O or memory mapping
*
* <bustype=0, addr=x, len=x>: memory
* <bustype=1, addr=x, len=x>: i/o
* <bustype>1, addr=0, len=x>: x86-compatibility i/o
*/
if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
cmn_err(CE_WARN, "rootnex: invalid register spec"
" <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
rp->regspec_addr, rp->regspec_size);
return (DDI_FAILURE);
}
if (rp->regspec_bustype != 0) {
/*
* I/O space - needs a handle.
*/
if (hp == NULL) {
return (DDI_FAILURE);
}
ap = (ddi_acc_impl_t *)hp->ah_platform_private;
ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
impl_acc_hdl_init(hp);
if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
#ifdef DDI_MAP_DEBUG
ddi_map_debug("rootnex_map_regspec: mmap() "
"to I/O space is not supported.\n");
#endif /* DDI_MAP_DEBUG */
return (DDI_ME_INVAL);
} else {
/*
* 1275-compliant vs. compatibility i/o mapping
*/
*vaddrp =
(rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
((caddr_t)(uintptr_t)rp->regspec_bustype) :
((caddr_t)(uintptr_t)rp->regspec_addr);
#ifdef __xpv
if (DOMAIN_IS_INITDOMAIN(xen_info)) {
hp->ah_pfn = xen_assign_pfn(
mmu_btop((ulong_t)rp->regspec_addr &
MMU_PAGEMASK));
} else {
hp->ah_pfn = mmu_btop(
(ulong_t)rp->regspec_addr & MMU_PAGEMASK);
}
#else
hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
MMU_PAGEMASK);
#endif
hp->ah_pnum = mmu_btopr(rp->regspec_size +
(ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
}
#ifdef DDI_MAP_DEBUG
ddi_map_debug(
"rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
rp->regspec_size, *vaddrp);
#endif /* DDI_MAP_DEBUG */
return (DDI_SUCCESS);
}
/*
* Memory space
*/
if (hp != NULL) {
/*
* hat layer ignores
* hp->ah_acc.devacc_attr_endian_flags.
*/
switch (hp->ah_acc.devacc_attr_dataorder) {
case DDI_STRICTORDER_ACC:
hat_acc_flags = HAT_STRICTORDER;
break;
case DDI_UNORDERED_OK_ACC:
hat_acc_flags = HAT_UNORDERED_OK;
break;
case DDI_MERGING_OK_ACC:
hat_acc_flags = HAT_MERGING_OK;
break;
case DDI_LOADCACHING_OK_ACC:
hat_acc_flags = HAT_LOADCACHING_OK;
break;
case DDI_STORECACHING_OK_ACC:
hat_acc_flags = HAT_STORECACHING_OK;
break;
}
ap = (ddi_acc_impl_t *)hp->ah_platform_private;
ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
impl_acc_hdl_init(hp);
hp->ah_hat_flags = hat_acc_flags;
} else {
hat_acc_flags = HAT_STRICTORDER;
}
rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
#ifdef __xpv
/*
* If we're dom0, we're using a real device so we need to translate
* the MA to a PA.
*/
if (DOMAIN_IS_INITDOMAIN(xen_info)) {
pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
} else {
pbase = rbase;
}
#else
pbase = rbase;
#endif
pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
if (rp->regspec_size == 0) {
#ifdef DDI_MAP_DEBUG
ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
#endif /* DDI_MAP_DEBUG */
return (DDI_ME_INVAL);
}
if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
/* extra cast to make gcc happy */
*vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
} else {
npages = mmu_btopr(rp->regspec_size + pgoffset);
#ifdef DDI_MAP_DEBUG
ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
"physical %llx", npages, pbase);
#endif /* DDI_MAP_DEBUG */
cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
if (cvaddr == NULL)
return (DDI_ME_NORESOURCES);
/*
* Now map in the pages we've allocated...
*/
hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
mmu_btop(pbase), mp->map_prot | hat_acc_flags,
HAT_LOAD_LOCK);
*vaddrp = (caddr_t)cvaddr + pgoffset;
/* save away pfn and npages for FMA */
hp = mp->map_handlep;
if (hp) {
hp->ah_pfn = mmu_btop(pbase);
hp->ah_pnum = npages;
}
}
#ifdef DDI_MAP_DEBUG
ddi_map_debug("at virtual 0x%x\n", *vaddrp);
#endif /* DDI_MAP_DEBUG */
return (DDI_SUCCESS);
}
/*
* rootnex_unmap_regspec()
*
*/
static int
rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
{
caddr_t addr = (caddr_t)*vaddrp;
uint_t npages, pgoffset;
struct regspec *rp;
if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
return (0);
rp = mp->map_obj.rp;
if (rp->regspec_size == 0) {
#ifdef DDI_MAP_DEBUG
ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
#endif /* DDI_MAP_DEBUG */
return (DDI_ME_INVAL);
}
/*
* I/O or memory mapping:
*
* <bustype=0, addr=x, len=x>: memory
* <bustype=1, addr=x, len=x>: i/o
* <bustype>1, addr=0, len=x>: x86-compatibility i/o
*/
if (rp->regspec_bustype != 0) {
/*
* This is I/O space, which requires no particular
* processing on unmap since it isn't mapped in the
* first place.
*/
return (DDI_SUCCESS);
}
/*
* Memory space
*/
pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
npages = mmu_btopr(rp->regspec_size + pgoffset);
hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
device_arena_free(addr - pgoffset, ptob(npages));
/*
* Destroy the pointer - the mapping has logically gone
*/
*vaddrp = NULL;
return (DDI_SUCCESS);
}
/*
* rootnex_map_handle()
*
*/
static int
rootnex_map_handle(ddi_map_req_t *mp)
{
rootnex_addr_t rbase;
ddi_acc_hdl_t *hp;
uint_t pgoffset;
struct regspec *rp;
paddr_t pbase;
rp = mp->map_obj.rp;
#ifdef DDI_MAP_DEBUG
ddi_map_debug(
"rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
rp->regspec_bustype, rp->regspec_addr,
rp->regspec_size, mp->map_handlep);
#endif /* DDI_MAP_DEBUG */
/*
* I/O or memory mapping:
*
* <bustype=0, addr=x, len=x>: memory
* <bustype=1, addr=x, len=x>: i/o
* <bustype>1, addr=0, len=x>: x86-compatibility i/o
*/
if (rp->regspec_bustype != 0) {
/*
* This refers to I/O space, and we don't support "mapping"
* I/O space to a user.
*/
return (DDI_FAILURE);
}
/*
* Set up the hat_flags for the mapping.
*/
hp = mp->map_handlep;
switch (hp->ah_acc.devacc_attr_endian_flags) {
case DDI_NEVERSWAP_ACC:
hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
break;
case DDI_STRUCTURE_LE_ACC:
hp->ah_hat_flags = HAT_STRUCTURE_LE;
break;
case DDI_STRUCTURE_BE_ACC:
return (DDI_FAILURE);
default:
return (DDI_REGS_ACC_CONFLICT);
}
switch (hp->ah_acc.devacc_attr_dataorder) {
case DDI_STRICTORDER_ACC:
break;
case DDI_UNORDERED_OK_ACC:
hp->ah_hat_flags |= HAT_UNORDERED_OK;
break;
case DDI_MERGING_OK_ACC:
hp->ah_hat_flags |= HAT_MERGING_OK;
break;
case DDI_LOADCACHING_OK_ACC:
hp->ah_hat_flags |= HAT_LOADCACHING_OK;
break;
case DDI_STORECACHING_OK_ACC:
hp->ah_hat_flags |= HAT_STORECACHING_OK;
break;
default:
return (DDI_FAILURE);
}
rbase = (rootnex_addr_t)rp->regspec_addr &
(~(rootnex_addr_t)MMU_PAGEOFFSET);
pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
if (rp->regspec_size == 0)
return (DDI_ME_INVAL);
#ifdef __xpv
/*
* If we're dom0, we're using a real device so we need to translate
* the MA to a PA.
*/
if (DOMAIN_IS_INITDOMAIN(xen_info)) {
pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
(rbase & MMU_PAGEOFFSET);
} else {
pbase = rbase;
}
#else
pbase = rbase;
#endif
hp->ah_pfn = mmu_btop(pbase);
hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
return (DDI_SUCCESS);
}
/*
* ************************
* interrupt related code
* ************************
*/
/*
* rootnex_intr_ops()
* bus_intr_op() function for interrupt support
*/
/* ARGSUSED */
static int
rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
ddi_intr_handle_impl_t *hdlp, void *result)
{
struct intrspec *ispec;
struct ddi_parent_private_data *pdp;
DDI_INTR_NEXDBG((CE_CONT,
"rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
(void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
/* Process the interrupt operation */
switch (intr_op) {
case DDI_INTROP_GETCAP:
/* First check with pcplusmp */
if (psm_intr_ops == NULL)
return (DDI_FAILURE);
if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
*(int *)result = 0;
return (DDI_FAILURE);
}
break;
case DDI_INTROP_SETCAP:
if (psm_intr_ops == NULL)
return (DDI_FAILURE);
if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
return (DDI_FAILURE);
break;
case DDI_INTROP_ALLOC:
if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
return (DDI_FAILURE);
hdlp->ih_pri = ispec->intrspec_pri;
*(int *)result = hdlp->ih_scratch1;
break;
case DDI_INTROP_FREE:
pdp = ddi_get_parent_data(rdip);
/*
* Special case for 'pcic' driver' only.
* If an intrspec was created for it, clean it up here
* See detailed comments on this in the function
* rootnex_get_ispec().
*/
if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
kmem_free(pdp->par_intr, sizeof (struct intrspec) *
pdp->par_nintr);
/*
* Set it to zero; so that
* DDI framework doesn't free it again
*/
pdp->par_intr = NULL;
pdp->par_nintr = 0;
}
break;
case DDI_INTROP_GETPRI:
if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
return (DDI_FAILURE);
*(int *)result = ispec->intrspec_pri;
break;
case DDI_INTROP_SETPRI:
/* Validate the interrupt priority passed to us */
if (*(int *)result > LOCK_LEVEL)
return (DDI_FAILURE);
/* Ensure that PSM is all initialized and ispec is ok */
if ((psm_intr_ops == NULL) ||
((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
return (DDI_FAILURE);
/* Change the priority */
if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
PSM_FAILURE)
return (DDI_FAILURE);
/* update the ispec with the new priority */
ispec->intrspec_pri = *(int *)result;
break;
case DDI_INTROP_ADDISR:
if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
return (DDI_FAILURE);
ispec->intrspec_func = hdlp->ih_cb_func;
break;
case DDI_INTROP_REMISR:
if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
return (DDI_FAILURE);
ispec->intrspec_func = (uint_t (*)()) 0;
break;
case DDI_INTROP_ENABLE:
if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
return (DDI_FAILURE);
/* Call psmi to translate irq with the dip */
if (psm_intr_ops == NULL)
return (DDI_FAILURE);
((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
(int *)&hdlp->ih_vector) == PSM_FAILURE)
return (DDI_FAILURE);
/* Add the interrupt handler */
if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
return (DDI_FAILURE);
break;
case DDI_INTROP_DISABLE:
if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
return (DDI_FAILURE);
/* Call psm_ops() to translate irq with the dip */
if (psm_intr_ops == NULL)
return (DDI_FAILURE);
((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
(void) (*psm_intr_ops)(rdip, hdlp,
PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
/* Remove the interrupt handler */
rem_avintr((void *)hdlp, ispec->intrspec_pri,
hdlp->ih_cb_func, hdlp->ih_vector);
break;
case DDI_INTROP_SETMASK:
if (psm_intr_ops == NULL)
return (DDI_FAILURE);
if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
return (DDI_FAILURE);
break;
case DDI_INTROP_CLRMASK:
if (psm_intr_ops == NULL)
return (DDI_FAILURE);
if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
return (DDI_FAILURE);
break;
case DDI_INTROP_GETPENDING:
if (psm_intr_ops == NULL)
return (DDI_FAILURE);
if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
result)) {
*(int *)result = 0;
return (DDI_FAILURE);
}
break;
case DDI_INTROP_NAVAIL:
case DDI_INTROP_NINTRS:
*(int *)result = i_ddi_get_intx_nintrs(rdip);
if (*(int *)result == 0) {
/*
* Special case for 'pcic' driver' only. This driver
* driver is a child of 'isa' and 'rootnex' drivers.
*
* See detailed comments on this in the function
* rootnex_get_ispec().
*
* Children of 'pcic' send 'NINITR' request all the
* way to rootnex driver. But, the 'pdp->par_nintr'
* field may not initialized. So, we fake it here
* to return 1 (a la what PCMCIA nexus does).
*/
if (strcmp(ddi_get_name(rdip), "pcic") == 0)
*(int *)result = 1;
else
return (DDI_FAILURE);
}
break;
case DDI_INTROP_SUPPORTED_TYPES:
*(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
break;
default:
return (DDI_FAILURE);
}
return (DDI_SUCCESS);
}
/*
* rootnex_get_ispec()
* convert an interrupt number to an interrupt specification.
* The interrupt number determines which interrupt spec will be
* returned if more than one exists.
*
* Look into the parent private data area of the 'rdip' to find out
* the interrupt specification. First check to make sure there is
* one that matchs "inumber" and then return a pointer to it.
*
* Return NULL if one could not be found.
*
* NOTE: This is needed for rootnex_intr_ops()
*/
static struct intrspec *
rootnex_get_ispec(dev_info_t *rdip, int inum)
{
struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
/*
* Special case handling for drivers that provide their own
* intrspec structures instead of relying on the DDI framework.
*
* A broken hardware driver in ON could potentially provide its
* own intrspec structure, instead of relying on the hardware.
* If these drivers are children of 'rootnex' then we need to
* continue to provide backward compatibility to them here.
*
* Following check is a special case for 'pcic' driver which
* was found to have broken hardwre andby provides its own intrspec.
*
* Verbatim comments from this driver are shown here:
* "Don't use the ddi_add_intr since we don't have a
* default intrspec in all cases."
*
* Since an 'ispec' may not be always created for it,
* check for that and create one if so.
*
* NOTE: Currently 'pcic' is the only driver found to do this.
*/
if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
pdp->par_nintr = 1;
pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
pdp->par_nintr, KM_SLEEP);
}
/* Validate the interrupt number */
if (inum >= pdp->par_nintr)
return (NULL);
/* Get the interrupt structure pointer and return that */
return ((struct intrspec *)&pdp->par_intr[inum]);
}
/*
* ******************
* dma related code
* ******************
*/
/*ARGSUSED*/
static int
rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
ddi_dma_handle_t *handlep)
{
uint64_t maxsegmentsize_ll;
uint_t maxsegmentsize;
ddi_dma_impl_t *hp;
rootnex_dma_t *dma;
uint64_t count_max;
uint64_t seg;
int kmflag;
int e;
/* convert our sleep flags */
if (waitfp == DDI_DMA_SLEEP) {
kmflag = KM_SLEEP;
} else {
kmflag = KM_NOSLEEP;
}
/*
* We try to do only one memory allocation here. We'll do a little
* pointer manipulation later. If the bind ends up taking more than
* our prealloc's space, we'll have to allocate more memory in the
* bind operation. Not great, but much better than before and the
* best we can do with the current bind interfaces.
*/
hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
if (hp == NULL) {
if (waitfp != DDI_DMA_DONTWAIT) {
ddi_set_callback(waitfp, arg,
&rootnex_state->r_dvma_call_list_id);
}
return (DDI_DMA_NORESOURCES);
}
/* Do our pointer manipulation now, align the structures */
hp->dmai_private = (void *)(((uintptr_t)hp +
(uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
dma = (rootnex_dma_t *)hp->dmai_private;
dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
sizeof (rootnex_dma_t) + 0x7) & ~0x7);
/* setup the handle */
rootnex_clean_dmahdl(hp);
dma->dp_dip = rdip;
dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
hp->dmai_minxfer = attr->dma_attr_minxfer;
hp->dmai_burstsizes = attr->dma_attr_burstsizes;
hp->dmai_rdip = rdip;
hp->dmai_attr = *attr;
/* we don't need to worry about the SPL since we do a tryenter */
mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
/*
* Figure out our maximum segment size. If the segment size is greater
* than 4G, we will limit it to (4G - 1) since the max size of a dma
* object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
* dma_attr_count_max are size-1 type values.
*
* Maximum segment size is the largest physically contiguous chunk of
* memory that we can return from a bind (i.e. the maximum size of a
* single cookie).
*/
/* handle the rollover cases */
seg = attr->dma_attr_seg + 1;
if (seg < attr->dma_attr_seg) {
seg = attr->dma_attr_seg;
}
count_max = attr->dma_attr_count_max + 1;
if (count_max < attr->dma_attr_count_max) {
count_max = attr->dma_attr_count_max;
}
/*
* granularity may or may not be a power of two. If it isn't, we can't
* use a simple mask.
*/
if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) {
dma->dp_granularity_power_2 = B_FALSE;
} else {
dma->dp_granularity_power_2 = B_TRUE;
}
/*
* maxxfer should be a whole multiple of granularity. If we're going to
* break up a window because we're greater than maxxfer, we might as
* well make sure it's maxxfer is a whole multiple so we don't have to
* worry about triming the window later on for this case.
*/
if (attr->dma_attr_granular > 1) {
if (dma->dp_granularity_power_2) {
dma->dp_maxxfer = attr->dma_attr_maxxfer -
(attr->dma_attr_maxxfer &
(attr->dma_attr_granular - 1));
} else {
dma->dp_maxxfer = attr->dma_attr_maxxfer -
(attr->dma_attr_maxxfer % attr->dma_attr_granular);
}
} else {
dma->dp_maxxfer = attr->dma_attr_maxxfer;
}
maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
maxsegmentsize = 0xFFFFFFFF;
} else {
maxsegmentsize = maxsegmentsize_ll;
}
dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
/* check the ddi_dma_attr arg to make sure it makes a little sense */
if (rootnex_alloc_check_parms) {
e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
if (e != DDI_SUCCESS) {
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
(void) rootnex_dma_freehdl(dip, rdip,
(ddi_dma_handle_t)hp);
return (e);
}
}
*handlep = (ddi_dma_handle_t)hp;
ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
return (DDI_SUCCESS);
}
/*
* rootnex_dma_allochdl()
* called from ddi_dma_alloc_handle().
*/
static int
rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
{
#if !defined(__xpv)
uint_t error = ENOTSUP;
int retval;
retval = iommulib_nex_open(rdip, &error);
if (retval != DDI_SUCCESS && error == ENOTSUP) {
/* No IOMMU */
return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
handlep));
} else if (retval != DDI_SUCCESS) {
return (DDI_FAILURE);
}
ASSERT(IOMMU_USED(rdip));
/* has an IOMMU */
return (iommulib_nexdma_allochdl(dip, rdip, attr,
waitfp, arg, handlep));
#else
return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
handlep));
#endif
}
/*ARGSUSED*/
static int
rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle)
{
ddi_dma_impl_t *hp;
rootnex_dma_t *dma;
hp = (ddi_dma_impl_t *)handle;
dma = (rootnex_dma_t *)hp->dmai_private;
/* unbind should have been called first */
ASSERT(!dma->dp_inuse);
mutex_destroy(&dma->dp_mutex);
kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
if (rootnex_state->r_dvma_call_list_id)
ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
return (DDI_SUCCESS);
}
/*
* rootnex_dma_freehdl()
* called from ddi_dma_free_handle().
*/
static int
rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
{
#if !defined(__xpv)
if (IOMMU_USED(rdip)) {
return (iommulib_nexdma_freehdl(dip, rdip, handle));
}
#endif
return (rootnex_coredma_freehdl(dip, rdip, handle));
}
/*ARGSUSED*/
static int
rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
ddi_dma_cookie_t *cookiep, uint_t *ccountp)
{
rootnex_sglinfo_t *sinfo;
ddi_dma_attr_t *attr;
ddi_dma_impl_t *hp;
rootnex_dma_t *dma;
int kmflag;
int e;
hp = (ddi_dma_impl_t *)handle;
dma = (rootnex_dma_t *)hp->dmai_private;
sinfo = &dma->dp_sglinfo;
attr = &hp->dmai_attr;
if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
dma->dp_sleep_flags = KM_SLEEP;
} else {
dma->dp_sleep_flags = KM_NOSLEEP;
}
hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
/*
* This is useful for debugging a driver. Not as useful in a production
* system. The only time this will fail is if you have a driver bug.
*/
if (rootnex_bind_check_inuse) {
/*
* No one else should ever have this lock unless someone else
* is trying to use this handle. So contention on the lock
* is the same as inuse being set.
*/
e = mutex_tryenter(&dma->dp_mutex);
if (e == 0) {
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
return (DDI_DMA_INUSE);
}
if (dma->dp_inuse) {
mutex_exit(&dma->dp_mutex);
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
return (DDI_DMA_INUSE);
}
dma->dp_inuse = B_TRUE;
mutex_exit(&dma->dp_mutex);
}
/* check the ddi_dma_attr arg to make sure it makes a little sense */
if (rootnex_bind_check_parms) {
e = rootnex_valid_bind_parms(dmareq, attr);
if (e != DDI_SUCCESS) {
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
rootnex_clean_dmahdl(hp);
return (e);
}
}
/* save away the original bind info */
dma->dp_dma = dmareq->dmar_object;
#if !defined(__xpv)
if (rootnex_state->r_intel_iommu_enabled) {
e = intel_iommu_map_sgl(handle, dmareq,
rootnex_state->r_prealloc_cookies);
switch (e) {
case IOMMU_SGL_SUCCESS:
goto rootnex_sgl_end;
case IOMMU_SGL_DISABLE:
goto rootnex_sgl_start;
case IOMMU_SGL_NORESOURCES:
cmn_err(CE_WARN, "iommu map sgl failed for %s",
ddi_node_name(dma->dp_dip));
rootnex_clean_dmahdl(hp);
return (DDI_DMA_NORESOURCES);
default:
cmn_err(CE_WARN,
"undefined value returned from"
" intel_iommu_map_sgl: %d",
e);
rootnex_clean_dmahdl(hp);
return (DDI_DMA_NORESOURCES);
}
}
#endif
rootnex_sgl_start:
/*
* Figure out a rough estimate of what maximum number of pages this
* buffer could use (a high estimate of course).
*/
sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
/*
* We'll use the pre-allocated cookies for any bind that will *always*
* fit (more important to be consistent, we don't want to create
* additional degenerate cases).
*/
if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
dma->dp_need_to_free_cookie = B_FALSE;
DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
uint_t, sinfo->si_max_pages);
/*
* For anything larger than that, we'll go ahead and allocate the
* maximum number of pages we expect to see. Hopefuly, we won't be
* seeing this path in the fast path for high performance devices very
* frequently.
*
* a ddi bind interface that allowed the driver to provide storage to
* the bind interface would speed this case up.
*/
} else {
/* convert the sleep flags */
if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
kmflag = KM_SLEEP;
} else {
kmflag = KM_NOSLEEP;
}
/*
* Save away how much memory we allocated. If we're doing a
* nosleep, the alloc could fail...
*/
dma->dp_cookie_size = sinfo->si_max_pages *
sizeof (ddi_dma_cookie_t);
dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
if (dma->dp_cookies == NULL) {
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
rootnex_clean_dmahdl(hp);
return (DDI_DMA_NORESOURCES);
}
dma->dp_need_to_free_cookie = B_TRUE;
DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t,
sinfo->si_max_pages);
}
hp->dmai_cookie = dma->dp_cookies;
/*
* Get the real sgl. rootnex_get_sgl will fill in cookie array while
* looking at the contraints in the dma structure. It will then put some
* additional state about the sgl in the dma struct (i.e. is the sgl
* clean, or do we need to do some munging; how many pages need to be
* copied, etc.)
*/
rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies,
&dma->dp_sglinfo);
rootnex_sgl_end:
ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
/* if we don't need a copy buffer, we don't need to sync */
if (sinfo->si_copybuf_req == 0) {
hp->dmai_rflags |= DMP_NOSYNC;
}
/*
* if we don't need the copybuf and we don't need to do a partial, we
* hit the fast path. All the high performance devices should be trying
* to hit this path. To hit this path, a device should be able to reach
* all of memory, shouldn't try to bind more than it can transfer, and
* the buffer shouldn't require more cookies than the driver/device can
* handle [sgllen]).
*/
if ((sinfo->si_copybuf_req == 0) &&
(sinfo->si_sgl_size <= attr->dma_attr_sgllen) &&
(dma->dp_dma.dmao_size < dma->dp_maxxfer)) {
/*
* If the driver supports FMA, insert the handle in the FMA DMA
* handle cache.
*/
if (attr->dma_attr_flags & DDI_DMA_FLAGERR) {
hp->dmai_error.err_cf = rootnex_dma_check;
(void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL);
}
/*
* copy out the first cookie and ccountp, set the cookie
* pointer to the second cookie. The first cookie is passed
* back on the stack. Additional cookies are accessed via
* ddi_dma_nextcookie()
*/
*cookiep = dma->dp_cookies[0];
*ccountp = sinfo->si_sgl_size;
hp->dmai_cookie++;
hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
hp->dmai_nwin = 1;
ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
ROOTNEX_DPROBE3(rootnex__bind__fast, dev_info_t *, rdip,
uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
dma->dp_dma.dmao_size);
return (DDI_DMA_MAPPED);
}
/*
* go to the slow path, we may need to alloc more memory, create
* multiple windows, and munge up a sgl to make the device happy.
*/
e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag);
if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
if (dma->dp_need_to_free_cookie) {
kmem_free(dma->dp_cookies, dma->dp_cookie_size);
}
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
rootnex_clean_dmahdl(hp); /* must be after free cookie */
return (e);
}
/*
* If the driver supports FMA, insert the handle in the FMA DMA handle
* cache.
*/
if (attr->dma_attr_flags & DDI_DMA_FLAGERR) {
hp->dmai_error.err_cf = rootnex_dma_check;
(void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL);
}
/* if the first window uses the copy buffer, sync it for the device */
if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
(hp->dmai_rflags & DDI_DMA_WRITE)) {
(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
DDI_DMA_SYNC_FORDEV);
}
/*
* copy out the first cookie and ccountp, set the cookie pointer to the
* second cookie. Make sure the partial flag is set/cleared correctly.
* If we have a partial map (i.e. multiple windows), the number of
* cookies we return is the number of cookies in the first window.
*/
if (e == DDI_DMA_MAPPED) {
hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
*ccountp = sinfo->si_sgl_size;
} else {
hp->dmai_rflags |= DDI_DMA_PARTIAL;
*ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
ASSERT(hp->dmai_nwin <= dma->dp_max_win);
}
*cookiep = dma->dp_cookies[0];
hp->dmai_cookie++;
ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
ROOTNEX_DPROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
dma->dp_dma.dmao_size);
return (e);
}
/*
* rootnex_dma_bindhdl()
* called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
*/
static int
rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
ddi_dma_cookie_t *cookiep, uint_t *ccountp)
{
#if !defined(__xpv)
if (IOMMU_USED(rdip)) {
return (iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
cookiep, ccountp));
}
#endif
return (rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
cookiep, ccountp));
}
/*ARGSUSED*/
static int
rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle)
{
ddi_dma_impl_t *hp;
rootnex_dma_t *dma;
int e;
hp = (ddi_dma_impl_t *)handle;
dma = (rootnex_dma_t *)hp->dmai_private;
/* make sure the buffer wasn't free'd before calling unbind */
if (rootnex_unbind_verify_buffer) {
e = rootnex_verify_buffer(dma);
if (e != DDI_SUCCESS) {
ASSERT(0);
return (DDI_FAILURE);
}
}
/* sync the current window before unbinding the buffer */
if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
(hp->dmai_rflags & DDI_DMA_READ)) {
(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
DDI_DMA_SYNC_FORCPU);
}
/*
* If the driver supports FMA, remove the handle in the FMA DMA handle
* cache.
*/
if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) {
if ((DEVI(rdip)->devi_fmhdl != NULL) &&
(DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) {
(void) ndi_fmc_remove(rdip, DMA_HANDLE, hp);
}
}
/*
* cleanup and copy buffer or window state. if we didn't use the copy
* buffer or windows, there won't be much to do :-)
*/
rootnex_teardown_copybuf(dma);
rootnex_teardown_windows(dma);
#if !defined(__xpv)
/*
* If intel iommu enabled, clean up the page tables and free the dvma
*/
if (rootnex_state->r_intel_iommu_enabled) {
intel_iommu_unmap_sgl(handle);
}
#endif
/*
* If we had to allocate space to for the worse case sgl (it didn't
* fit into our pre-allocate buffer), free that up now
*/
if (dma->dp_need_to_free_cookie) {
kmem_free(dma->dp_cookies, dma->dp_cookie_size);
}
/*
* clean up the handle so it's ready for the next bind (i.e. if the
* handle is reused).
*/
rootnex_clean_dmahdl(hp);
if (rootnex_state->r_dvma_call_list_id)
ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
return (DDI_SUCCESS);
}
/*
* rootnex_dma_unbindhdl()
* called from ddi_dma_unbind_handle()
*/
/*ARGSUSED*/
static int
rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
ddi_dma_handle_t handle)
{
#if !defined(__xpv)
if (IOMMU_USED(rdip)) {
return (iommulib_nexdma_unbindhdl(dip, rdip, handle));
}
#endif
return (rootnex_coredma_unbindhdl(dip, rdip, handle));
}
#if !defined(__xpv)
static int
rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
{
ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
if (dma->dp_sleep_flags != KM_SLEEP &&
dma->dp_sleep_flags != KM_NOSLEEP)
cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
return (dma->dp_sleep_flags);
}
/*ARGSUSED*/
static void
rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
{
ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
rootnex_window_t *window;
if (dma->dp_window) {
window = &dma->dp_window[dma->dp_current_win];
hp->dmai_cookie = window->wd_first_cookie;
} else {
hp->dmai_cookie = dma->dp_cookies;
}
hp->dmai_cookie++;
}
/*ARGSUSED*/
static int
rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
{
int i;
int km_flags;
ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
rootnex_window_t *window;
ddi_dma_cookie_t *cp;
ddi_dma_cookie_t *cookie;
ASSERT(*cookiepp == NULL);
ASSERT(*ccountp == 0);
if (dma->dp_window) {
window = &dma->dp_window[dma->dp_current_win];
cp = window->wd_first_cookie;
*ccountp = window->wd_cookie_cnt;
} else {
cp = dma->dp_cookies;
*ccountp = dma->dp_sglinfo.si_sgl_size;
}
km_flags = rootnex_coredma_get_sleep_flags(handle);
cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
if (cookie == NULL) {
return (DDI_DMA_NORESOURCES);
}
for (i = 0; i < *ccountp; i++) {
cookie[i].dmac_notused = cp[i].dmac_notused;
cookie[i].dmac_type = cp[i].dmac_type;
cookie[i].dmac_address = cp[i].dmac_address;
cookie[i].dmac_size = cp[i].dmac_size;
}
*cookiepp = cookie;
return (DDI_SUCCESS);
}
/*ARGSUSED*/
static int
rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
ddi_dma_cookie_t *cookiep, uint_t ccount)
{
ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
rootnex_window_t *window;
ddi_dma_cookie_t *cur_cookiep;
ASSERT(cookiep);
ASSERT(ccount != 0);
ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
if (dma->dp_window) {
window = &dma->dp_window[dma->dp_current_win];
dma->dp_saved_cookies = window->wd_first_cookie;
window->wd_first_cookie = cookiep;
ASSERT(ccount == window->wd_cookie_cnt);
cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
+ window->wd_first_cookie;
} else {
dma->dp_saved_cookies = dma->dp_cookies;
dma->dp_cookies = cookiep;
ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
+ dma->dp_cookies;
}
dma->dp_need_to_switch_cookies = B_TRUE;
hp->dmai_cookie = cur_cookiep;
return (DDI_SUCCESS);
}
/*ARGSUSED*/
static int
rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
{
ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
rootnex_window_t *window;
ddi_dma_cookie_t *cur_cookiep;
ddi_dma_cookie_t *cookie_array;
uint_t ccount;
/* check if cookies have not been switched */
if (dma->dp_need_to_switch_cookies == B_FALSE)
return (DDI_SUCCESS);
ASSERT(dma->dp_saved_cookies);
if (dma->dp_window) {
window = &dma->dp_window[dma->dp_current_win];
cookie_array = window->wd_first_cookie;
window->wd_first_cookie = dma->dp_saved_cookies;
dma->dp_saved_cookies = NULL;
ccount = window->wd_cookie_cnt;
cur_cookiep = (hp->dmai_cookie - cookie_array)
+ window->wd_first_cookie;
} else {
cookie_array = dma->dp_cookies;
dma->dp_cookies = dma->dp_saved_cookies;
dma->dp_saved_cookies = NULL;
ccount = dma->dp_sglinfo.si_sgl_size;
cur_cookiep = (hp->dmai_cookie - cookie_array)
+ dma->dp_cookies;
}
kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
hp->dmai_cookie = cur_cookiep;
dma->dp_need_to_switch_cookies = B_FALSE;
return (DDI_SUCCESS);
}
#endif
/*
* rootnex_verify_buffer()
* verify buffer wasn't free'd
*/
static int
rootnex_verify_buffer(rootnex_dma_t *dma)
{
page_t **pplist;
caddr_t vaddr;
uint_t pcnt;
uint_t poff;
page_t *pp;
char b;
int i;
/* Figure out how many pages this buffer occupies */
if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
} else {
vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
}
pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
switch (dma->dp_dma.dmao_type) {
case DMA_OTYP_PAGES:
/*
* for a linked list of pp's walk through them to make sure
* they're locked and not free.
*/
pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
for (i = 0; i < pcnt; i++) {
if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
return (DDI_FAILURE);
}
pp = pp->p_next;
}
break;
case DMA_OTYP_VADDR:
case DMA_OTYP_BUFVADDR:
pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
/*
* for an array of pp's walk through them to make sure they're
* not free. It's possible that they may not be locked.
*/
if (pplist) {
for (i = 0; i < pcnt; i++) {
if (PP_ISFREE(pplist[i])) {
return (DDI_FAILURE);
}
}
/* For a virtual address, try to peek at each page */
} else {
if (dma->dp_sglinfo.si_asp == &kas) {
for (i = 0; i < pcnt; i++) {
if (ddi_peek8(NULL, vaddr, &b) ==
DDI_FAILURE)
return (DDI_FAILURE);
vaddr += MMU_PAGESIZE;
}
}
}
break;
default:
ASSERT(0);
break;
}
return (DDI_SUCCESS);
}
/*
* rootnex_clean_dmahdl()
* Clean the dma handle. This should be called on a handle alloc and an
* unbind handle. Set the handle state to the default settings.
*/
static void
rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
{
rootnex_dma_t *dma;
dma = (rootnex_dma_t *)hp->dmai_private;
hp->dmai_nwin = 0;
dma->dp_current_cookie = 0;
dma->dp_copybuf_size = 0;
dma->dp_window = NULL;
dma->dp_cbaddr = NULL;
dma->dp_inuse = B_FALSE;
dma->dp_need_to_free_cookie = B_FALSE;
dma->dp_need_to_switch_cookies = B_FALSE;
dma->dp_saved_cookies = NULL;
dma->dp_sleep_flags = KM_PANIC;
dma->dp_need_to_free_window = B_FALSE;
dma->dp_partial_required = B_FALSE;
dma->dp_trim_required = B_FALSE;
dma->dp_sglinfo.si_copybuf_req = 0;
#if !defined(__amd64)
dma->dp_cb_remaping = B_FALSE;
dma->dp_kva = NULL;
#endif
/* FMA related initialization */
hp->dmai_fault = 0;
hp->dmai_fault_check = NULL;
hp->dmai_fault_notify = NULL;
hp->dmai_error.err_ena = 0;
hp->dmai_error.err_status = DDI_FM_OK;
hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
hp->dmai_error.err_ontrap = NULL;
hp->dmai_error.err_fep = NULL;
hp->dmai_error.err_cf = NULL;
}
/*
* rootnex_valid_alloc_parms()
* Called in ddi_dma_alloc_handle path to validate its parameters.
*/
static int
rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
{
if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
(attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
(attr->dma_attr_granular > MMU_PAGESIZE) ||
(attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
return (DDI_DMA_BADATTR);
}
if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
return (DDI_DMA_BADATTR);
}
if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
attr->dma_attr_sgllen <= 0) {
return (DDI_DMA_BADATTR);
}
/* We should be able to DMA into every byte offset in a page */
if (maxsegmentsize < MMU_PAGESIZE) {
return (DDI_DMA_BADATTR);
}
return (DDI_SUCCESS);
}
/*
* rootnex_valid_bind_parms()
* Called in ddi_dma_*_bind_handle path to validate its parameters.
*/
/* ARGSUSED */
static int
rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
{
#if !defined(__amd64)
/*
* we only support up to a 2G-1 transfer size on 32-bit kernels so
* we can track the offset for the obsoleted interfaces.
*/
if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
return (DDI_DMA_TOOBIG);
}
#endif
return (DDI_SUCCESS);
}
/*
* rootnex_get_sgl()
* Called in bind fastpath to get the sgl. Most of this will be replaced
* with a call to the vm layer when vm2.0 comes around...
*/
static void
rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
rootnex_sglinfo_t *sglinfo)
{
ddi_dma_atyp_t buftype;
rootnex_addr_t raddr;
uint64_t last_page;
uint64_t offset;
uint64_t addrhi;
uint64_t addrlo;
uint64_t maxseg;
page_t **pplist;
uint64_t paddr;
uint32_t psize;
uint32_t size;
caddr_t vaddr;
uint_t pcnt;
page_t *pp;
uint_t cnt;
/* shortcuts */
pplist = dmar_object->dmao_obj.virt_obj.v_priv;
vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
maxseg = sglinfo->si_max_cookie_size;
buftype = dmar_object->dmao_type;
addrhi = sglinfo->si_max_addr;
addrlo = sglinfo->si_min_addr;
size = dmar_object->dmao_size;
pcnt = 0;
cnt = 0;
/*
* if we were passed down a linked list of pages, i.e. pointer to
* page_t, use this to get our physical address and buf offset.
*/
if (buftype == DMA_OTYP_PAGES) {
pp = dmar_object->dmao_obj.pp_obj.pp_pp;
ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
offset = dmar_object->dmao_obj.pp_obj.pp_offset &
MMU_PAGEOFFSET;
paddr = pfn_to_pa(pp->p_pagenum) + offset;
psize = MIN(size, (MMU_PAGESIZE - offset));
pp = pp->p_next;
sglinfo->si_asp = NULL;
/*
* We weren't passed down a linked list of pages, but if we were passed
* down an array of pages, use this to get our physical address and buf
* offset.
*/
} else if (pplist != NULL) {
ASSERT((buftype == DMA_OTYP_VADDR) ||
(buftype == DMA_OTYP_BUFVADDR));
offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
if (sglinfo->si_asp == NULL) {
sglinfo->si_asp = &kas;
}
ASSERT(!PP_ISFREE(pplist[pcnt]));
paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
paddr += offset;
psize = MIN(size, (MMU_PAGESIZE - offset));
pcnt++;
/*
* All we have is a virtual address, we'll need to call into the VM
* to get the physical address.
*/
} else {
ASSERT((buftype == DMA_OTYP_VADDR) ||
(buftype == DMA_OTYP_BUFVADDR));
offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
if (sglinfo->si_asp == NULL) {
sglinfo->si_asp = &kas;
}
paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
paddr += offset;
psize = MIN(size, (MMU_PAGESIZE - offset));
vaddr += psize;
}
#ifdef __xpv
/*
* If we're dom0, we're using a real device so we need to load
* the cookies with MFNs instead of PFNs.
*/
raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
#else
raddr = paddr;
#endif
/*
* Setup the first cookie with the physical address of the page and the
* size of the page (which takes into account the initial offset into
* the page.
*/
sgl[cnt].dmac_laddress = raddr;
sgl[cnt].dmac_size = psize;
sgl[cnt].dmac_type = 0;
/*
* Save away the buffer offset into the page. We'll need this later in
* the copy buffer code to help figure out the page index within the
* buffer and the offset into the current page.
*/
sglinfo->si_buf_offset = offset;
/*
* If the DMA engine can't reach the physical address, increase how
* much copy buffer we need. We always increase by pagesize so we don't
* have to worry about converting offsets. Set a flag in the cookies
* dmac_type to indicate that it uses the copy buffer. If this isn't the
* last cookie, go to the next cookie (since we separate each page which
* uses the copy buffer in case the copy buffer is not physically
* contiguous.
*/
if ((raddr < addrlo) || ((raddr + psize) > addrhi)) {
sglinfo->si_copybuf_req += MMU_PAGESIZE;
sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
if ((cnt + 1) < sglinfo->si_max_pages) {
cnt++;
sgl[cnt].dmac_laddress = 0;
sgl[cnt].dmac_size = 0;
sgl[cnt].dmac_type = 0;
}
}
/*
* save this page's physical address so we can figure out if the next
* page is physically contiguous. Keep decrementing size until we are
* done with the buffer.
*/
last_page = raddr & MMU_PAGEMASK;
size -= psize;
while (size > 0) {
/* Get the size for this page (i.e. partial or full page) */
psize = MIN(size, MMU_PAGESIZE);
if (buftype == DMA_OTYP_PAGES) {
/* get the paddr from the page_t */
ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
paddr = pfn_to_pa(pp->p_pagenum);
pp = pp->p_next;
} else if (pplist != NULL) {
/* index into the array of page_t's to get the paddr */
ASSERT(!PP_ISFREE(pplist[pcnt]));
paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
pcnt++;
} else {
/* call into the VM to get the paddr */
paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
vaddr));
vaddr += psize;
}
#ifdef __xpv
/*
* If we're dom0, we're using a real device so we need to load
* the cookies with MFNs instead of PFNs.
*/
raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
#else
raddr = paddr;
#endif
/* check to see if this page needs the copy buffer */
if ((raddr < addrlo) || ((raddr + psize) > addrhi)) {
sglinfo->si_copybuf_req += MMU_PAGESIZE;
/*
* if there is something in the current cookie, go to
* the next one. We only want one page in a cookie which
* uses the copybuf since the copybuf doesn't have to
* be physically contiguous.
*/
if (sgl[cnt].dmac_size != 0) {
cnt++;
}
sgl[cnt].dmac_laddress = raddr;
sgl[cnt].dmac_size = psize;
#if defined(__amd64)
sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
#else
/*
* save the buf offset for 32-bit kernel. used in the
* obsoleted interfaces.
*/
sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
(dmar_object->dmao_size - size);
#endif
/* if this isn't the last cookie, go to the next one */
if ((cnt + 1) < sglinfo->si_max_pages) {
cnt++;
sgl[cnt].dmac_laddress = 0;
sgl[cnt].dmac_size = 0;
sgl[cnt].dmac_type = 0;
}
/*
* this page didn't need the copy buffer, if it's not physically
* contiguous, or it would put us over a segment boundary, or it
* puts us over the max cookie size, or the current sgl doesn't
* have anything in it.
*/
} else if (((last_page + MMU_PAGESIZE) != raddr) ||
!(raddr & sglinfo->si_segmask) ||
((sgl[cnt].dmac_size + psize) > maxseg) ||
(sgl[cnt].dmac_size == 0)) {
/*
* if we're not already in a new cookie, go to the next
* cookie.
*/
if (sgl[cnt].dmac_size != 0) {
cnt++;
}
/* save the cookie information */
sgl[cnt].dmac_laddress = raddr;
sgl[cnt].dmac_size = psize;
#if defined(__amd64)
sgl[cnt].dmac_type = 0;
#else
/*
* save the buf offset for 32-bit kernel. used in the
* obsoleted interfaces.
*/
sgl[cnt].dmac_type = dmar_object->dmao_size - size;
#endif
/*
* this page didn't need the copy buffer, it is physically
* contiguous with the last page, and it's <= the max cookie
* size.
*/
} else {
sgl[cnt].dmac_size += psize;
/*
* if this exactly == the maximum cookie size, and
* it isn't the last cookie, go to the next cookie.
*/
if (((sgl[cnt].dmac_size + psize) == maxseg) &&
((cnt + 1) < sglinfo->si_max_pages)) {
cnt++;
sgl[cnt].dmac_laddress = 0;
sgl[cnt].dmac_size = 0;
sgl[cnt].dmac_type = 0;
}
}
/*
* save this page's physical address so we can figure out if the
* next page is physically contiguous. Keep decrementing size
* until we are done with the buffer.
*/
last_page = raddr;
size -= psize;
}
/* we're done, save away how many cookies the sgl has */
if (sgl[cnt].dmac_size == 0) {
ASSERT(cnt < sglinfo->si_max_pages);
sglinfo->si_sgl_size = cnt;
} else {
sglinfo->si_sgl_size = cnt + 1;
}
}
/*
* rootnex_bind_slowpath()
* Call in the bind path if the calling driver can't use the sgl without
* modifying it. We either need to use the copy buffer and/or we will end up
* with a partial bind.
*/
static int
rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag)
{
rootnex_sglinfo_t *sinfo;
rootnex_window_t *window;
ddi_dma_cookie_t *cookie;
size_t copybuf_used;
size_t dmac_size;
boolean_t partial;
off_t cur_offset;
page_t *cur_pp;
major_t mnum;
int e;
int i;
sinfo = &dma->dp_sglinfo;
copybuf_used = 0;
partial = B_FALSE;
/*
* If we're using the copybuf, set the copybuf state in dma struct.
* Needs to be first since it sets the copy buffer size.
*/
if (sinfo->si_copybuf_req != 0) {
e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
if (e != DDI_SUCCESS) {
return (e);
}
} else {
dma->dp_copybuf_size = 0;
}
/*
* Figure out if we need to do a partial mapping. If so, figure out
* if we need to trim the buffers when we munge the sgl.
*/
if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
(dma->dp_dma.dmao_size > dma->dp_maxxfer) ||
(attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
dma->dp_partial_required = B_TRUE;
if (attr->dma_attr_granular != 1) {
dma->dp_trim_required = B_TRUE;
}
} else {
dma->dp_partial_required = B_FALSE;
dma->dp_trim_required = B_FALSE;
}
/* If we need to do a partial bind, make sure the driver supports it */
if (dma->dp_partial_required &&
!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
mnum = ddi_driver_major(dma->dp_dip);
/*
* patchable which allows us to print one warning per major
* number.
*/
if ((rootnex_bind_warn) &&
((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
cmn_err(CE_WARN, "!%s: coding error detected, the "
"driver is using ddi_dma_attr(9S) incorrectly. "
"There is a small risk of data corruption in "
"particular with large I/Os. The driver should be "
"replaced with a corrected version for proper "
"system operation. To disable this warning, add "
"'set rootnex:rootnex_bind_warn=0' to "
"/etc/system(4).", ddi_driver_name(dma->dp_dip));
}
return (DDI_DMA_TOOBIG);
}
/*
* we might need multiple windows, setup state to handle them. In this
* code path, we will have at least one window.
*/
e = rootnex_setup_windows(hp, dma, attr, kmflag);
if (e != DDI_SUCCESS) {
rootnex_teardown_copybuf(dma);
return (e);
}
window = &dma->dp_window[0];
cookie = &dma->dp_cookies[0];
cur_offset = 0;
rootnex_init_win(hp, dma, window, cookie, cur_offset);
if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) {
cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
}
/* loop though all the cookies we got back from get_sgl() */
for (i = 0; i < sinfo->si_sgl_size; i++) {
/*
* If we're using the copy buffer, check this cookie and setup
* its associated copy buffer state. If this cookie uses the
* copy buffer, make sure we sync this window during dma_sync.
*/
if (dma->dp_copybuf_size > 0) {
rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie,
cur_offset, &copybuf_used, &cur_pp);
if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
window->wd_dosync = B_TRUE;
}
}
/*
* save away the cookie size, since it could be modified in
* the windowing code.
*/
dmac_size = cookie->dmac_size;
/* if we went over max copybuf size */
if (dma->dp_copybuf_size &&
(copybuf_used > dma->dp_copybuf_size)) {
partial = B_TRUE;
e = rootnex_copybuf_window_boundary(hp, dma, &window,
cookie, cur_offset, &copybuf_used);
if (e != DDI_SUCCESS) {
rootnex_teardown_copybuf(dma);
rootnex_teardown_windows(dma);
return (e);
}
/*
* if the coookie uses the copy buffer, make sure the
* new window we just moved to is set to sync.
*/
if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
window->wd_dosync = B_TRUE;
}
DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *,
dma->dp_dip);
/* if the cookie cnt == max sgllen, move to the next window */
} else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) {
partial = B_TRUE;
ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
e = rootnex_sgllen_window_boundary(hp, dma, &window,
cookie, attr, cur_offset);
if (e != DDI_SUCCESS) {
rootnex_teardown_copybuf(dma);
rootnex_teardown_windows(dma);
return (e);
}
/*
* if the coookie uses the copy buffer, make sure the
* new window we just moved to is set to sync.
*/
if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
window->wd_dosync = B_TRUE;
}
DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *,
dma->dp_dip);
/* else if we will be over maxxfer */
} else if ((window->wd_size + dmac_size) >
dma->dp_maxxfer) {
partial = B_TRUE;
e = rootnex_maxxfer_window_boundary(hp, dma, &window,
cookie);
if (e != DDI_SUCCESS) {
rootnex_teardown_copybuf(dma);
rootnex_teardown_windows(dma);
return (e);
}
/*
* if the coookie uses the copy buffer, make sure the
* new window we just moved to is set to sync.
*/
if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
window->wd_dosync = B_TRUE;
}
DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *,
dma->dp_dip);
/* else this cookie fits in the current window */
} else {
window->wd_cookie_cnt++;
window->wd_size += dmac_size;
}
/* track our offset into the buffer, go to the next cookie */
ASSERT(dmac_size <= dma->dp_dma.dmao_size);
ASSERT(cookie->dmac_size <= dmac_size);
cur_offset += dmac_size;
cookie++;
}
/* if we ended up with a zero sized window in the end, clean it up */
if (window->wd_size == 0) {
hp->dmai_nwin--;
window--;
}
ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
if (!partial) {
return (DDI_DMA_MAPPED);
}
ASSERT(dma->dp_partial_required);
return (DDI_DMA_PARTIAL_MAP);
}
/*
* rootnex_setup_copybuf()
* Called in bind slowpath. Figures out if we're going to use the copy
* buffer, and if we do, sets up the basic state to handle it.
*/
static int
rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
rootnex_dma_t *dma, ddi_dma_attr_t *attr)
{
rootnex_sglinfo_t *sinfo;
ddi_dma_attr_t lattr;
size_t max_copybuf;
int cansleep;
int e;
#if !defined(__amd64)
int vmflag;
#endif
sinfo = &dma->dp_sglinfo;
/* read this first so it's consistent through the routine */
max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
/* We need to call into the rootnex on ddi_dma_sync() */
hp->dmai_rflags &= ~DMP_NOSYNC;
/* make sure the copybuf size <= the max size */
dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
#if !defined(__amd64)
/*
* if we don't have kva space to copy to/from, allocate the KVA space
* now. We only do this for the 32-bit kernel. We use seg kpm space for
* the 64-bit kernel.
*/
if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
(dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
/* convert the sleep flags */
if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
vmflag = VM_SLEEP;
} else {
vmflag = VM_NOSLEEP;
}
/* allocate Kernel VA space that we can bcopy to/from */
dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
vmflag);
if (dma->dp_kva == NULL) {
return (DDI_DMA_NORESOURCES);
}
}
#endif
/* convert the sleep flags */
if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
cansleep = 1;
} else {
cansleep = 0;
}
/*
* Allocate the actual copy buffer. This needs to fit within the DMA
* engine limits, so we can't use kmem_alloc... We don't need
* contiguous memory (sgllen) since we will be forcing windows on
* sgllen anyway.
*/
lattr = *attr;
lattr.dma_attr_align = MMU_PAGESIZE;
/*
* this should be < 0 to indicate no limit, but due to a bug in
* the rootnex, we'll set it to the maximum positive int.
*/
lattr.dma_attr_sgllen = 0x7fffffff;
e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
if (e != DDI_SUCCESS) {
#if !defined(__amd64)
if (dma->dp_kva != NULL) {
vmem_free(heap_arena, dma->dp_kva,
dma->dp_copybuf_size);
}
#endif
return (DDI_DMA_NORESOURCES);
}
DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
size_t, dma->dp_copybuf_size);
return (DDI_SUCCESS);
}
/*
* rootnex_setup_windows()
* Called in bind slowpath to setup the window state. We always have windows
* in the slowpath. Even if the window count = 1.
*/
static int
rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
ddi_dma_attr_t *attr, int kmflag)
{
rootnex_window_t *windowp;
rootnex_sglinfo_t *sinfo;
size_t copy_state_size;
size_t win_state_size;
size_t state_available;
size_t space_needed;
uint_t copybuf_win;
uint_t maxxfer_win;
size_t space_used;
uint_t sglwin;
sinfo = &dma->dp_sglinfo;
dma->dp_current_win = 0;
hp->dmai_nwin = 0;
/* If we don't need to do a partial, we only have one window */
if (!dma->dp_partial_required) {
dma->dp_max_win = 1;
/*
* we need multiple windows, need to figure out the worse case number
* of windows.
*/
} else {
/*
* if we need windows because we need more copy buffer that
* we allow, the worse case number of windows we could need
* here would be (copybuf space required / copybuf space that
* we have) plus one for remainder, and plus 2 to handle the
* extra pages on the trim for the first and last pages of the
* buffer (a page is the minimum window size so under the right
* attr settings, you could have a window for each page).
* The last page will only be hit here if the size is not a
* multiple of the granularity (which theoretically shouldn't
* be the case but never has been enforced, so we could have
* broken things without it).
*/
if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
ASSERT(dma->dp_copybuf_size > 0);
copybuf_win = (sinfo->si_copybuf_req /
dma->dp_copybuf_size) + 1 + 2;
} else {
copybuf_win = 0;
}
/*
* if we need windows because we have more cookies than the H/W
* can handle, the number of windows we would need here would
* be (cookie count / cookies count H/W supports) plus one for
* remainder, and plus 2 to handle the extra pages on the trim
* (see above comment about trim)
*/
if (attr->dma_attr_sgllen < sinfo->si_sgl_size) {
sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen)
+ 1) + 2;
} else {
sglwin = 0;
}
/*
* if we need windows because we're binding more memory than the
* H/W can transfer at once, the number of windows we would need
* here would be (xfer count / max xfer H/W supports) plus one
* for remainder, and plus 2 to handle the extra pages on the
* trim (see above comment about trim)
*/
if (dma->dp_dma.dmao_size > dma->dp_maxxfer) {
maxxfer_win = (dma->dp_dma.dmao_size /
dma->dp_maxxfer) + 1 + 2;
} else {
maxxfer_win = 0;
}
dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
ASSERT(dma->dp_max_win > 0);
}
win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
/*
* Get space for window and potential copy buffer state. Before we
* go and allocate memory, see if we can get away with using what's
* left in the pre-allocted state or the dynamically allocated sgl.
*/
space_used = (uintptr_t)(sinfo->si_sgl_size *
sizeof (ddi_dma_cookie_t));
/* if we dynamically allocated space for the cookies */
if (dma->dp_need_to_free_cookie) {
/* if we have more space in the pre-allocted buffer, use it */
ASSERT(space_used <= dma->dp_cookie_size);
if ((dma->dp_cookie_size - space_used) <=
rootnex_state->r_prealloc_size) {
state_available = rootnex_state->r_prealloc_size;
windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
/*
* else, we have more free space in the dynamically allocated
* buffer, i.e. the buffer wasn't worse case fragmented so we
* didn't need a lot of cookies.
*/
} else {
state_available = dma->dp_cookie_size - space_used;
windowp = (rootnex_window_t *)
&dma->dp_cookies[sinfo->si_sgl_size];
}
/* we used the pre-alloced buffer */
} else {
ASSERT(space_used <= rootnex_state->r_prealloc_size);
state_available = rootnex_state->r_prealloc_size - space_used;
windowp = (rootnex_window_t *)
&dma->dp_cookies[sinfo->si_sgl_size];
}
/*
* figure out how much state we need to track the copy buffer. Add an
* addition 8 bytes for pointer alignemnt later.
*/
if (dma->dp_copybuf_size > 0) {
copy_state_size = sinfo->si_max_pages *
sizeof (rootnex_pgmap_t);
} else {
copy_state_size = 0;
}
/* add an additional 8 bytes for pointer alignment */
space_needed = win_state_size + copy_state_size + 0x8;
/* if we have enough space already, use it */
if (state_available >= space_needed) {
dma->dp_window = windowp;
dma->dp_need_to_free_window = B_FALSE;
/* not enough space, need to allocate more. */
} else {
dma->dp_window = kmem_alloc(space_needed, kmflag);
if (dma->dp_window == NULL) {
return (DDI_DMA_NORESOURCES);
}
dma->dp_need_to_free_window = B_TRUE;
dma->dp_window_size = space_needed;
DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *,
dma->dp_dip, size_t, space_needed);
}
/*
* we allocate copy buffer state and window state at the same time.
* setup our copy buffer state pointers. Make sure it's aligned.
*/
if (dma->dp_copybuf_size > 0) {
dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
&dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
#if !defined(__amd64)
/*
* make sure all pm_mapped, pm_vaddr, and pm_pp are set to
* false/NULL. Should be quicker to bzero vs loop and set.
*/
bzero(dma->dp_pgmap, copy_state_size);
#endif
} else {
dma->dp_pgmap = NULL;
}
return (DDI_SUCCESS);
}
/*
* rootnex_teardown_copybuf()
* cleans up after rootnex_setup_copybuf()
*/
static void
rootnex_teardown_copybuf(rootnex_dma_t *dma)
{
#if !defined(__amd64)
int i;
/*
* if we allocated kernel heap VMEM space, go through all the pages and
* map out any of the ones that we're mapped into the kernel heap VMEM
* arena. Then free the VMEM space.
*/
if (dma->dp_kva != NULL) {
for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
if (dma->dp_pgmap[i].pm_mapped) {
hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
MMU_PAGESIZE, HAT_UNLOAD);
dma->dp_pgmap[i].pm_mapped = B_FALSE;
}
}
vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
}
#endif
/* if we allocated a copy buffer, free it */
if (dma->dp_cbaddr != NULL) {
i_ddi_mem_free(dma->dp_cbaddr, NULL);
}
}
/*
* rootnex_teardown_windows()
* cleans up after rootnex_setup_windows()
*/
static void
rootnex_teardown_windows(rootnex_dma_t *dma)
{
/*
* if we had to allocate window state on the last bind (because we
* didn't have enough pre-allocated space in the handle), free it.
*/
if (dma->dp_need_to_free_window) {
kmem_free(dma->dp_window, dma->dp_window_size);
}
}
/*
* rootnex_init_win()
* Called in bind slow path during creation of a new window. Initializes
* window state to default values.
*/
/*ARGSUSED*/
static void
rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
{
hp->dmai_nwin++;
window->wd_dosync = B_FALSE;
window->wd_offset = cur_offset;
window->wd_size = 0;
window->wd_first_cookie = cookie;
window->wd_cookie_cnt = 0;
window->wd_trim.tr_trim_first = B_FALSE;
window->wd_trim.tr_trim_last = B_FALSE;
window->wd_trim.tr_first_copybuf_win = B_FALSE;
window->wd_trim.tr_last_copybuf_win = B_FALSE;
#if !defined(__amd64)
window->wd_remap_copybuf = dma->dp_cb_remaping;
#endif
}
/*
* rootnex_setup_cookie()
* Called in the bind slow path when the sgl uses the copy buffer. If any of
* the sgl uses the copy buffer, we need to go through each cookie, figure
* out if it uses the copy buffer, and if it does, save away everything we'll
* need during sync.
*/
static void
rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
page_t **cur_pp)
{
boolean_t copybuf_sz_power_2;
rootnex_sglinfo_t *sinfo;
paddr_t paddr;
uint_t pidx;
uint_t pcnt;
off_t poff;
#if defined(__amd64)
pfn_t pfn;
#else
page_t **pplist;
#endif
sinfo = &dma->dp_sglinfo;
/*
* Calculate the page index relative to the start of the buffer. The
* index to the current page for our buffer is the offset into the
* first page of the buffer plus our current offset into the buffer
* itself, shifted of course...
*/
pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
ASSERT(pidx < sinfo->si_max_pages);
/* if this cookie uses the copy buffer */
if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
/*
* NOTE: we know that since this cookie uses the copy buffer, it
* is <= MMU_PAGESIZE.
*/
/*
* get the offset into the page. For the 64-bit kernel, get the
* pfn which we'll use with seg kpm.
*/
poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
#if defined(__amd64)
/* mfn_to_pfn() is a NOP on i86pc */
pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
#endif /* __amd64 */
/* figure out if the copybuf size is a power of 2 */
if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) {
copybuf_sz_power_2 = B_FALSE;
} else {
copybuf_sz_power_2 = B_TRUE;
}
/* This page uses the copy buffer */
dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
/*
* save the copy buffer KVA that we'll use with this page.
* if we still fit within the copybuf, it's a simple add.
* otherwise, we need to wrap over using & or % accordingly.
*/
if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
*copybuf_used;
} else {
if (copybuf_sz_power_2) {
dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
(uintptr_t)dma->dp_cbaddr +
(*copybuf_used &
(dma->dp_copybuf_size - 1)));
} else {
dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
(uintptr_t)dma->dp_cbaddr +
(*copybuf_used % dma->dp_copybuf_size));
}
}
/*
* over write the cookie physical address with the address of
* the physical address of the copy buffer page that we will
* use.
*/
paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
#ifdef __xpv
/*
* If we're dom0, we're using a real device so we need to load
* the cookies with MAs instead of PAs.
*/
cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
#else
cookie->dmac_laddress = paddr;
#endif
/* if we have a kernel VA, it's easy, just save that address */
if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
(sinfo->si_asp == &kas)) {
/*
* save away the page aligned virtual address of the
* driver buffer. Offsets are handled in the sync code.
*/
dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
& MMU_PAGEMASK);
#if !defined(__amd64)
/*
* we didn't need to, and will never need to map this
* page.
*/
dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
#endif
/* we don't have a kernel VA. We need one for the bcopy. */
} else {
#if defined(__amd64)
/*
* for the 64-bit kernel, it's easy. We use seg kpm to
* get a Kernel VA for the corresponding pfn.
*/
dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
#else
/*
* for the 32-bit kernel, this is a pain. First we'll
* save away the page_t or user VA for this page. This
* is needed in rootnex_dma_win() when we switch to a
* new window which requires us to re-map the copy
* buffer.
*/
pplist = dmar_object->dmao_obj.virt_obj.v_priv;
if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
dma->dp_pgmap[pidx].pm_pp = *cur_pp;
dma->dp_pgmap[pidx].pm_vaddr = NULL;
} else if (pplist != NULL) {
dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
dma->dp_pgmap[pidx].pm_vaddr = NULL;
} else {
dma->dp_pgmap[pidx].pm_pp = NULL;
dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
(((uintptr_t)
dmar_object->dmao_obj.virt_obj.v_addr +
cur_offset) & MMU_PAGEMASK);
}
/*
* save away the page aligned virtual address which was
* allocated from the kernel heap arena (taking into
* account if we need more copy buffer than we alloced
* and use multiple windows to handle this, i.e. &,%).
* NOTE: there isn't and physical memory backing up this
* virtual address space currently.
*/
if ((*copybuf_used + MMU_PAGESIZE) <=
dma->dp_copybuf_size) {
dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
(((uintptr_t)dma->dp_kva + *copybuf_used) &
MMU_PAGEMASK);
} else {
if (copybuf_sz_power_2) {
dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
(((uintptr_t)dma->dp_kva +
(*copybuf_used &
(dma->dp_copybuf_size - 1))) &
MMU_PAGEMASK);
} else {
dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
(((uintptr_t)dma->dp_kva +
(*copybuf_used %
dma->dp_copybuf_size)) &
MMU_PAGEMASK);
}
}
/*
* if we haven't used up the available copy buffer yet,
* map the kva to the physical page.
*/
if (!dma->dp_cb_remaping && ((*copybuf_used +
MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
if (dma->dp_pgmap[pidx].pm_pp != NULL) {
i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
dma->dp_pgmap[pidx].pm_kaddr);
} else {
i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
sinfo->si_asp,
dma->dp_pgmap[pidx].pm_kaddr);
}
/*
* we've used up the available copy buffer, this page
* will have to be mapped during rootnex_dma_win() when
* we switch to a new window which requires a re-map
* the copy buffer. (32-bit kernel only)
*/
} else {
dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
}
#endif
/* go to the next page_t */
if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
*cur_pp = (*cur_pp)->p_next;
}
}
/* add to the copy buffer count */
*copybuf_used += MMU_PAGESIZE;
/*
* This cookie doesn't use the copy buffer. Walk through the pages this
* cookie occupies to reflect this.
*/
} else {
/*
* figure out how many pages the cookie occupies. We need to
* use the original page offset of the buffer and the cookies
* offset in the buffer to do this.
*/
poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
pcnt = mmu_btopr(cookie->dmac_size + poff);
while (pcnt > 0) {
#if !defined(__amd64)
/*
* the 32-bit kernel doesn't have seg kpm, so we need
* to map in the driver buffer (if it didn't come down
* with a kernel VA) on the fly. Since this page doesn't
* use the copy buffer, it's not, or will it ever, have
* to be mapped in.
*/
dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
#endif
dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
/*
* we need to update pidx and cur_pp or we'll loose
* track of where we are.
*/
if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
*cur_pp = (*cur_pp)->p_next;
}
pidx++;
pcnt--;
}
}
}
/*
* rootnex_sgllen_window_boundary()
* Called in the bind slow path when the next cookie causes us to exceed (in
* this case == since we start at 0 and sgllen starts at 1) the maximum sgl
* length supported by the DMA H/W.
*/
static int
rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
off_t cur_offset)
{
off_t new_offset;
size_t trim_sz;
off_t coffset;
/*
* if we know we'll never have to trim, it's pretty easy. Just move to
* the next window and init it. We're done.
*/
if (!dma->dp_trim_required) {
(*windowp)++;
rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_size = cookie->dmac_size;
return (DDI_SUCCESS);
}
/* figure out how much we need to trim from the window */
ASSERT(attr->dma_attr_granular != 0);
if (dma->dp_granularity_power_2) {
trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
} else {
trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
}
/* The window's a whole multiple of granularity. We're done */
if (trim_sz == 0) {
(*windowp)++;
rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_size = cookie->dmac_size;
return (DDI_SUCCESS);
}
/*
* The window's not a whole multiple of granularity, since we know this
* is due to the sgllen, we need to go back to the last cookie and trim
* that one, add the left over part of the old cookie into the new
* window, and then add in the new cookie into the new window.
*/
/*
* make sure the driver isn't making us do something bad... Trimming and
* sgllen == 1 don't go together.
*/
if (attr->dma_attr_sgllen == 1) {
return (DDI_DMA_NOMAPPING);
}
/*
* first, setup the current window to account for the trim. Need to go
* back to the last cookie for this.
*/
cookie--;
(*windowp)->wd_trim.tr_trim_last = B_TRUE;
(*windowp)->wd_trim.tr_last_cookie = cookie;
(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
ASSERT(cookie->dmac_size > trim_sz);
(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
(*windowp)->wd_size -= trim_sz;
/* save the buffer offsets for the next window */
coffset = cookie->dmac_size - trim_sz;
new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
/*
* set this now in case this is the first window. all other cases are
* set in dma_win()
*/
cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
/*
* initialize the next window using what's left over in the previous
* cookie.
*/
(*windowp)++;
rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_trim.tr_trim_first = B_TRUE;
(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
(*windowp)->wd_trim.tr_first_size = trim_sz;
if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
(*windowp)->wd_dosync = B_TRUE;
}
/*
* now go back to the current cookie and add it to the new window. set
* the new window size to the what was left over from the previous
* cookie and what's in the current cookie.
*/
cookie++;
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_size = trim_sz + cookie->dmac_size;
/*
* trim plus the next cookie could put us over maxxfer (a cookie can be
* a max size of maxxfer). Handle that case.
*/
if ((*windowp)->wd_size > dma->dp_maxxfer) {
/*
* maxxfer is already a whole multiple of granularity, and this
* trim will be <= the previous trim (since a cookie can't be
* larger than maxxfer). Make things simple here.
*/
trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
(*windowp)->wd_trim.tr_trim_last = B_TRUE;
(*windowp)->wd_trim.tr_last_cookie = cookie;
(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
(*windowp)->wd_size -= trim_sz;
ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
/* save the buffer offsets for the next window */
coffset = cookie->dmac_size - trim_sz;
new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
/* setup the next window */
(*windowp)++;
rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_trim.tr_trim_first = B_TRUE;
(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
coffset;
(*windowp)->wd_trim.tr_first_size = trim_sz;
}
return (DDI_SUCCESS);
}
/*
* rootnex_copybuf_window_boundary()
* Called in bind slowpath when we get to a window boundary because we used
* up all the copy buffer that we have.
*/
static int
rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
size_t *copybuf_used)
{
rootnex_sglinfo_t *sinfo;
off_t new_offset;
size_t trim_sz;
paddr_t paddr;
off_t coffset;
uint_t pidx;
off_t poff;
sinfo = &dma->dp_sglinfo;
/*
* the copy buffer should be a whole multiple of page size. We know that
* this cookie is <= MMU_PAGESIZE.
*/
ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
/*
* from now on, all new windows in this bind need to be re-mapped during
* ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
* space...
*/
#if !defined(__amd64)
dma->dp_cb_remaping = B_TRUE;
#endif
/* reset copybuf used */
*copybuf_used = 0;
/*
* if we don't have to trim (since granularity is set to 1), go to the
* next window and add the current cookie to it. We know the current
* cookie uses the copy buffer since we're in this code path.
*/
if (!dma->dp_trim_required) {
(*windowp)++;
rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
/* Add this cookie to the new window */
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_size += cookie->dmac_size;
*copybuf_used += MMU_PAGESIZE;
return (DDI_SUCCESS);
}
/*
* *** may need to trim, figure it out.
*/
/* figure out how much we need to trim from the window */
if (dma->dp_granularity_power_2) {
trim_sz = (*windowp)->wd_size &
(hp->dmai_attr.dma_attr_granular - 1);
} else {
trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
}
/*
* if the window's a whole multiple of granularity, go to the next
* window, init it, then add in the current cookie. We know the current
* cookie uses the copy buffer since we're in this code path.
*/
if (trim_sz == 0) {
(*windowp)++;
rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
/* Add this cookie to the new window */
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_size += cookie->dmac_size;
*copybuf_used += MMU_PAGESIZE;
return (DDI_SUCCESS);
}
/*
* *** We figured it out, we definitly need to trim
*/
/*
* make sure the driver isn't making us do something bad...
* Trimming and sgllen == 1 don't go together.
*/
if (hp->dmai_attr.dma_attr_sgllen == 1) {
return (DDI_DMA_NOMAPPING);
}
/*
* first, setup the current window to account for the trim. Need to go
* back to the last cookie for this. Some of the last cookie will be in
* the current window, and some of the last cookie will be in the new
* window. All of the current cookie will be in the new window.
*/
cookie--;
(*windowp)->wd_trim.tr_trim_last = B_TRUE;
(*windowp)->wd_trim.tr_last_cookie = cookie;
(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
ASSERT(cookie->dmac_size > trim_sz);
(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
(*windowp)->wd_size -= trim_sz;
/*
* we're trimming the last cookie (not the current cookie). So that
* last cookie may have or may not have been using the copy buffer (
* we know the cookie passed in uses the copy buffer since we're in
* this code path).
*
* If the last cookie doesn't use the copy buffer, nothing special to
* do. However, if it does uses the copy buffer, it will be both the
* last page in the current window and the first page in the next
* window. Since we are reusing the copy buffer (and KVA space on the
* 32-bit kernel), this page will use the end of the copy buffer in the
* current window, and the start of the copy buffer in the next window.
* Track that info... The cookie physical address was already set to
* the copy buffer physical address in setup_cookie..
*/
if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
(*windowp)->wd_size) >> MMU_PAGESHIFT;
(*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
(*windowp)->wd_trim.tr_last_pidx = pidx;
(*windowp)->wd_trim.tr_last_cbaddr =
dma->dp_pgmap[pidx].pm_cbaddr;
#if !defined(__amd64)
(*windowp)->wd_trim.tr_last_kaddr =
dma->dp_pgmap[pidx].pm_kaddr;
#endif
}
/* save the buffer offsets for the next window */
coffset = cookie->dmac_size - trim_sz;
new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
/*
* set this now in case this is the first window. all other cases are
* set in dma_win()
*/
cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
/*
* initialize the next window using what's left over in the previous
* cookie.
*/
(*windowp)++;
rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_trim.tr_trim_first = B_TRUE;
(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
(*windowp)->wd_trim.tr_first_size = trim_sz;
/*
* again, we're tracking if the last cookie uses the copy buffer.
* read the comment above for more info on why we need to track
* additional state.
*
* For the first cookie in the new window, we need reset the physical
* address to DMA into to the start of the copy buffer plus any
* initial page offset which may be present.
*/
if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
(*windowp)->wd_dosync = B_TRUE;
(*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
(*windowp)->wd_trim.tr_first_pidx = pidx;
(*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
poff;
#ifdef __xpv
/*
* If we're dom0, we're using a real device so we need to load
* the cookies with MAs instead of PAs.
*/
(*windowp)->wd_trim.tr_first_paddr =
ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
#else
(*windowp)->wd_trim.tr_first_paddr = paddr;
#endif
#if !defined(__amd64)
(*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
#endif
/* account for the cookie copybuf usage in the new window */
*copybuf_used += MMU_PAGESIZE;
/*
* every piece of code has to have a hack, and here is this
* ones :-)
*
* There is a complex interaction between setup_cookie and the
* copybuf window boundary. The complexity had to be in either
* the maxxfer window, or the copybuf window, and I chose the
* copybuf code.
*
* So in this code path, we have taken the last cookie,
* virtually broken it in half due to the trim, and it happens
* to use the copybuf which further complicates life. At the
* same time, we have already setup the current cookie, which
* is now wrong. More background info: the current cookie uses
* the copybuf, so it is only a page long max. So we need to
* fix the current cookies copy buffer address, physical
* address, and kva for the 32-bit kernel. We due this by
* bumping them by page size (of course, we can't due this on
* the physical address since the copy buffer may not be
* physically contiguous).
*/
cookie++;
dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
#ifdef __xpv
/*
* If we're dom0, we're using a real device so we need to load
* the cookies with MAs instead of PAs.
*/
cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
#else
cookie->dmac_laddress = paddr;
#endif
#if !defined(__amd64)
ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
#endif
} else {
/* go back to the current cookie */
cookie++;
}
/*
* add the current cookie to the new window. set the new window size to
* the what was left over from the previous cookie and what's in the
* current cookie.
*/
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_size = trim_sz + cookie->dmac_size;
ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
/*
* we know that the cookie passed in always uses the copy buffer. We
* wouldn't be here if it didn't.
*/
*copybuf_used += MMU_PAGESIZE;
return (DDI_SUCCESS);
}
/*
* rootnex_maxxfer_window_boundary()
* Called in bind slowpath when we get to a window boundary because we will
* go over maxxfer.
*/
static int
rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
{
size_t dmac_size;
off_t new_offset;
size_t trim_sz;
off_t coffset;
/*
* calculate how much we have to trim off of the current cookie to equal
* maxxfer. We don't have to account for granularity here since our
* maxxfer already takes that into account.
*/
trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
ASSERT(trim_sz <= cookie->dmac_size);
ASSERT(trim_sz <= dma->dp_maxxfer);
/* save cookie size since we need it later and we might change it */
dmac_size = cookie->dmac_size;
/*
* if we're not trimming the entire cookie, setup the current window to
* account for the trim.
*/
if (trim_sz < cookie->dmac_size) {
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_trim.tr_trim_last = B_TRUE;
(*windowp)->wd_trim.tr_last_cookie = cookie;
(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
(*windowp)->wd_size = dma->dp_maxxfer;
/*
* set the adjusted cookie size now in case this is the first
* window. All other windows are taken care of in get win
*/
cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
}
/*
* coffset is the current offset within the cookie, new_offset is the
* current offset with the entire buffer.
*/
coffset = dmac_size - trim_sz;
new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
/* initialize the next window */
(*windowp)++;
rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
(*windowp)->wd_cookie_cnt++;
(*windowp)->wd_size = trim_sz;
if (trim_sz < dmac_size) {
(*windowp)->wd_trim.tr_trim_first = B_TRUE;
(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
coffset;
(*windowp)->wd_trim.tr_first_size = trim_sz;
}
return (DDI_SUCCESS);
}
/*ARGSUSED*/
static int
rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
off_t off, size_t len, uint_t cache_flags)
{
rootnex_sglinfo_t *sinfo;
rootnex_pgmap_t *cbpage;
rootnex_window_t *win;
ddi_dma_impl_t *hp;
rootnex_dma_t *dma;
caddr_t fromaddr;
caddr_t toaddr;
uint_t psize;
off_t offset;
uint_t pidx;
size_t size;
off_t poff;
int e;
hp = (ddi_dma_impl_t *)handle;
dma = (rootnex_dma_t *)hp->dmai_private;
sinfo = &dma->dp_sglinfo;
/*
* if we don't have any windows, we don't need to sync. A copybuf
* will cause us to have at least one window.
*/
if (dma->dp_window == NULL) {
return (DDI_SUCCESS);
}
/* This window may not need to be sync'd */
win = &dma->dp_window[dma->dp_current_win];
if (!win->wd_dosync) {
return (DDI_SUCCESS);
}
/* handle off and len special cases */
if ((off == 0) || (rootnex_sync_ignore_params)) {
offset = win->wd_offset;
} else {
offset = off;
}
if ((len == 0) || (rootnex_sync_ignore_params)) {
size = win->wd_size;
} else {
size = len;
}
/* check the sync args to make sure they make a little sense */
if (rootnex_sync_check_parms) {
e = rootnex_valid_sync_parms(hp, win, offset, size,
cache_flags);
if (e != DDI_SUCCESS) {
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
return (DDI_FAILURE);
}
}
/*
* special case the first page to handle the offset into the page. The
* offset to the current page for our buffer is the offset into the
* first page of the buffer plus our current offset into the buffer
* itself, masked of course.
*/
poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
psize = MIN((MMU_PAGESIZE - poff), size);
/* go through all the pages that we want to sync */
while (size > 0) {
/*
* Calculate the page index relative to the start of the buffer.
* The index to the current page for our buffer is the offset
* into the first page of the buffer plus our current offset
* into the buffer itself, shifted of course...
*/
pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
ASSERT(pidx < sinfo->si_max_pages);
/*
* if this page uses the copy buffer, we need to sync it,
* otherwise, go on to the next page.
*/
cbpage = &dma->dp_pgmap[pidx];
ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
(cbpage->pm_uses_copybuf == B_FALSE));
if (cbpage->pm_uses_copybuf) {
/* cbaddr and kaddr should be page aligned */
ASSERT(((uintptr_t)cbpage->pm_cbaddr &
MMU_PAGEOFFSET) == 0);
ASSERT(((uintptr_t)cbpage->pm_kaddr &
MMU_PAGEOFFSET) == 0);
/*
* if we're copying for the device, we are going to
* copy from the drivers buffer and to the rootnex
* allocated copy buffer.
*/
if (cache_flags == DDI_DMA_SYNC_FORDEV) {
fromaddr = cbpage->pm_kaddr + poff;
toaddr = cbpage->pm_cbaddr + poff;
DTRACE_PROBE2(rootnex__sync__dev,
dev_info_t *, dma->dp_dip, size_t, psize);
/*
* if we're copying for the cpu/kernel, we are going to
* copy from the rootnex allocated copy buffer to the
* drivers buffer.
*/
} else {
fromaddr = cbpage->pm_cbaddr + poff;
toaddr = cbpage->pm_kaddr + poff;
DTRACE_PROBE2(rootnex__sync__cpu,
dev_info_t *, dma->dp_dip, size_t, psize);
}
bcopy(fromaddr, toaddr, psize);
}
/*
* decrement size until we're done, update our offset into the
* buffer, and get the next page size.
*/
size -= psize;
offset += psize;
psize = MIN(MMU_PAGESIZE, size);
/* page offset is zero for the rest of this loop */
poff = 0;
}
return (DDI_SUCCESS);
}
/*
* rootnex_dma_sync()
* called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
* We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
* is set, ddi_dma_sync() returns immediately passing back success.
*/
/*ARGSUSED*/
static int
rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
off_t off, size_t len, uint_t cache_flags)
{
#if !defined(__xpv)
if (IOMMU_USED(rdip)) {
return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
cache_flags));
}
#endif
return (rootnex_coredma_sync(dip, rdip, handle, off, len,
cache_flags));
}
/*
* rootnex_valid_sync_parms()
* checks the parameters passed to sync to verify they are correct.
*/
static int
rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
off_t offset, size_t size, uint_t cache_flags)
{
off_t woffset;
/*
* the first part of the test to make sure the offset passed in is
* within the window.
*/
if (offset < win->wd_offset) {
return (DDI_FAILURE);
}
/*
* second and last part of the test to make sure the offset and length
* passed in is within the window.
*/
woffset = offset - win->wd_offset;
if ((woffset + size) > win->wd_size) {
return (DDI_FAILURE);
}
/*
* if we are sync'ing for the device, the DDI_DMA_WRITE flag should
* be set too.
*/
if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
(hp->dmai_rflags & DDI_DMA_WRITE)) {
return (DDI_SUCCESS);
}
/*
* at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
* should be set. Also DDI_DMA_READ should be set in the flags.
*/
if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
(cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
(hp->dmai_rflags & DDI_DMA_READ)) {
return (DDI_SUCCESS);
}
return (DDI_FAILURE);
}
/*ARGSUSED*/
static int
rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
uint_t *ccountp)
{
rootnex_window_t *window;
rootnex_trim_t *trim;
ddi_dma_impl_t *hp;
rootnex_dma_t *dma;
#if !defined(__amd64)
rootnex_sglinfo_t *sinfo;
rootnex_pgmap_t *pmap;
uint_t pidx;
uint_t pcnt;
off_t poff;
int i;
#endif
hp = (ddi_dma_impl_t *)handle;
dma = (rootnex_dma_t *)hp->dmai_private;
#if !defined(__amd64)
sinfo = &dma->dp_sglinfo;
#endif
/* If we try and get a window which doesn't exist, return failure */
if (win >= hp->dmai_nwin) {
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
return (DDI_FAILURE);
}
/*
* if we don't have any windows, and they're asking for the first
* window, setup the cookie pointer to the first cookie in the bind.
* setup our return values, then increment the cookie since we return
* the first cookie on the stack.
*/
if (dma->dp_window == NULL) {
if (win != 0) {
ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
return (DDI_FAILURE);
}
hp->dmai_cookie = dma->dp_cookies;
*offp = 0;
*lenp = dma->dp_dma.dmao_size;
*ccountp = dma->dp_sglinfo.si_sgl_size;
*cookiep = hp->dmai_cookie[0];
hp->dmai_cookie++;
return (DDI_SUCCESS);
}
/* sync the old window before moving on to the new one */
window = &dma->dp_window[dma->dp_current_win];
if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
DDI_DMA_SYNC_FORCPU);
}
#if !defined(__amd64)
/*
* before we move to the next window, if we need to re-map, unmap all
* the pages in this window.
*/
if (dma->dp_cb_remaping) {
/*
* If we switch to this window again, we'll need to map in
* on the fly next time.
*/
window->wd_remap_copybuf = B_TRUE;
/*
* calculate the page index into the buffer where this window
* starts, and the number of pages this window takes up.
*/
pidx = (sinfo->si_buf_offset + window->wd_offset) >>
MMU_PAGESHIFT;
poff = (sinfo->si_buf_offset + window->wd_offset) &
MMU_PAGEOFFSET;
pcnt = mmu_btopr(window->wd_size + poff);
ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
/* unmap pages which are currently mapped in this window */
for (i = 0; i < pcnt; i++) {
if (dma->dp_pgmap[pidx].pm_mapped) {
hat_unload(kas.a_hat,
dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
HAT_UNLOAD);
dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
}
pidx++;
}
}
#endif
/*
* Move to the new window.
* NOTE: current_win must be set for sync to work right
*/
dma->dp_current_win = win;
window = &dma->dp_window[win];
/* if needed, adjust the first and/or last cookies for trim */
trim = &window->wd_trim;
if (trim->tr_trim_first) {
window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
window->wd_first_cookie->dmac_size = trim->tr_first_size;
#if !defined(__amd64)
window->wd_first_cookie->dmac_type =
(window->wd_first_cookie->dmac_type &
ROOTNEX_USES_COPYBUF) + window->wd_offset;
#endif
if (trim->tr_first_copybuf_win) {
dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
trim->tr_first_cbaddr;
#if !defined(__amd64)
dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
trim->tr_first_kaddr;
#endif
}
}
if (trim->tr_trim_last) {
trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
trim->tr_last_cookie->dmac_size = trim->tr_last_size;
if (trim->tr_last_copybuf_win) {
dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
trim->tr_last_cbaddr;
#if !defined(__amd64)
dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
trim->tr_last_kaddr;
#endif
}
}
/*
* setup the cookie pointer to the first cookie in the window. setup
* our return values, then increment the cookie since we return the
* first cookie on the stack.
*/
hp->dmai_cookie = window->wd_first_cookie;
*offp = window->wd_offset;
*lenp = window->wd_size;
*ccountp = window->wd_cookie_cnt;
*cookiep = hp->dmai_cookie[0];
hp->dmai_cookie++;
#if !defined(__amd64)
/* re-map copybuf if required for this window */
if (dma->dp_cb_remaping) {
/*
* calculate the page index into the buffer where this
* window starts.
*/
pidx = (sinfo->si_buf_offset + window->wd_offset) >>
MMU_PAGESHIFT;
ASSERT(pidx < sinfo->si_max_pages);
/*
* the first page can get unmapped if it's shared with the
* previous window. Even if the rest of this window is already
* mapped in, we need to still check this one.
*/
pmap = &dma->dp_pgmap[pidx];
if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
if (pmap->pm_pp != NULL) {
pmap->pm_mapped = B_TRUE;
i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
} else if (pmap->pm_vaddr != NULL) {
pmap->pm_mapped = B_TRUE;
i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
pmap->pm_kaddr);
}
}
pidx++;
/* map in the rest of the pages if required */
if (window->wd_remap_copybuf) {
window->wd_remap_copybuf = B_FALSE;
/* figure out many pages this window takes up */
poff = (sinfo->si_buf_offset + window->wd_offset) &
MMU_PAGEOFFSET;
pcnt = mmu_btopr(window->wd_size + poff);
ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
/* map pages which require it */
for (i = 1; i < pcnt; i++) {
pmap = &dma->dp_pgmap[pidx];
if (pmap->pm_uses_copybuf) {
ASSERT(pmap->pm_mapped == B_FALSE);
if (pmap->pm_pp != NULL) {
pmap->pm_mapped = B_TRUE;
i86_pp_map(pmap->pm_pp,
pmap->pm_kaddr);
} else if (pmap->pm_vaddr != NULL) {
pmap->pm_mapped = B_TRUE;
i86_va_map(pmap->pm_vaddr,
sinfo->si_asp,
pmap->pm_kaddr);
}
}
pidx++;
}
}
}
#endif
/* if the new window uses the copy buffer, sync it for the device */
if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
DDI_DMA_SYNC_FORDEV);
}
return (DDI_SUCCESS);
}
/*
* rootnex_dma_win()
* called from ddi_dma_getwin()
*/
/*ARGSUSED*/
static int
rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
uint_t *ccountp)
{
#if !defined(__xpv)
if (IOMMU_USED(rdip)) {
return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
cookiep, ccountp));
}
#endif
return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
cookiep, ccountp));
}
/*
* ************************
* obsoleted dma routines
* ************************
*/
/*
* rootnex_dma_map()
* called from ddi_dma_setup()
* NO IOMMU in 32 bit mode. The below routines doesn't work in 64 bit mode.
*/
/* ARGSUSED */
static int
rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip,
struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep)
{
#if defined(__amd64)
/*
* this interface is not supported in 64-bit x86 kernel. See comment in
* rootnex_dma_mctl()
*/
return (DDI_DMA_NORESOURCES);
#else /* 32-bit x86 kernel */
ddi_dma_handle_t *lhandlep;
ddi_dma_handle_t lhandle;
ddi_dma_cookie_t cookie;
ddi_dma_attr_t dma_attr;
ddi_dma_lim_t *dma_lim;
uint_t ccnt;
int e;
/*
* if the driver is just testing to see if it's possible to do the bind,
* we'll use local state. Otherwise, use the handle pointer passed in.
*/
if (handlep == NULL) {
lhandlep = &lhandle;
} else {
lhandlep = handlep;
}
/* convert the limit structure to a dma_attr one */
dma_lim = dmareq->dmar_limits;
dma_attr.dma_attr_version = DMA_ATTR_V0;
dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo;
dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi;
dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer;
dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max;
dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max;
dma_attr.dma_attr_granular = dma_lim->dlim_granular;
dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen;
dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize;
dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes;
dma_attr.dma_attr_align = MMU_PAGESIZE;
dma_attr.dma_attr_flags = 0;
e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp,
dmareq->dmar_arg, lhandlep);
if (e != DDI_SUCCESS) {
return (e);
}
e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt);
if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
(void) rootnex_dma_freehdl(dip, rdip, *lhandlep);
return (e);
}
/*
* if the driver is just testing to see if it's possible to do the bind,
* free up the local state and return the result.
*/
if (handlep == NULL) {
(void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep);
(void) rootnex_dma_freehdl(dip, rdip, *lhandlep);
if (e == DDI_DMA_MAPPED) {
return (DDI_DMA_MAPOK);
} else {
return (DDI_DMA_NOMAPPING);
}
}
return (e);
#endif /* defined(__amd64) */
}
/*
* rootnex_dma_mctl()
*
* No IOMMU in 32 bit mode. The below routine doesn't work in 64 bit mode.
*/
/* ARGSUSED */
static int
rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
uint_t cache_flags)
{
#if defined(__amd64)
/*
* DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a
* common implementation in genunix, so they no longer have x86
* specific functionality which called into dma_ctl.
*
* The rest of the obsoleted interfaces were never supported in the
* 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface
* was not ported to the x86 64-bit kernel do to serious x86 rootnex
* implementation issues.
*
* If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and
* DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we
* reflect that now too...
*
* Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are
* not going to put this functionality into the 64-bit x86 kernel now.
* It wasn't ported to the 64-bit kernel for s10, no reason to change
* that in a future release.
*/
return (DDI_FAILURE);
#else /* 32-bit x86 kernel */
ddi_dma_cookie_t lcookie;
ddi_dma_cookie_t *cookie;
rootnex_window_t *window;
ddi_dma_impl_t *hp;
rootnex_dma_t *dma;
uint_t nwin;
uint_t ccnt;
size_t len;
off_t off;
int e;
/*
* DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little
* hacky since were optimizing for the current interfaces and so we can
* cleanup the mess in genunix. Hopefully we will remove the this
* obsoleted routines someday soon.
*/
switch (request) {
case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */
hp = (ddi_dma_impl_t *)handle;
cookie = (ddi_dma_cookie_t *)objpp;
/*
* convert segment to cookie. We don't distinguish between the
* two :-)
*/
*cookie = *hp->dmai_cookie;
*lenp = cookie->dmac_size;
*offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF;
return (DDI_SUCCESS);
case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */
hp = (ddi_dma_impl_t *)handle;
dma = (rootnex_dma_t *)hp->dmai_private;
if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) {
return (DDI_DMA_STALE);
}
/* handle the case where we don't have any windows */
if (dma->dp_window == NULL) {
/*
* if seg == NULL, and we don't have any windows,
* return the first cookie in the sgl.
*/
if (*lenp == NULL) {
dma->dp_current_cookie = 0;
hp->dmai_cookie = dma->dp_cookies;
*objpp = (caddr_t)handle;
return (DDI_SUCCESS);
/* if we have more cookies, go to the next cookie */
} else {
if ((dma->dp_current_cookie + 1) >=
dma->dp_sglinfo.si_sgl_size) {
return (DDI_DMA_DONE);
}
dma->dp_current_cookie++;
hp->dmai_cookie++;
return (DDI_SUCCESS);
}
}
/* We have one or more windows */
window = &dma->dp_window[dma->dp_current_win];
/*
* if seg == NULL, return the first cookie in the current
* window
*/
if (*lenp == NULL) {
dma->dp_current_cookie = 0;
hp->dmai_cookie = window->wd_first_cookie;
/*
* go to the next cookie in the window then see if we done with
* this window.
*/
} else {
if ((dma->dp_current_cookie + 1) >=
window->wd_cookie_cnt) {
return (DDI_DMA_DONE);
}
dma->dp_current_cookie++;
hp->dmai_cookie++;
}
*objpp = (caddr_t)handle;
return (DDI_SUCCESS);
case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */
hp = (ddi_dma_impl_t *)handle;
dma = (rootnex_dma_t *)hp->dmai_private;
if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) {
return (DDI_DMA_STALE);
}
/* if win == NULL, return the first window in the bind */
if (*offp == NULL) {
nwin = 0;
/*
* else, go to the next window then see if we're done with all
* the windows.
*/
} else {
nwin = dma->dp_current_win + 1;
if (nwin >= hp->dmai_nwin) {
return (DDI_DMA_DONE);
}
}
/* switch to the next window */
e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len,
&lcookie, &ccnt);
ASSERT(e == DDI_SUCCESS);
if (e != DDI_SUCCESS) {
return (DDI_DMA_STALE);
}
/* reset the cookie back to the first cookie in the window */
if (dma->dp_window != NULL) {
window = &dma->dp_window[dma->dp_current_win];
hp->dmai_cookie = window->wd_first_cookie;
} else {
hp->dmai_cookie = dma->dp_cookies;
}
*objpp = (caddr_t)handle;
return (DDI_SUCCESS);
case DDI_DMA_FREE: /* ddi_dma_free() */
(void) rootnex_dma_unbindhdl(dip, rdip, handle);
(void) rootnex_dma_freehdl(dip, rdip, handle);
if (rootnex_state->r_dvma_call_list_id) {
ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
}
return (DDI_SUCCESS);
case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */
case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */
/* should never get here, handled in genunix */
ASSERT(0);
return (DDI_FAILURE);
case DDI_DMA_KVADDR:
case DDI_DMA_GETERR:
case DDI_DMA_COFF:
return (DDI_FAILURE);
}
return (DDI_FAILURE);
#endif /* defined(__amd64) */
}
/*
* *********
* FMA Code
* *********
*/
/*
* rootnex_fm_init()
* FMA init busop
*/
/* ARGSUSED */
static int
rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
ddi_iblock_cookie_t *ibc)
{
*ibc = rootnex_state->r_err_ibc;
return (ddi_system_fmcap);
}
/*
* rootnex_dma_check()
* Function called after a dma fault occurred to find out whether the
* fault address is associated with a driver that is able to handle faults
* and recover from faults.
*/
/* ARGSUSED */
static int
rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
const void *not_used)
{
rootnex_window_t *window;
uint64_t start_addr;
uint64_t fault_addr;
ddi_dma_impl_t *hp;
rootnex_dma_t *dma;
uint64_t end_addr;
size_t csize;
int i;
int j;
/* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
hp = (ddi_dma_impl_t *)handle;
ASSERT(hp);
dma = (rootnex_dma_t *)hp->dmai_private;
/* Get the address that we need to search for */
fault_addr = *(uint64_t *)addr;
/*
* if we don't have any windows, we can just walk through all the
* cookies.
*/
if (dma->dp_window == NULL) {
/* for each cookie */
for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
/*
* if the faulted address is within the physical address
* range of the cookie, return DDI_FM_NONFATAL.
*/
if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
(fault_addr <= (dma->dp_cookies[i].dmac_laddress +
dma->dp_cookies[i].dmac_size))) {
return (DDI_FM_NONFATAL);
}
}
/* fault_addr not within this DMA handle */
return (DDI_FM_UNKNOWN);
}
/* we have mutiple windows, walk through each window */
for (i = 0; i < hp->dmai_nwin; i++) {
window = &dma->dp_window[i];
/* Go through all the cookies in the window */
for (j = 0; j < window->wd_cookie_cnt; j++) {
start_addr = window->wd_first_cookie[j].dmac_laddress;
csize = window->wd_first_cookie[j].dmac_size;
/*
* if we are trimming the first cookie in the window,
* and this is the first cookie, adjust the start
* address and size of the cookie to account for the
* trim.
*/
if (window->wd_trim.tr_trim_first && (j == 0)) {
start_addr = window->wd_trim.tr_first_paddr;
csize = window->wd_trim.tr_first_size;
}
/*
* if we are trimming the last cookie in the window,
* and this is the last cookie, adjust the start
* address and size of the cookie to account for the
* trim.
*/
if (window->wd_trim.tr_trim_last &&
(j == (window->wd_cookie_cnt - 1))) {
start_addr = window->wd_trim.tr_last_paddr;
csize = window->wd_trim.tr_last_size;
}
end_addr = start_addr + csize;
/*
* if the faulted address is within the physical address
* range of the cookie, return DDI_FM_NONFATAL.
*/
if ((fault_addr >= start_addr) &&
(fault_addr <= end_addr)) {
return (DDI_FM_NONFATAL);
}
}
}
/* fault_addr not within this DMA handle */
return (DDI_FM_UNKNOWN);
}