zuluvm.c revision b0fc0e77220f1fa4c933fd58a4e1dedcd650b0f1
d62bc4badc1c1f1549c961cfb8b420e650e1272byz * CDDL HEADER START
75ab5f91d942eea4138efe4799ca0589870c3899lh * The contents of this file are subject to the terms of the
75ab5f91d942eea4138efe4799ca0589870c3899lh * Common Development and Distribution License (the "License").
75ab5f91d942eea4138efe4799ca0589870c3899lh * You may not use this file except in compliance with the License.
75ab5f91d942eea4138efe4799ca0589870c3899lh * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
75ab5f91d942eea4138efe4799ca0589870c3899lh * See the License for the specific language governing permissions
75ab5f91d942eea4138efe4799ca0589870c3899lh * and limitations under the License.
75ab5f91d942eea4138efe4799ca0589870c3899lh * When distributing Covered Code, include this CDDL HEADER in each
75ab5f91d942eea4138efe4799ca0589870c3899lh * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
75ab5f91d942eea4138efe4799ca0589870c3899lh * If applicable, add the following below this CDDL HEADER, with the
75ab5f91d942eea4138efe4799ca0589870c3899lh * fields enclosed by brackets "[]" replaced with your own identifying
75ab5f91d942eea4138efe4799ca0589870c3899lh * information: Portions Copyright [yyyy] [name of copyright owner]
75ab5f91d942eea4138efe4799ca0589870c3899lh * CDDL HEADER END
75ab5f91d942eea4138efe4799ca0589870c3899lh * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
75ab5f91d942eea4138efe4799ca0589870c3899lh * Use is subject to license terms.
75ab5f91d942eea4138efe4799ca0589870c3899lh#pragma ident "%Z%%M% %I% %E% SMI"
75ab5f91d942eea4138efe4799ca0589870c3899lh * zuluvm module
75ab5f91d942eea4138efe4799ca0589870c3899lh * Provides services required by the XVR-4000 graphics accelerator (zulu)
75ab5f91d942eea4138efe4799ca0589870c3899lh * that are not provided by the ddi. See PSARC 2002/231.
75ab5f91d942eea4138efe4799ca0589870c3899lh * Zulu has 2 dma engines with built in MMUs. zuluvm provides TLB miss
75ab5f91d942eea4138efe4799ca0589870c3899lh * interrupt support obtaining virtual to physical address translations
75ab5f91d942eea4138efe4799ca0589870c3899lh * using the XHAT interface PSARC/2003/517.
75ab5f91d942eea4138efe4799ca0589870c3899lh * The module has 3 components. This file, sun4u/vm/zulu_hat.c, and the
75ab5f91d942eea4138efe4799ca0589870c3899lh * assembly language routines in sun4u/ml/zulu_asm.s and
75ab5f91d942eea4138efe4799ca0589870c3899lh * The interrupt handler is a data bearing mondo interrupt handled at TL=1
75ab5f91d942eea4138efe4799ca0589870c3899lh * If no translation is found in the zulu hat's tsb, or if the tsb is locked by
75ab5f91d942eea4138efe4799ca0589870c3899lh * C code, the handler posts a soft interrupt which wakes up a parked
75ab5f91d942eea4138efe4799ca0589870c3899lh * thread belonging to zuludaemon(1M).
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic zuluvm_proc_t *zuluvm_find_proc(zuluvm_state_t *, struct as *);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int zuluvm_proc_release(zuluvm_state_t *zdev, zuluvm_proc_t *proc);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int zuluvm_get_intr_props(zuluvm_state_t *zdev, dev_info_t *devi);
75ab5f91d942eea4138efe4799ca0589870c3899lhextern const unsigned int _mmu_pageshift;
75ab5f91d942eea4138efe4799ca0589870c3899lhunsigned long zuluvm_ctx_locked = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh * Module linkage information for the kernel.
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zulu_hat_init() != 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * currently the kernel driver makes the following assumptions:
75ab5f91d942eea4138efe4799ca0589870c3899lh * - there is only one TLB miss per zulu device handled at
75ab5f91d942eea4138efe4799ca0589870c3899lh * any given time
75ab5f91d942eea4138efe4799ca0589870c3899lh * ==> we only need local data storage per device, not per DMA
75ab5f91d942eea4138efe4799ca0589870c3899lh * ==> a page fault will block the DMA engine until the fault
75ab5f91d942eea4138efe4799ca0589870c3899lh * is resolved
75ab5f91d942eea4138efe4799ca0589870c3899lh * ==> a pagefault will not trigger a zulu DMA context switch
75ab5f91d942eea4138efe4799ca0589870c3899lh * If we want to implement asynnchronous zulu page fault, then we
75ab5f91d942eea4138efe4799ca0589870c3899lh * need to keep track of outstanding faults while zulu DMA runs
75ab5f91d942eea4138efe4799ca0589870c3899lh * in a different context.
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_write_tte(zuluvm_state_t *zdev, void *arg, caddr_t addr,
75ab5f91d942eea4138efe4799ca0589870c3899lh int state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_WRITE_TTE,
75ab5f91d942eea4138efe4799ca0589870c3899lh * if the caller is zuluvm_preload, then we need to pass
75ab5f91d942eea4138efe4799ca0589870c3899lh * back the page size so it can add the right offset.
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void
75ab5f91d942eea4138efe4799ca0589870c3899lh * Executed with the context of the parked zulu deamon thread,
75ab5f91d942eea4138efe4799ca0589870c3899lh * uses zulu_hat_load to resolve the miss.
75ab5f91d942eea4138efe4799ca0589870c3899lh * The tte is loaded and miss done called by the function zuluvm_load_tte
75ab5f91d942eea4138efe4799ca0589870c3899lh * which is called from zulu_hat
75ab5f91d942eea4138efe4799ca0589870c3899lh * This function is synchronized with the zuluvm_as_free.
75ab5f91d942eea4138efe4799ca0589870c3899lh * zuluvm_as_free will block until miss servicing is complete.
75ab5f91d942eea4138efe4799ca0589870c3899lh * There is a race condition between as_free and the zulu tlb miss
75ab5f91d942eea4138efe4799ca0589870c3899lh * soft interrupt:
75ab5f91d942eea4138efe4799ca0589870c3899lh * - queue zulu interrupt
75ab5f91d942eea4138efe4799ca0589870c3899lh * - process dies, as_free runs
75ab5f91d942eea4138efe4799ca0589870c3899lh * - interrupt gets scheduled and runs as_fault on the
75ab5f91d942eea4138efe4799ca0589870c3899lh * already freed as.
75ab5f91d942eea4138efe4799ca0589870c3899lh * This is solved by keeping track of current zulu dma processes
75ab5f91d942eea4138efe4799ca0589870c3899lh * and invalidating them in zuluvm_as_free.
75ab5f91d942eea4138efe4799ca0589870c3899lh * select the correct dma engine and remember the
75ab5f91d942eea4138efe4799ca0589870c3899lh * the as_free synchronization flags.
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_INTR_PENDING,
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_tlb_handler: state %d\n", state);
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_MISS_CANCELED);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lh * block the as_free callback in case it comes in
75ab5f91d942eea4138efe4799ca0589870c3899lh * check if this as is still valid
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc == NULL || proc->valid == 0 || proc->zhat == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * we are on our way out, wake up the as_free
75ab5f91d942eea4138efe4799ca0589870c3899lh * callback if it is waiting for us
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (error) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * trap handler found that zulu_hat had the lock bit set
75ab5f91d942eea4138efe4799ca0589870c3899lh * rather than block in the fast trap handler, it punts
75ab5f91d942eea4138efe4799ca0589870c3899lh * in this rare instance
75ab5f91d942eea4138efe4799ca0589870c3899lh /*FALLTHROUGH*/
75ab5f91d942eea4138efe4799ca0589870c3899lh * fast tlb handler was skipped, see zuluvm_fast_tlb flag
75ab5f91d942eea4138efe4799ca0589870c3899lh /*FALLTHROUGH*/
75ab5f91d942eea4138efe4799ca0589870c3899lh * no TSB entry and TTE in the hash
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lh * error case, fall through and tell zulu driver to abort DMA
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_tlb_handler_state_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh * synchronize with as_free callback
75ab5f91d942eea4138efe4799ca0589870c3899lh * It will set the wait flag, in that case we send
75ab5f91d942eea4138efe4799ca0589870c3899lh * a wake up.
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_load_tte(struct zulu_hat *zhat, caddr_t addr, uint64_t pfn,
75ab5f91d942eea4138efe4799ca0589870c3899lh * synchronize with as_free callback
75ab5f91d942eea4138efe4799ca0589870c3899lh * It will set the wait flag, in that case we send
75ab5f91d942eea4138efe4799ca0589870c3899lh * a wake up.
75ab5f91d942eea4138efe4799ca0589870c3899lh * This function provides the faulting thread for zulu page faults
75ab5f91d942eea4138efe4799ca0589870c3899lh * It is call from the device driver in response to an ioctl issued
75ab5f91d942eea4138efe4799ca0589870c3899lh * by a zuludaemon thread.
75ab5f91d942eea4138efe4799ca0589870c3899lh * It sits in cv_wait_sig until it gets woken up by a signal or
75ab5f91d942eea4138efe4799ca0589870c3899lh * zulu tlb miss soft interrupt.
75ab5f91d942eea4138efe4799ca0589870c3899lh for (;;) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * zulu soft interrupt handler, just triggers the parked zulu fault
75ab5f91d942eea4138efe4799ca0589870c3899lh/*ARGSUSED*/
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_stop(zdev, ZULUVM_STATE_INTR_QUEUED, "fast_intr");
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lh/* ***** public interface for process mapping events (hat layer) ***** */
75ab5f91d942eea4138efe4799ca0589870c3899lh * If the page size matches the Zulu page sizes then just pass
75ab5f91d942eea4138efe4799ca0589870c3899lh * it thru. If not then emulate the page demap with demaps of
75ab5f91d942eea4138efe4799ca0589870c3899lh * smaller page size.
75ab5f91d942eea4138efe4799ca0589870c3899lh/* ARGSUSED */
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_demap_page(void *arg, struct hat *hat_ptr, short ctx,
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < cnt; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_0(zuluvm_demap_page_null_ddarg, "zuluvm", /* */);
75ab5f91d942eea4138efe4799ca0589870c3899lh * An entire context has gone away, just pass it thru
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUVM_MAX_DEV; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < 50; i++)
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUVM_MAX_DEV; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * init the zulu kernel driver (variables, locks, etc)
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUM_MAX_PG_SIZES && size <= ZULU_TTE4M; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * cleanup afterwards
75ab5f91d942eea4138efe4799ca0589870c3899lh * allocate a zulu kernel driver instance for this zulu device
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_alloc_device(dev_info_t *devi, void *arg, zuluvm_info_t *devp,
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.dmv_intr = dmv_add_softintr(zuluvm_dmv_tlbmiss_tl1,
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_set_itlb_pc(zdev, arg, DMV_MAKE_DMV(zdev->zvm.dmv_intr,
75ab5f91d942eea4138efe4799ca0589870c3899lh (void *)zdev));
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_set_dtlb_pc(zdev, arg, DMV_MAKE_DMV(zdev->zvm.dmv_intr,
75ab5f91d942eea4138efe4799ca0589870c3899lh (void *)zdev));
75ab5f91d942eea4138efe4799ca0589870c3899lh * free a zulu kernel driver instance
75ab5f91d942eea4138efe4799ca0589870c3899lh * find the as in the list of active zulu processes
75ab5f91d942eea4138efe4799ca0589870c3899lh * The caller has to hold zdev->proc_lck
75ab5f91d942eea4138efe4799ca0589870c3899lh return (p);
75ab5f91d942eea4138efe4799ca0589870c3899lh * if this entry is still valid, then we need to sync
75ab5f91d942eea4138efe4799ca0589870c3899lh * with zuluvm_tlb_handler rountine.
75ab5f91d942eea4138efe4799ca0589870c3899lh * wait until the tlb miss is resloved
75ab5f91d942eea4138efe4799ca0589870c3899lh * prevent any further tlb miss processing for this hat
75ab5f91d942eea4138efe4799ca0589870c3899lh * decrement the ref count and do the appropriate
75ab5f91d942eea4138efe4799ca0589870c3899lh * if it drops to zero.
75ab5f91d942eea4138efe4799ca0589870c3899lh * notify zulu vm driver about a new process going to
75ab5f91d942eea4138efe4799ca0589870c3899lh * use zulu DMA. Create a zulu_hat.
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_proc_hold(zuluvm_state_t *zdev, zuluvm_proc_t *proc)
75ab5f91d942eea4138efe4799ca0589870c3899lh * decrement ref count and free data if it drops to zero
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_proc_release(zuluvm_state_t *zdev, zuluvm_proc_t *proc)
75ab5f91d942eea4138efe4799ca0589870c3899lh if (refcnt == 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * this process is not longer using DMA, all entries
75ab5f91d942eea4138efe4799ca0589870c3899lh * have been removed from the TLB.
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_dma_delete_proc(zuluvm_info_t devp, uint64_t cookie)
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_dma_delete_proc_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh * barrier sync for device driver
75ab5f91d942eea4138efe4799ca0589870c3899lh * blocks until zuluvm_tlbmiss_tl1 function is done
75ab5f91d942eea4138efe4799ca0589870c3899lh * setup DMA handling for this handle
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_dma_alloc_ctx(zuluvm_info_t devp, int dma, short *mmuctx,
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_dma_alloc_ctx: state %d\n", state);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_STOPPED && state != ZULUVM_STATE_IDLE) {
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (dma) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * preload TLB
75ab5f91d942eea4138efe4799ca0589870c3899lh * this will try to pre-set the zulu tlb, mainly used for dma engine 2,
75ab5f91d942eea4138efe4799ca0589870c3899lh * video read-back.
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (dma) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc == NULL || proc->valid == 0 || proc->zhat == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * need to release this to avoid recursive enter in zuluvm_load_tte
75ab5f91d942eea4138efe4799ca0589870c3899lh * which gets called from zulu_hat_memload()
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < num; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh while (size > 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh * destroy DMA handling for this handle
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_dma_free_ctx: state %d\n", state);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_IDLE && state != ZULUVM_STATE_STOPPED) {
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (state) {
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (dma) {
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUVM_MAX_INTR; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void
75ab5f91d942eea4138efe4799ca0589870c3899lh if (ddi_add_intr(zdev->dip, ino, NULL, NULL, handler, arg)
75ab5f91d942eea4138efe4799ca0589870c3899lh /* remove from distributin list */
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->imr[zdev->interrupts[ino].offset] &= ~ZULUVM_IMR_V_MASK;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm_intr", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm_intr", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->agentid = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUVM_MAX_INTR; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
75ab5f91d942eea4138efe4799ca0589870c3899lh if (nintr == 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < nintr; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh/* *** enf of zulu *** */