zuluvm.c revision b0fc0e77220f1fa4c933fd58a4e1dedcd650b0f1
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
d62bc4badc1c1f1549c961cfb8b420e650e1272byz * CDDL HEADER START
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * The contents of this file are subject to the terms of the
75ab5f91d942eea4138efe4799ca0589870c3899lh * Common Development and Distribution License (the "License").
75ab5f91d942eea4138efe4799ca0589870c3899lh * You may not use this file except in compliance with the License.
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
75ab5f91d942eea4138efe4799ca0589870c3899lh * or http://www.opensolaris.org/os/licensing.
75ab5f91d942eea4138efe4799ca0589870c3899lh * See the License for the specific language governing permissions
75ab5f91d942eea4138efe4799ca0589870c3899lh * and limitations under the License.
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * When distributing Covered Code, include this CDDL HEADER in each
75ab5f91d942eea4138efe4799ca0589870c3899lh * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
75ab5f91d942eea4138efe4799ca0589870c3899lh * If applicable, add the following below this CDDL HEADER, with the
75ab5f91d942eea4138efe4799ca0589870c3899lh * fields enclosed by brackets "[]" replaced with your own identifying
75ab5f91d942eea4138efe4799ca0589870c3899lh * information: Portions Copyright [yyyy] [name of copyright owner]
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * CDDL HEADER END
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
75ab5f91d942eea4138efe4799ca0589870c3899lh * Use is subject to license terms.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh#pragma ident "%Z%%M% %I% %E% SMI"
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * zuluvm module
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * Provides services required by the XVR-4000 graphics accelerator (zulu)
75ab5f91d942eea4138efe4799ca0589870c3899lh * that are not provided by the ddi. See PSARC 2002/231.
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * Zulu has 2 dma engines with built in MMUs. zuluvm provides TLB miss
75ab5f91d942eea4138efe4799ca0589870c3899lh * interrupt support obtaining virtual to physical address translations
75ab5f91d942eea4138efe4799ca0589870c3899lh * using the XHAT interface PSARC/2003/517.
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * The module has 3 components. This file, sun4u/vm/zulu_hat.c, and the
75ab5f91d942eea4138efe4799ca0589870c3899lh * assembly language routines in sun4u/ml/zulu_asm.s and
75ab5f91d942eea4138efe4799ca0589870c3899lh * sun4u/ml/zulu_hat_asm.s.
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * The interrupt handler is a data bearing mondo interrupt handled at TL=1
75ab5f91d942eea4138efe4799ca0589870c3899lh * If no translation is found in the zulu hat's tsb, or if the tsb is locked by
75ab5f91d942eea4138efe4799ca0589870c3899lh * C code, the handler posts a soft interrupt which wakes up a parked
75ab5f91d942eea4138efe4799ca0589870c3899lh * thread belonging to zuludaemon(1M).
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/conf.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/types.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/kmem.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/debug.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/modctl.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/autoconf.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/ddi_impldefs.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/ddi_subrdefs.h>
d62bc4badc1c1f1549c961cfb8b420e650e1272byz#include <sys/intr.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/ddi.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/sunndi.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/proc.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/thread.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/machsystm.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/ivintr.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/tnf_probe.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/intreg.h>
193974072f41a843678abf5f61979c748687e66bSherry Moore#include <sys/atomic.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <vm/as.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <vm/seg_enum.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <vm/faultcode.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/dmv.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/zulumod.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh#include <sys/zulu_hat.h>
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_GET_PAGE(val) \
75ab5f91d942eea4138efe4799ca0589870c3899lh (caddr_t)((uintptr_t)(val) & PAGEMASK)
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_GET_AS curthread->t_procp->p_as
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_LOCK mutex_enter(&(zdev->dev_lck))
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_UNLOCK mutex_exit(&(zdev->dev_lck))
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_SET_STATE(_z, b, c) \
75ab5f91d942eea4138efe4799ca0589870c3899lh cas32((uint32_t *)&((_z)->zvm.state), c, b)
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_GET_STATE(_z) \
75ab5f91d942eea4138efe4799ca0589870c3899lh (_z)->zvm.state
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_SET_IDLE(_z) \
75ab5f91d942eea4138efe4799ca0589870c3899lh (_z)->zvm.state = ZULUVM_STATE_IDLE;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_INO_MASK ((1<<INO_SIZE)-1)
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_IGN_MASK ((1<<IGN_SIZE)-1)
75ab5f91d942eea4138efe4799ca0589870c3899lh#define ZULUVM_MONDO(_zdev, _n) \
75ab5f91d942eea4138efe4799ca0589870c3899lh ((ZULUVM_IGN_MASK & _zdev->agentid) << INO_SIZE) | \
75ab5f91d942eea4138efe4799ca0589870c3899lh (ZULUVM_INO_MASK & (_n))
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void zuluvm_stop(zuluvm_state_t *, int, char *);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic zuluvm_proc_t *zuluvm_find_proc(zuluvm_state_t *, struct as *);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int zuluvm_proc_release(zuluvm_state_t *zdev, zuluvm_proc_t *proc);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int zuluvm_get_intr_props(zuluvm_state_t *zdev, dev_info_t *devi);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int zuluvm_driver_attach(zuluvm_state_t *);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int zuluvm_driver_detach(zuluvm_state_t *);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void zuluvm_retarget_intr(void *arg);
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void zuluvm_do_retarget(zuluvm_state_t *zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhextern const unsigned int _mmu_pageshift;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhextern int zuluvm_base_pgsize;
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int zuluvm_pagesizes[ZULUM_MAX_PG_SIZES + 1];
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhint zuluvm_fast_tlb = 1;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_state_t *zuluvm_devtab[ZULUVM_MAX_DEV];
75ab5f91d942eea4138efe4799ca0589870c3899lhkmutex_t zuluvm_lck;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef DEBUG
75ab5f91d942eea4138efe4799ca0589870c3899lhint zuluvm_debug_state = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhunsigned long zuluvm_ctx_locked = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * Module linkage information for the kernel.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhextern struct mod_ops mod_miscops;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic struct modlmisc modlmisc = {
75ab5f91d942eea4138efe4799ca0589870c3899lh &mod_miscops,
75ab5f91d942eea4138efe4799ca0589870c3899lh "sun4u support " ZULUVM_MOD_VERSION
75ab5f91d942eea4138efe4799ca0589870c3899lh};
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic struct modlinkage modlinkage = {
75ab5f91d942eea4138efe4799ca0589870c3899lh MODREV_1,
75ab5f91d942eea4138efe4799ca0589870c3899lh (void *)&modlmisc,
75ab5f91d942eea4138efe4799ca0589870c3899lh NULL
75ab5f91d942eea4138efe4799ca0589870c3899lh};
193974072f41a843678abf5f61979c748687e66bSherry Moore
193974072f41a843678abf5f61979c748687e66bSherry Mooreint
75ab5f91d942eea4138efe4799ca0589870c3899lh_init(void)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_base_pgsize = (_mmu_pageshift - 13) / 3;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zulu_hat_init() != 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_init(&zuluvm_lck, NULL, MUTEX_DEFAULT, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (mod_install(&modlinkage));
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lh_fini(void)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_destroy(&zuluvm_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) zulu_hat_destroy();
75ab5f91d942eea4138efe4799ca0589870c3899lh return (mod_remove(&modlinkage));
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lh_info(struct modinfo *modinfop)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh return (mod_info(&modlinkage, modinfop));
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * currently the kernel driver makes the following assumptions:
75ab5f91d942eea4138efe4799ca0589870c3899lh * - there is only one TLB miss per zulu device handled at
75ab5f91d942eea4138efe4799ca0589870c3899lh * any given time
75ab5f91d942eea4138efe4799ca0589870c3899lh * ==> we only need local data storage per device, not per DMA
75ab5f91d942eea4138efe4799ca0589870c3899lh * ==> a page fault will block the DMA engine until the fault
75ab5f91d942eea4138efe4799ca0589870c3899lh * is resolved
75ab5f91d942eea4138efe4799ca0589870c3899lh * ==> a pagefault will not trigger a zulu DMA context switch
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * If we want to implement asynnchronous zulu page fault, then we
75ab5f91d942eea4138efe4799ca0589870c3899lh * need to keep track of outstanding faults while zulu DMA runs
75ab5f91d942eea4138efe4799ca0589870c3899lh * in a different context.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_write_tte(zuluvm_state_t *zdev, void *arg, caddr_t addr,
75ab5f91d942eea4138efe4799ca0589870c3899lh int t_pfn, int t_perm, int t_size, uint64_t tag,
75ab5f91d942eea4138efe4799ca0589870c3899lh int tlbtype, int *size)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int error;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) addr;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATS_MISS(zdev, t_size);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (tag == 0) { /* not coming from preload */
75ab5f91d942eea4138efe4799ca0589870c3899lh int state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_WRITE_TTE,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_INTR_PENDING);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_INTR_PENDING) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_stop(zdev, state, "zuluvm_write_tte");
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_MISS_CANCELED);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (!(tlbtype & ZULUVM_ITLB_FLAG) &&
75ab5f91d942eea4138efe4799ca0589870c3899lh t_size != zuluvm_base_pgsize &&
75ab5f91d942eea4138efe4799ca0589870c3899lh t_size != ZULU_TTE4M) {
75ab5f91d942eea4138efe4799ca0589870c3899lh t_size = zuluvm_base_pgsize;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_write_tte_new_pfn, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, t_pfn, t_pfn, tnf_int, pagesize, t_size);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_write_tte, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, t_pfn, t_pfn);
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * if the caller is zuluvm_preload, then we need to pass
75ab5f91d942eea4138efe4799ca0589870c3899lh * back the page size so it can add the right offset.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh if (size)
75ab5f91d942eea4138efe4799ca0589870c3899lh *size = t_size;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh error = zulud_write_tte(zdev, arg, t_size, tag, t_pfn,
75ab5f91d942eea4138efe4799ca0589870c3899lh t_perm, tlbtype);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh return (error);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_stop(zuluvm_state_t *zdev, int state, char *tag)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int ostate = state;
75ab5f91d942eea4138efe4799ca0589870c3899lh while (state != ZULUVM_STATE_STOPPED) {
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_STOPPED, state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef DEBUG
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_debug_state)
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_stop(%s): (loop) state %d\n",
75ab5f91d942eea4138efe4799ca0589870c3899lh tag, state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_stop, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_string, tag, tag,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, state, ostate);
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATS_CANCEL(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * Executed with the context of the parked zulu deamon thread,
75ab5f91d942eea4138efe4799ca0589870c3899lh * uses zulu_hat_load to resolve the miss.
75ab5f91d942eea4138efe4799ca0589870c3899lh * The tte is loaded and miss done called by the function zuluvm_load_tte
75ab5f91d942eea4138efe4799ca0589870c3899lh * which is called from zulu_hat
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * This function is synchronized with the zuluvm_as_free.
75ab5f91d942eea4138efe4799ca0589870c3899lh * zuluvm_as_free will block until miss servicing is complete.
75ab5f91d942eea4138efe4799ca0589870c3899lh *
75ab5f91d942eea4138efe4799ca0589870c3899lh * There is a race condition between as_free and the zulu tlb miss
75ab5f91d942eea4138efe4799ca0589870c3899lh * soft interrupt:
75ab5f91d942eea4138efe4799ca0589870c3899lh * - queue zulu interrupt
75ab5f91d942eea4138efe4799ca0589870c3899lh * - process dies, as_free runs
75ab5f91d942eea4138efe4799ca0589870c3899lh * - interrupt gets scheduled and runs as_fault on the
75ab5f91d942eea4138efe4799ca0589870c3899lh * already freed as.
75ab5f91d942eea4138efe4799ca0589870c3899lh * This is solved by keeping track of current zulu dma processes
75ab5f91d942eea4138efe4799ca0589870c3899lh * and invalidating them in zuluvm_as_free.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhuint_t
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_tlb_handler(caddr_t data)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)data;
75ab5f91d942eea4138efe4799ca0589870c3899lh int error;
75ab5f91d942eea4138efe4799ca0589870c3899lh int flag = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh int wait = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_proc_t *proc = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh struct zulu_hat *zhat = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh caddr_t addr;
75ab5f91d942eea4138efe4799ca0589870c3899lh int tlbtype;
75ab5f91d942eea4138efe4799ca0589870c3899lh void *arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh int state, newstate;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_tlb_handler_lwp, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, lwp, ttolwp(curthread));
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_GET_TLB_ERRCODE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh addr = (caddr_t)ZULUVM_GET_TLB_ADDR(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh tlbtype = ZULUVM_GET_TLB_TYPE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh arg = zdev->zvm.arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * select the correct dma engine and remember the
75ab5f91d942eea4138efe4799ca0589870c3899lh * the as_free synchronization flags.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (tlbtype) {
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_ITLB1:
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA1:
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = zdev->zvm.proc1;
75ab5f91d942eea4138efe4799ca0589870c3899lh flag |= ZULUVM_DO_INTR1;
75ab5f91d942eea4138efe4799ca0589870c3899lh wait |= ZULUVM_WAIT_INTR1;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_ITLB2:
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA2:
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = zdev->zvm.proc2;
75ab5f91d942eea4138efe4799ca0589870c3899lh flag |= ZULUVM_DO_INTR2;
75ab5f91d942eea4138efe4799ca0589870c3899lh wait |= ZULUVM_WAIT_INTR2;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_INTR_PENDING,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_INTR_QUEUED);
75ab5f91d942eea4138efe4799ca0589870c3899lh newstate = ZULUVM_GET_STATE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_tlb_handler_state, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, oldstate, state,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, newstate, newstate);
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef DEBUG
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_debug_state)
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_tlb_handler: state %d\n", state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_INTR_PENDING &&
75ab5f91d942eea4138efe4799ca0589870c3899lh state != ZULUVM_STATE_INTR_QUEUED) {
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_stop(zdev, state, "softintr1");
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_MISS_CANCELED);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * block the as_free callback in case it comes in
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->intr_flags |= flag;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * check if this as is still valid
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc == NULL || proc->valid == 0 || proc->zhat == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * we are on our way out, wake up the as_free
75ab5f91d942eea4138efe4799ca0589870c3899lh * callback if it is waiting for us
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->intr_flags &= ~flag;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->intr_flags | wait)
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_broadcast(&zdev->intr_wait);
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_INTR_PENDING);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_INTR_PENDING) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_stop(zdev, state, "softintr3");
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_NO_HAT);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh zhat = proc->zhat;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_tlb_handler, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, addr, addr);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (error) {
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_CTX_LOCKED:
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * trap handler found that zulu_hat had the lock bit set
75ab5f91d942eea4138efe4799ca0589870c3899lh * rather than block in the fast trap handler, it punts
75ab5f91d942eea4138efe4799ca0589870c3899lh * in this rare instance
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh ++zuluvm_ctx_locked;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_ctx_locked, "zuluvm", /* CSTYLED */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_ulong, zuluvm_ctx_locked, zuluvm_ctx_locked);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh /*FALLTHROUGH*/
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_TTE_DELAY:
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * fast tlb handler was skipped, see zuluvm_fast_tlb flag
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh /*FALLTHROUGH*/
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_NO_TTE:
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * no TSB entry and TTE in the hash
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->load_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->in_intr = 1;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = zulu_hat_load(zhat, addr,
75ab5f91d942eea4138efe4799ca0589870c3899lh (tlbtype == ZULUVM_DMA2) ? S_WRITE : S_READ, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->in_intr = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->load_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (error) {
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_NO_MAP;
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_SUCCESS;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_tlb_handler_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, error);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh default:
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * error case, fall through and tell zulu driver to abort DMA
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (error != ZULUVM_MISS_CANCELED) {
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_WRITE_TTE);
75ab5f91d942eea4138efe4799ca0589870c3899lh newstate = ZULUVM_GET_STATE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_tlb_handler_state_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, oldstate, state,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, newstate, newstate);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_WRITE_TTE) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_stop(zdev, state, "softintr4");
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * synchronize with as_free callback
75ab5f91d942eea4138efe4799ca0589870c3899lh * It will set the wait flag, in that case we send
75ab5f91d942eea4138efe4799ca0589870c3899lh * a wake up.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->intr_flags &= ~flag;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->intr_flags | wait)
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_broadcast(&zdev->intr_wait);
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_tlb_handler_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, error);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_tlb_done(zdev, arg, tlbtype, error);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
22eb7cb54d8a6bcf6fe2674cb4b1f0cf2d85cfb6gd}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhvoid
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_load_tte(struct zulu_hat *zhat, caddr_t addr, uint64_t pfn,
75ab5f91d942eea4138efe4799ca0589870c3899lh int perm, int size)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = zhat->zdev;
75ab5f91d942eea4138efe4799ca0589870c3899lh int tlbtype = ZULUVM_GET_TLB_TYPE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh
22eb7cb54d8a6bcf6fe2674cb4b1f0cf2d85cfb6gd ASSERT(MUTEX_HELD(&zdev->load_lck));
75ab5f91d942eea4138efe4799ca0589870c3899lh ASSERT(pfn != 0);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->in_intr) {
75ab5f91d942eea4138efe4799ca0589870c3899lh int error;
75ab5f91d942eea4138efe4799ca0589870c3899lh int flag = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh int wait = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh error = zuluvm_write_tte(zdev, zdev->zvm.arg, addr, pfn,
75ab5f91d942eea4138efe4799ca0589870c3899lh perm, size, 0, tlbtype, NULL);
22eb7cb54d8a6bcf6fe2674cb4b1f0cf2d85cfb6gd
75ab5f91d942eea4138efe4799ca0589870c3899lh if (error != ZULUVM_MISS_CANCELED) {
75ab5f91d942eea4138efe4799ca0589870c3899lh int state, newstate;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_WRITE_TTE);
75ab5f91d942eea4138efe4799ca0589870c3899lh newstate = ZULUVM_GET_STATE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_tlb_handler_state_done, "zuluvm",
75ab5f91d942eea4138efe4799ca0589870c3899lh /* */, tnf_int, oldstate, state,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, newstate, newstate);
22eb7cb54d8a6bcf6fe2674cb4b1f0cf2d85cfb6gd if (state != ZULUVM_STATE_WRITE_TTE) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_stop(zdev, state, "softintr4");
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * synchronize with as_free callback
75ab5f91d942eea4138efe4799ca0589870c3899lh * It will set the wait flag, in that case we send
75ab5f91d942eea4138efe4799ca0589870c3899lh * a wake up.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (tlbtype) {
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_ITLB1:
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA1:
75ab5f91d942eea4138efe4799ca0589870c3899lh flag = ZULUVM_DO_INTR1;
75ab5f91d942eea4138efe4799ca0589870c3899lh wait = ZULUVM_WAIT_INTR1;
22eb7cb54d8a6bcf6fe2674cb4b1f0cf2d85cfb6gd break;
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_ITLB2:
22eb7cb54d8a6bcf6fe2674cb4b1f0cf2d85cfb6gd case ZULUVM_DMA2:
75ab5f91d942eea4138efe4799ca0589870c3899lh flag = ZULUVM_DO_INTR2;
75ab5f91d942eea4138efe4799ca0589870c3899lh wait = ZULUVM_WAIT_INTR2;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->intr_flags &= ~flag;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->intr_flags | wait)
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_broadcast(&zdev->intr_wait);
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_tlb_done(zdev, zdev->zvm.arg, tlbtype, error);
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) zuluvm_write_tte(zdev, zdev->zvm.arg, addr, pfn,
75ab5f91d942eea4138efe4799ca0589870c3899lh perm, size, (uint64_t)addr |
75ab5f91d942eea4138efe4799ca0589870c3899lh zhat->zulu_ctx, tlbtype, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * This function provides the faulting thread for zulu page faults
75ab5f91d942eea4138efe4799ca0589870c3899lh * It is call from the device driver in response to an ioctl issued
75ab5f91d942eea4138efe4799ca0589870c3899lh * by a zuludaemon thread.
75ab5f91d942eea4138efe4799ca0589870c3899lh * It sits in cv_wait_sig until it gets woken up by a signal or
75ab5f91d942eea4138efe4799ca0589870c3899lh * zulu tlb miss soft interrupt.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_park(zuluvm_info_t devp)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int rval;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->park_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->parking = 1;
75ab5f91d942eea4138efe4799ca0589870c3899lh for (;;) {
75ab5f91d942eea4138efe4799ca0589870c3899lh rval = cv_wait_sig(&zdev->park_cv, &zdev->park_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (rval == 0)
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh rval = zuluvm_tlb_handler(devp);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->parking = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->park_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (rval);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * zulu soft interrupt handler, just triggers the parked zulu fault
75ab5f91d942eea4138efe4799ca0589870c3899lh * thread
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh/*ARGSUSED*/
75ab5f91d942eea4138efe4799ca0589870c3899lhuint_t
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_softintr(caddr_t devp, caddr_t arg2)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int tlbtype;
75ab5f91d942eea4138efe4799ca0589870c3899lh void *arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->park_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->parking) {
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_signal(&zdev->park_cv);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->park_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_fast_intr, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, devp, devp);
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->park_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm: no page fault thread\n");
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh tlbtype = ZULUVM_GET_TLB_TYPE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh arg = zdev->zvm.arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_0(zuluvm_fast_intr, "zuluvm", /* */);
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_stop(zdev, ZULUVM_STATE_INTR_QUEUED, "fast_intr");
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_NO_TTE);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh return (1);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/* ***** public interface for process mapping events (hat layer) ***** */
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * If the page size matches the Zulu page sizes then just pass
75ab5f91d942eea4138efe4799ca0589870c3899lh * it thru. If not then emulate the page demap with demaps of
75ab5f91d942eea4138efe4799ca0589870c3899lh * smaller page size.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh/* ARGSUSED */
75ab5f91d942eea4138efe4799ca0589870c3899lhvoid
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_demap_page(void *arg, struct hat *hat_ptr, short ctx,
75ab5f91d942eea4138efe4799ca0589870c3899lh caddr_t vaddr, uint_t size)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh void *ddarg;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (arg == NULL)
75ab5f91d942eea4138efe4799ca0589870c3899lh return;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATS_DEMAP_PAGE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh ddarg = zdev->zvm.arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_3(zuluvm_demap_page, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, addr, vaddr,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, size, size,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, ctx, ctx);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (ddarg != NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (size != zuluvm_base_pgsize &&
75ab5f91d942eea4138efe4799ca0589870c3899lh size != ZULU_TTE4M) {
75ab5f91d942eea4138efe4799ca0589870c3899lh int i;
75ab5f91d942eea4138efe4799ca0589870c3899lh int cnt = size - zuluvm_base_pgsize;
75ab5f91d942eea4138efe4799ca0589870c3899lh cnt = ZULU_HAT_SZ_SHIFT(cnt);
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < cnt; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh uintptr_t addr = (uintptr_t)vaddr |
75ab5f91d942eea4138efe4799ca0589870c3899lh i << ZULU_HAT_BP_SHIFT;
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_demap_page(zdev, ddarg,
75ab5f91d942eea4138efe4799ca0589870c3899lh (caddr_t)addr, ctx);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_demap_page(zdev, ddarg, vaddr, ctx);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_0(zuluvm_demap_page_done, "zuluvm", /* */);
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_0(zuluvm_demap_page_null_ddarg, "zuluvm", /* */);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * An entire context has gone away, just pass it thru
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhvoid
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_demap_ctx(void *arg, short ctx)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh void *ddarg;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (arg == NULL)
75ab5f91d942eea4138efe4799ca0589870c3899lh return;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATS_DEMAP_CTX(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_demap_ctx, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, ctx, ctx);
75ab5f91d942eea4138efe4799ca0589870c3899lh ddarg = zdev->zvm.arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (ddarg != NULL)
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_demap_ctx(zdev, ddarg, ctx);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_driver_attach(zuluvm_state_t *zdev)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int i;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zuluvm_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUVM_MAX_DEV; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_devtab[i] == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_devtab[i] = zdev;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_SET_IDLE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zuluvm_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (i >= ZULUVM_MAX_DEV)
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zulu_hat_attach((void *)zdev) != 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_init(&zdev->dev_lck, NULL, MUTEX_DEFAULT, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_init(&zdev->load_lck, NULL, MUTEX_DEFAULT, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_init(&zdev->proc_lck, NULL, MUTEX_DEFAULT, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_init(&zdev->park_lck, NULL, MUTEX_DEFAULT, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_init(&zdev->park_cv, NULL, CV_DEFAULT, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_init(&zdev->intr_wait, NULL, CV_DEFAULT, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->parking = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef ZULUVM_STATS
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.cancel = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.pagefault = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.no_mapping = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.preload = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.migrate = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.pagesize = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.tlb_miss[0] = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.tlb_miss[1] = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.tlb_miss[2] = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.tlb_miss[3] = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.itlb1miss = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.dtlb1miss = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.itlb2miss = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.dtlb2miss = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.pfncnt = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < 50; i++)
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.pfnbuf[i] = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.mmu_pa = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.proc1 = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.proc2 = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->procs = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_driver_detach(zuluvm_state_t *zdev)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int i;
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_destroy(&zdev->intr_wait);
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_destroy(&zdev->park_cv);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_destroy(&zdev->park_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_destroy(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_destroy(&zdev->dev_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_destroy(&zdev->load_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->dops = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zuluvm_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUVM_MAX_DEV; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_devtab[i] == zdev) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_devtab[i] = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zuluvm_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zulu_hat_detach((void *)zdev) == 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhzulud_ops_t *zuluvm_dops = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * init the zulu kernel driver (variables, locks, etc)
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_init(zulud_ops_t *ops, int **pagesizes)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int error = ZULUVM_SUCCESS;
75ab5f91d942eea4138efe4799ca0589870c3899lh int i;
75ab5f91d942eea4138efe4799ca0589870c3899lh int size = zuluvm_base_pgsize; /* MMU_PAGESIZE; */
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (ops->version != ZULUVM_INTERFACE_VERSION)
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_VERSION_MISMATCH);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_dops = ops;
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUM_MAX_PG_SIZES && size <= ZULU_TTE4M; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_pagesizes[i] = size++;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_pagesizes[i] = -1;
75ab5f91d942eea4138efe4799ca0589870c3899lh *pagesizes = zuluvm_pagesizes;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh return (error);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * cleanup afterwards
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_fini(void)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_dops = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * allocate a zulu kernel driver instance for this zulu device
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_alloc_device(dev_info_t *devi, void *arg, zuluvm_info_t *devp,
75ab5f91d942eea4138efe4799ca0589870c3899lh caddr_t mmu, caddr_t imr)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh uint64_t intr_num;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev;
75ab5f91d942eea4138efe4799ca0589870c3899lh int error = ZULUVM_SUCCESS;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_3(zuluvm_alloc_device, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, arg, arg,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, mmu, mmu,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, imr, imr);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev = kmem_zalloc(sizeof (zuluvm_state_t), KM_SLEEP);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->dip = devi;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->dops = zuluvm_dops;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = zuluvm_driver_attach(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (error != ZULUVM_SUCCESS) {
75ab5f91d942eea4138efe4799ca0589870c3899lh kmem_free(zdev, sizeof (zuluvm_state_t));
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = zuluvm_get_intr_props(zdev, devi);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (error != ZULUVM_SUCCESS) {
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = zuluvm_driver_detach(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (error != ZULUVM_SUCCESS)
75ab5f91d942eea4138efe4799ca0589870c3899lh return (error);
75ab5f91d942eea4138efe4799ca0589870c3899lh kmem_free(zdev, sizeof (zuluvm_state_t));
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.arg = arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.mmu_pa = (uint64_t)va_to_pa((void *)mmu);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->imr = (uint64_t *)imr;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.dmv_intr = dmv_add_softintr(zuluvm_dmv_tlbmiss_tl1,
75ab5f91d942eea4138efe4799ca0589870c3899lh (void *)zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_set_itlb_pc(zdev, arg, DMV_MAKE_DMV(zdev->zvm.dmv_intr,
75ab5f91d942eea4138efe4799ca0589870c3899lh (void *)zdev));
75ab5f91d942eea4138efe4799ca0589870c3899lh zulud_set_dtlb_pc(zdev, arg, DMV_MAKE_DMV(zdev->zvm.dmv_intr,
75ab5f91d942eea4138efe4799ca0589870c3899lh (void *)zdev));
75ab5f91d942eea4138efe4799ca0589870c3899lh intr_dist_add(zuluvm_retarget_intr, (void *)zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_do_retarget(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh intr_num = add_softintr(ZULUVM_PIL, zuluvm_softintr,
75ab5f91d942eea4138efe4799ca0589870c3899lh (caddr_t)zdev, SOFTINT_ST);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.intr_num = intr_num;
75ab5f91d942eea4138efe4799ca0589870c3899lh *devp = (caddr_t)zdev;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_alloc_device_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, devp, *devp);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * free a zulu kernel driver instance
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_free_device(zuluvm_info_t devp)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int error;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_free_device, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, zdev, zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev == NULL)
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->zvm.arg == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_free_device_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) dmv_rem_intr(zdev->zvm.dmv_intr);
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) rem_softintr(zdev->zvm.intr_num);
75ab5f91d942eea4138efe4799ca0589870c3899lh intr_dist_rem(zuluvm_retarget_intr, (void *)zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.arg = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = zuluvm_driver_detach(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (error != ZULUVM_SUCCESS)
75ab5f91d942eea4138efe4799ca0589870c3899lh return (error);
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->dops = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh kmem_free(zdev, sizeof (zuluvm_state_t));
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_0(zuluvm_free_device_done, "zuluvm", /* */);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * find the as in the list of active zulu processes
75ab5f91d942eea4138efe4799ca0589870c3899lh * The caller has to hold zdev->proc_lck
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic zuluvm_proc_t *
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_find_proc(zuluvm_state_t *zdev, struct as *asp)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_proc_t *p;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_find_proc, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, zdev, zdev,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, asp, asp);
75ab5f91d942eea4138efe4799ca0589870c3899lh for (p = zdev->procs; p != NULL; p = p->next) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (ZULU_HAT2AS(p->zhat) == asp) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_find_proc_done,
75ab5f91d942eea4138efe4799ca0589870c3899lh "zuluvm", /* */, tnf_opaque, proc, p);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (p);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_0(zuluvm_find_proc_fail, "zuluvm", /* */);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhvoid
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_as_free(struct as *as, void *arg, uint_t events)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_proc_t *proc = (zuluvm_proc_t *)arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = proc->zdev;
75ab5f91d942eea4138efe4799ca0589870c3899lh int wait = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh int flag = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh int valid;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) events;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_as_free, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, arg, arg);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) as_delete_callback(as, arg);
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * if this entry is still valid, then we need to sync
75ab5f91d942eea4138efe4799ca0589870c3899lh * with zuluvm_tlb_handler rountine.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh valid = proc->valid;
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->valid = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (valid) {
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc == zdev->zvm.proc1) {
75ab5f91d942eea4138efe4799ca0589870c3899lh flag |= ZULUVM_WAIT_INTR1;
75ab5f91d942eea4138efe4799ca0589870c3899lh wait |= ZULUVM_DO_INTR1;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc == zdev->zvm.proc2) {
75ab5f91d942eea4138efe4799ca0589870c3899lh flag |= ZULUVM_WAIT_INTR2;
75ab5f91d942eea4138efe4799ca0589870c3899lh wait |= ZULUVM_DO_INTR2;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh if (flag) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->intr_flags |= flag;
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * wait until the tlb miss is resloved
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh while (zdev->intr_flags & wait) {
75ab5f91d942eea4138efe4799ca0589870c3899lh cv_wait(&zdev->intr_wait, &zdev->dev_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->intr_flags &= ~flag;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc->zhat != NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * prevent any further tlb miss processing for this hat
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh zulu_hat_terminate(proc->zhat);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * decrement the ref count and do the appropriate
75ab5f91d942eea4138efe4799ca0589870c3899lh * if it drops to zero.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) zuluvm_proc_release(zdev, proc);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * notify zulu vm driver about a new process going to
75ab5f91d942eea4138efe4799ca0589870c3899lh * use zulu DMA. Create a zulu_hat.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_dma_add_proc(zuluvm_info_t devp, uint64_t *cookie)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_proc_t *proc;
75ab5f91d942eea4138efe4799ca0589870c3899lh int refcnt;
75ab5f91d942eea4138efe4799ca0589870c3899lh struct as *asp = ZULUVM_GET_AS;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_dma_add_proc, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, zdev, zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = zuluvm_find_proc(zdev, asp);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = kmem_zalloc(sizeof (zuluvm_proc_t), KM_SLEEP);
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->zhat = zulu_hat_proc_attach(asp, zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc->zhat == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh kmem_free(proc, sizeof (zuluvm_proc_t));
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, valid, 0,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->zdev = zdev;
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->valid = 1;
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->refcnt = 1;
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->next = zdev->procs;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->procs)
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->procs->prev = proc;
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->prev = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->procs = proc;
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->refcnt++;
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) as_add_callback(asp, zuluvm_as_free, proc,
75ab5f91d942eea4138efe4799ca0589870c3899lh AS_FREE_EVENT, 0, -1, KM_SLEEP);
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc->valid == 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, valid, 0,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->refcnt++;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh refcnt = proc->refcnt;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh *cookie = (uint64_t)proc;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, refcnt, refcnt,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhvoid
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_proc_hold(zuluvm_state_t *zdev, zuluvm_proc_t *proc)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->refcnt++;
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * decrement ref count and free data if it drops to zero
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_proc_release(zuluvm_state_t *zdev, zuluvm_proc_t *proc)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int refcnt;
75ab5f91d942eea4138efe4799ca0589870c3899lh ASSERT(MUTEX_HELD(&zdev->proc_lck));
75ab5f91d942eea4138efe4799ca0589870c3899lh refcnt = --proc->refcnt;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_3(zuluvm_proc_release, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, zdev, zdev,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, proc, proc,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, refcnt, refcnt);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (refcnt == 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc->next)
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->next->prev = proc->prev;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc->prev)
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->prev->next = proc->next;
75ab5f91d942eea4138efe4799ca0589870c3899lh else
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->procs = proc->next;
75ab5f91d942eea4138efe4799ca0589870c3899lh kmem_free(proc, sizeof (zuluvm_proc_t));
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh return (refcnt);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * this process is not longer using DMA, all entries
75ab5f91d942eea4138efe4799ca0589870c3899lh * have been removed from the TLB.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_dma_delete_proc(zuluvm_info_t devp, uint64_t cookie)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int refcnt;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_proc_t *proc = (zuluvm_proc_t *)cookie;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_dma_delete_proc, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, zdev, zdev,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, cookie, cookie);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc != NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_dma_delete_proc, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, proc, proc);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc->zhat != NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zulu_hat_proc_detach(proc->zhat);
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->zhat = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh refcnt = zuluvm_proc_release(zdev, proc);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_dma_delete_proc_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, refcnt, refcnt,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * barrier sync for device driver
75ab5f91d942eea4138efe4799ca0589870c3899lh * blocks until zuluvm_tlbmiss_tl1 function is done
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhvoid
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_fast_tlb_wait(caddr_t devp)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int state;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh int cnt = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh do {
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_GET_STATE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh cnt++;
75ab5f91d942eea4138efe4799ca0589870c3899lh } while (state == ZULUVM_STATE_TLB_PENDING);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_fast_tlb_wait, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, loop_cnt, cnt);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * setup DMA handling for this handle
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_dma_alloc_ctx(zuluvm_info_t devp, int dma, short *mmuctx,
75ab5f91d942eea4138efe4799ca0589870c3899lh uint64_t *tsbreg)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh struct as *asp = ZULUVM_GET_AS;
75ab5f91d942eea4138efe4799ca0589870c3899lh int error = ZULUVM_NO_DEV;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh int state, newstate;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (asp == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_dma_alloc_ctx_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_NO_HAT);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_HAT);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh *tsbreg = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_STOPPED);
75ab5f91d942eea4138efe4799ca0589870c3899lh newstate = ZULUVM_GET_STATE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_4(zuluvm_dma_alloc_ctx, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, devp, devp,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, dma, dma,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, oldstate, state,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, newstate, newstate);
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef DEBUG
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_debug_state)
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_dma_alloc_ctx: state %d\n", state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_STOPPED && state != ZULUVM_STATE_IDLE) {
75ab5f91d942eea4138efe4799ca0589870c3899lh while (state != ZULUVM_STATE_IDLE) {
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_STOPPED);
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef DEBUG
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_debug_state)
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_dma_alloc_ctx: (loop)"
75ab5f91d942eea4138efe4799ca0589870c3899lh " state %d\n", state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_IDLE)
75ab5f91d942eea4138efe4799ca0589870c3899lh delay(1);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->zvm.arg != NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh struct zulu_hat *zhat;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_proc_t *proc;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = zuluvm_find_proc(zdev, asp);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc != NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zhat = proc->zhat;
75ab5f91d942eea4138efe4799ca0589870c3899lh proc->refcnt++;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (dma) {
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA1:
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.proc1 = proc;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_SUCCESS;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA2:
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.proc2 = proc;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_SUCCESS;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh default:
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) zuluvm_proc_release(zdev, proc);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (error == ZULUVM_SUCCESS) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zulu_hat_validate_ctx(zhat);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zhat->zulu_ctx >= 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh *mmuctx = zhat->zulu_ctx;
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh printf("invalid context value: %d\n",
75ab5f91d942eea4138efe4799ca0589870c3899lh zhat->zulu_ctx);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) zuluvm_proc_release(zdev, proc);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_ERROR;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_ERROR;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_dma_alloc_ctx_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, error);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (error);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * preload TLB
75ab5f91d942eea4138efe4799ca0589870c3899lh * this will try to pre-set the zulu tlb, mainly used for dma engine 2,
75ab5f91d942eea4138efe4799ca0589870c3899lh * video read-back.
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_dma_preload(zuluvm_info_t devp, int dma,
75ab5f91d942eea4138efe4799ca0589870c3899lh int num, zulud_preload_t *list)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int i;
75ab5f91d942eea4138efe4799ca0589870c3899lh int error = ZULUVM_SUCCESS;
75ab5f91d942eea4138efe4799ca0589870c3899lh struct zulu_hat *zhat;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_proc_t *proc = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_4(zuluvm_dma_preload, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, devp, devp,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, dma, dma,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, num, num,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, list, list);
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (dma) {
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA1:
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = zdev->zvm.proc1;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA2:
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = zdev->zvm.proc2;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc == NULL || proc->valid == 0 || proc->zhat == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_HAT);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zhat = proc->zhat;
75ab5f91d942eea4138efe4799ca0589870c3899lh /*
75ab5f91d942eea4138efe4799ca0589870c3899lh * need to release this to avoid recursive enter in zuluvm_load_tte
75ab5f91d942eea4138efe4799ca0589870c3899lh * which gets called from zulu_hat_memload()
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->load_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < num; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh int pg_size;
75ab5f91d942eea4138efe4799ca0589870c3899lh int res;
75ab5f91d942eea4138efe4799ca0589870c3899lh int first = 1;
75ab5f91d942eea4138efe4799ca0589870c3899lh caddr_t addr = ZULUVM_GET_PAGE(list[i].addr);
75ab5f91d942eea4138efe4799ca0589870c3899lh int64_t size = (int64_t)list[i].len;
75ab5f91d942eea4138efe4799ca0589870c3899lh while (size > 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (list[i].tlbtype & ~ZULUVM_DMA_MASK) {
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_INVALID_MISS;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh res = zulu_hat_load(zhat, addr,
75ab5f91d942eea4138efe4799ca0589870c3899lh (list[i].tlbtype == ZULUVM_DMA2) ? S_WRITE : S_READ,
75ab5f91d942eea4138efe4799ca0589870c3899lh &pg_size);
75ab5f91d942eea4138efe4799ca0589870c3899lh if ((res != 0) || (pg_size < 0)) {
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_NO_MAP;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATS_PRELOAD(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_dma_preload_addr, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, addr, addr,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, size, size);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (first) {
75ab5f91d942eea4138efe4799ca0589870c3899lh first = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh size -= ZULU_HAT_PGDIFF(list[i].addr,
75ab5f91d942eea4138efe4799ca0589870c3899lh pg_size);
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh size -= ZULU_HAT_PGSZ(pg_size);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh addr += ZULU_HAT_PGSZ(pg_size);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->load_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_dma_preload_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, error);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/*
75ab5f91d942eea4138efe4799ca0589870c3899lh * destroy DMA handling for this handle
75ab5f91d942eea4138efe4799ca0589870c3899lh */
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_dma_free_ctx(zuluvm_info_t devp, int dma)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int error = ZULUVM_NO_DEV;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh int state, newstate;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_STOPPED,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_IDLE);
75ab5f91d942eea4138efe4799ca0589870c3899lh newstate = ZULUVM_GET_STATE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_4(zuluvm_dma_free_ctx, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, devp, devp,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, dma, dma,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, oldstate, state,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, newstate, newstate);
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef DEBUG
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_debug_state)
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_dma_free_ctx: state %d\n", state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_IDLE && state != ZULUVM_STATE_STOPPED) {
75ab5f91d942eea4138efe4799ca0589870c3899lh int doit = 1;
22eb7cb54d8a6bcf6fe2674cb4b1f0cf2d85cfb6gd while (doit) {
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (state) {
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_STATE_CANCELED:
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_STATE_STOPPED:
75ab5f91d942eea4138efe4799ca0589870c3899lh doit = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_STATE_IDLE:
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_STOPPED,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_IDLE);
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh default:
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_SET_STATE(zdev,
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_STATE_CANCELED, state);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_dma_free_ctx, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, state, state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef DEBUG
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_debug_state)
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_dma_free_ctx: (loop1)"
75ab5f91d942eea4138efe4799ca0589870c3899lh " state %d\n", state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_dma_free_ctx, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, state, state);
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_SUCCESS;
75ab5f91d942eea4138efe4799ca0589870c3899lh while (state != ZULUVM_STATE_STOPPED) {
75ab5f91d942eea4138efe4799ca0589870c3899lh state = ZULUVM_GET_STATE(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh#ifdef DEBUG
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zuluvm_debug_state)
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_NOTE, "zuluvm_dma_free: (loop2) state %d\n",
75ab5f91d942eea4138efe4799ca0589870c3899lh state);
75ab5f91d942eea4138efe4799ca0589870c3899lh#endif
75ab5f91d942eea4138efe4799ca0589870c3899lh if (state != ZULUVM_STATE_STOPPED)
75ab5f91d942eea4138efe4799ca0589870c3899lh delay(1);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->zvm.arg != NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_proc_t *proc = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh switch (dma) {
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA1:
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = zdev->zvm.proc1;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.proc1 = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh case ZULUVM_DMA2:
75ab5f91d942eea4138efe4799ca0589870c3899lh proc = zdev->zvm.proc2;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->zvm.proc2 = NULL;
75ab5f91d942eea4138efe4799ca0589870c3899lh break;
75ab5f91d942eea4138efe4799ca0589870c3899lh default:
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_NO_DEV;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (proc) {
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_enter(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh (void) zuluvm_proc_release(zdev, proc);
75ab5f91d942eea4138efe4799ca0589870c3899lh mutex_exit(&zdev->proc_lck);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh error = ZULUVM_NO_DEV;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_dma_free_ctx_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, error);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (error);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_do_retarget(zuluvm_state_t *zdev)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int i, idx;
75ab5f91d942eea4138efe4799ca0589870c3899lh uint_t cpu;
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUVM_MAX_INTR; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->interrupts[i].ino != -1) {
75ab5f91d942eea4138efe4799ca0589870c3899lh cpu = intr_dist_cpuid();
75ab5f91d942eea4138efe4799ca0589870c3899lh idx = zdev->interrupts[i].offset;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->imr[idx] & ZULUVM_IMR_V_MASK)
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->imr[idx] = ZULUVM_IMR_V_MASK |
75ab5f91d942eea4138efe4799ca0589870c3899lh (cpu<<ZULUVM_IMR_TARGET_SHIFT);
75ab5f91d942eea4138efe4799ca0589870c3899lh else
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->imr[idx] =
75ab5f91d942eea4138efe4799ca0589870c3899lh cpu<<ZULUVM_IMR_TARGET_SHIFT;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic void
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_retarget_intr(void *arg)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)arg;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_do_retarget(zdev);
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_add_intr(zuluvm_info_t devp, int ino,
75ab5f91d942eea4138efe4799ca0589870c3899lh uint_t (*handler)(caddr_t), caddr_t arg)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (devp == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_add_intr_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh if (ddi_add_intr(zdev->dip, ino, NULL, NULL, handler, arg)
75ab5f91d942eea4138efe4799ca0589870c3899lh != DDI_SUCCESS) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_add_intr_done, "zuluvm", /* */,
22eb7cb54d8a6bcf6fe2674cb4b1f0cf2d85cfb6gd tnf_int, error, ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_rem_intr(zuluvm_info_t devp, int ino)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh if (devp == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_rem_intr_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh /* remove from distributin list */
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->imr[zdev->interrupts[ino].offset] &= ~ZULUVM_IMR_V_MASK;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_remove_intr(zdev->dip, ino, NULL);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_enable_intr(zuluvm_info_t devp, int num)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_enable_intr, "zuluvm_intr", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, devp, devp,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, num, num);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (devp == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh if (num < 0 || num > ZULUVM_IMR_MAX) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_BAD_IDX);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_BAD_IDX);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->imr[num] |= ZULUVM_IMR_V_MASK;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm_intr", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhint
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_disable_intr(zuluvm_info_t devp, int num)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh zuluvm_state_t *zdev = (zuluvm_state_t *)devp;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_2(zuluvm_disable_intr, "zuluvm_intr", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_opaque, devp, devp,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, num, num);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (devp == NULL) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_NO_DEV);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh if (num < 0 || num > ZULUVM_IMR_MAX) {
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_BAD_IDX);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_BAD_IDX);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_LOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->imr[num] &= ~ZULUVM_IMR_V_MASK;
75ab5f91d942eea4138efe4799ca0589870c3899lh ZULUVM_UNLOCK;
75ab5f91d942eea4138efe4799ca0589870c3899lh TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm_intr", /* */,
75ab5f91d942eea4138efe4799ca0589870c3899lh tnf_int, error, ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lhstatic int
75ab5f91d942eea4138efe4799ca0589870c3899lhzuluvm_get_intr_props(zuluvm_state_t *zdev,
75ab5f91d942eea4138efe4799ca0589870c3899lh dev_info_t *devi)
75ab5f91d942eea4138efe4799ca0589870c3899lh{
75ab5f91d942eea4138efe4799ca0589870c3899lh int *intr;
75ab5f91d942eea4138efe4799ca0589870c3899lh int i;
75ab5f91d942eea4138efe4799ca0589870c3899lh uint_t nintr;
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->agentid = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
75ab5f91d942eea4138efe4799ca0589870c3899lh "portid", -1);
75ab5f91d942eea4138efe4799ca0589870c3899lh if (zdev->agentid == -1) {
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_WARN, "%s%d: no portid property",
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_get_name(devi),
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_get_instance(devi));
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < ZULUVM_MAX_INTR; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->interrupts[i].offset = 0;
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->interrupts[i].ino = -1;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
75ab5f91d942eea4138efe4799ca0589870c3899lh "interrupts", &intr, &nintr) == DDI_PROP_SUCCESS) {
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh if (nintr == 0) {
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_WARN, "%s%d: no interrupts in property",
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_get_name(devi),
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_get_instance(devi));
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_prop_free(intr);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh if (nintr >= ZULUVM_MAX_INTR) {
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_WARN, "%s%d: to many interrupts (%d)",
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_get_name(devi),
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_get_instance(devi), nintr);
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_prop_free(intr);
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_ERROR);
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh for (i = 0; i < nintr; i++) {
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->interrupts[i].offset = intr[i];
75ab5f91d942eea4138efe4799ca0589870c3899lh zdev->interrupts[i].ino = i;
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_prop_free(intr);
75ab5f91d942eea4138efe4799ca0589870c3899lh } else {
75ab5f91d942eea4138efe4799ca0589870c3899lh cmn_err(CE_WARN, "%s%d: no interrupts property",
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_get_name(devi),
75ab5f91d942eea4138efe4799ca0589870c3899lh ddi_get_instance(devi));
75ab5f91d942eea4138efe4799ca0589870c3899lh }
75ab5f91d942eea4138efe4799ca0589870c3899lh return (ZULUVM_SUCCESS);
75ab5f91d942eea4138efe4799ca0589870c3899lh}
75ab5f91d942eea4138efe4799ca0589870c3899lh
75ab5f91d942eea4138efe4799ca0589870c3899lh/* *** enf of zulu *** */
75ab5f91d942eea4138efe4799ca0589870c3899lh