trap_table.s revision 1bd453f385f392a0415fad0b14efc9f5a545320f
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
#if !defined(lint)
#include "assym.h"
#endif /* !lint */
#include <sys/asm_linkage.h>
#include <sys/privregs.h>
#include <sys/sun4asi.h>
#include <sys/cheetahregs.h>
#include <sys/machtrap.h>
#include <sys/machthread.h>
#include <sys/pcb.h>
#include <sys/pte.h>
#include <sys/mmu.h>
#include <sys/machpcb.h>
#include <sys/async.h>
#include <sys/intreg.h>
#include <sys/scb.h>
#include <sys/psr_compat.h>
#include <sys/syscall.h>
#include <sys/machparam.h>
#include <sys/traptrace.h>
#include <vm/hat_sfmmu.h>
#include <sys/archsystm.h>
#include <sys/utrap.h>
#include <sys/clock.h>
#include <sys/intr.h>
#include <sys/fpu/fpu_simulator.h>
#include <vm/seg_spt.h>
/*
* WARNING: If you add a fast trap handler which can be invoked by a
* non-privileged user, you may have to use the FAST_TRAP_DONE macro
* instead of "done" instruction to return back to the user mode. See
* comments for the "fast_trap_done" entry point for more information.
*
* An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
* cases where you always want to process any pending interrupts before
* returning back to the user mode.
*/
#define FAST_TRAP_DONE \
ba,a fast_trap_done
#define FAST_TRAP_DONE_CHK_INTR \
ba,a fast_trap_done_chk_intr
/*
* SPARC V9 Trap Table
*
* Most of the trap handlers are made from common building
* blocks, and some are instantiated multiple times within
* the trap table. So, I build a bunch of macros, then
* populate the table using only the macros.
*
* Many macros branch to sys_trap. Its calling convention is:
* %g1 kernel trap handler
* %g2, %g3 args for above
* %g4 desire %pil
*/
#ifdef TRAPTRACE
/*
* Tracing macro. Adds two instructions if TRAPTRACE is defined.
*/
#define TT_TRACE(label) \
ba label ;\
rd %pc, %g7
#define TT_TRACE_INS 2
#define TT_TRACE_L(label) \
ba label ;\
rd %pc, %l4 ;\
clr %l4
#define TT_TRACE_L_INS 3
#else
#define TT_TRACE(label)
#define TT_TRACE_INS 0
#define TT_TRACE_L(label)
#define TT_TRACE_L_INS 0
#endif
/*
* This macro is used to update per cpu mmu stats in perf critical
* paths. It is only enabled in debug kernels or if SFMMU_STAT_GATHER
* is defined.
*/
#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
#define HAT_PERCPU_DBSTAT(stat) \
mov stat, %g1 ;\
ba stat_mmu ;\
rd %pc, %g7
#else
#define HAT_PERCPU_DBSTAT(stat)
#endif /* DEBUG || SFMMU_STAT_GATHER */
/*
* This first set are funneled to trap() with %tt as the type.
* Trap will then either panic or send the user a signal.
*/
/*
* NOT is used for traps that just shouldn't happen.
* It comes in both single and quadruple flavors.
*/
#if !defined(lint)
.global trap
#endif /* !lint */
#define NOT \
TT_TRACE(trace_gen) ;\
set trap, %g1 ;\
rdpr %tt, %g3 ;\
ba,pt %xcc, sys_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define NOT4 NOT; NOT; NOT; NOT
/*
* RED is for traps that use the red mode handler.
* We should never see these either.
*/
#define RED NOT
/*
* BAD is used for trap vectors we don't have a kernel
* handler for.
* It also comes in single and quadruple versions.
*/
#define BAD NOT
#define BAD4 NOT4
#define DONE \
done; \
.align 32
/*
* TRAP vectors to the trap() function.
* It's main use is for user errors.
*/
#if !defined(lint)
.global trap
#endif /* !lint */
#define TRAP(arg) \
TT_TRACE(trace_gen) ;\
set trap, %g1 ;\
mov arg, %g3 ;\
ba,pt %xcc, sys_trap ;\
sub %g0, 1, %g4 ;\
.align 32
/*
* SYSCALL is used for system calls on both ILP32 and LP64 kernels
* depending on the "which" parameter (should be either syscall_trap
* or syscall_trap32).
*/
#define SYSCALL(which) \
TT_TRACE(trace_gen) ;\
set (which), %g1 ;\
ba,pt %xcc, sys_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define FLUSHW() \
set trap, %g1 ;\
mov T_FLUSHW, %g3 ;\
sub %g0, 1, %g4 ;\
save ;\
flushw ;\
restore ;\
FAST_TRAP_DONE ;\
.align 32
/*
* GOTO just jumps to a label.
* It's used for things that can be fixed without going thru sys_trap.
*/
#define GOTO(label) \
.global label ;\
ba,a label ;\
.empty ;\
.align 32
/*
* GOTO_TT just jumps to a label.
* correctable ECC error traps at level 0 and 1 will use this macro.
* It's used for things that can be fixed without going thru sys_trap.
*/
#define GOTO_TT(label, ttlabel) \
.global label ;\
TT_TRACE(ttlabel) ;\
ba,a label ;\
.empty ;\
.align 32
/*
* Privileged traps
* Takes breakpoint if privileged, calls trap() if not.
*/
#define PRIV(label) \
rdpr %tstate, %g1 ;\
btst TSTATE_PRIV, %g1 ;\
bnz label ;\
rdpr %tt, %g3 ;\
set trap, %g1 ;\
ba,pt %xcc, sys_trap ;\
sub %g0, 1, %g4 ;\
.align 32
/*
* DTrace traps.
*/
#define DTRACE_FASTTRAP \
.global dtrace_fasttrap_probe ;\
.global dtrace_fasttrap_probe_ptr ;\
sethi %hi(dtrace_fasttrap_probe_ptr), %g4 ;\
ldn [%g4 + %lo(dtrace_fasttrap_probe_ptr)], %g4 ;\
set dtrace_fasttrap_probe, %g1 ;\
brnz,pn %g4, user_trap ;\
sub %g0, 1, %g4 ;\
FAST_TRAP_DONE ;\
.align 32
#define DTRACE_PID \
.global dtrace_pid_probe ;\
set dtrace_pid_probe, %g1 ;\
ba,pt %xcc, user_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define DTRACE_RETURN \
.global dtrace_return_probe ;\
set dtrace_return_probe, %g1 ;\
ba,pt %xcc, user_trap ;\
sub %g0, 1, %g4 ;\
.align 32
/*
* REGISTER WINDOW MANAGEMENT MACROS
*/
/*
* various convenient units of padding
*/
#define SKIP(n) .skip 4*(n)
/*
* CLEAN_WINDOW is the simple handler for cleaning a register window.
*/
#define CLEAN_WINDOW \
TT_TRACE_L(trace_win) ;\
rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin ;\
clr %l0; clr %l1; clr %l2; clr %l3 ;\
clr %l4; clr %l5; clr %l6; clr %l7 ;\
clr %o0; clr %o1; clr %o2; clr %o3 ;\
clr %o4; clr %o5; clr %o6; clr %o7 ;\
retry; .align 128
#if !defined(lint)
/*
* If we get an unresolved tlb miss while in a window handler, the fault
* handler will resume execution at the last instruction of the window
* hander, instead of delivering the fault to the kernel. Spill handlers
* use this to spill windows into the wbuf.
*
* The mixed handler works by checking %sp, and branching to the correct
* handler. This is done by branching back to label 1: for 32b frames,
* or label 2: for 64b frames; which implies the handler order is: 32b,
* 64b, mixed. The 1: and 2: labels are offset into the routines to
* allow the branchs' delay slots to contain useful instructions.
*/
/*
* SPILL_32bit spills a 32-bit-wide kernel register window. It
* assumes that the kernel context and the nucleus context are the
* same. The stack pointer is required to be eight-byte aligned even
* though this code only needs it to be four-byte aligned.
*/
#define SPILL_32bit(tail) \
srl %sp, 0, %sp ;\
1: st %l0, [%sp + 0] ;\
st %l1, [%sp + 4] ;\
st %l2, [%sp + 8] ;\
st %l3, [%sp + 12] ;\
st %l4, [%sp + 16] ;\
st %l5, [%sp + 20] ;\
st %l6, [%sp + 24] ;\
st %l7, [%sp + 28] ;\
st %i0, [%sp + 32] ;\
st %i1, [%sp + 36] ;\
st %i2, [%sp + 40] ;\
st %i3, [%sp + 44] ;\
st %i4, [%sp + 48] ;\
st %i5, [%sp + 52] ;\
st %i6, [%sp + 56] ;\
st %i7, [%sp + 60] ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-19-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_/**/tail ;\
.empty
/*
* SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
* wide address space via the designated asi. It is used to spill
* non-kernel windows. The stack pointer is required to be eight-byte
* aligned even though this code only needs it to be four-byte
* aligned.
*/
#define SPILL_32bit_asi(asi_num, tail) \
srl %sp, 0, %sp ;\
1: sta %l0, [%sp + %g0]asi_num ;\
mov 4, %g1 ;\
sta %l1, [%sp + %g1]asi_num ;\
mov 8, %g2 ;\
sta %l2, [%sp + %g2]asi_num ;\
mov 12, %g3 ;\
sta %l3, [%sp + %g3]asi_num ;\
add %sp, 16, %g4 ;\
sta %l4, [%g4 + %g0]asi_num ;\
sta %l5, [%g4 + %g1]asi_num ;\
sta %l6, [%g4 + %g2]asi_num ;\
sta %l7, [%g4 + %g3]asi_num ;\
add %g4, 16, %g4 ;\
sta %i0, [%g4 + %g0]asi_num ;\
sta %i1, [%g4 + %g1]asi_num ;\
sta %i2, [%g4 + %g2]asi_num ;\
sta %i3, [%g4 + %g3]asi_num ;\
add %g4, 16, %g4 ;\
sta %i4, [%g4 + %g0]asi_num ;\
sta %i5, [%g4 + %g1]asi_num ;\
sta %i6, [%g4 + %g2]asi_num ;\
sta %i7, [%g4 + %g3]asi_num ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_/**/tail ;\
.empty
/*
* SPILL_32bit_tt1 spills a 32-bit-wide register window into a 32-bit
* wide address space via the designated asi. It is used to spill
* windows at tl>1 where performance isn't the primary concern and
* where we don't want to use unnecessary registers. The stack
* pointer is required to be eight-byte aligned even though this code
* only needs it to be four-byte aligned.
*/
#define SPILL_32bit_tt1(asi_num, tail) \
mov asi_num, %asi ;\
1: srl %sp, 0, %sp ;\
sta %l0, [%sp + 0]%asi ;\
sta %l1, [%sp + 4]%asi ;\
sta %l2, [%sp + 8]%asi ;\
sta %l3, [%sp + 12]%asi ;\
sta %l4, [%sp + 16]%asi ;\
sta %l5, [%sp + 20]%asi ;\
sta %l6, [%sp + 24]%asi ;\
sta %l7, [%sp + 28]%asi ;\
sta %i0, [%sp + 32]%asi ;\
sta %i1, [%sp + 36]%asi ;\
sta %i2, [%sp + 40]%asi ;\
sta %i3, [%sp + 44]%asi ;\
sta %i4, [%sp + 48]%asi ;\
sta %i5, [%sp + 52]%asi ;\
sta %i6, [%sp + 56]%asi ;\
sta %i7, [%sp + 60]%asi ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-20-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_/**/tail ;\
.empty
/*
* FILL_32bit fills a 32-bit-wide kernel register window. It assumes
* that the kernel context and the nucleus context are the same. The
* stack pointer is required to be eight-byte aligned even though this
* code only needs it to be four-byte aligned.
*/
#define FILL_32bit(tail) \
srl %sp, 0, %sp ;\
1: TT_TRACE_L(trace_win) ;\
ld [%sp + 0], %l0 ;\
ld [%sp + 4], %l1 ;\
ld [%sp + 8], %l2 ;\
ld [%sp + 12], %l3 ;\
ld [%sp + 16], %l4 ;\
ld [%sp + 20], %l5 ;\
ld [%sp + 24], %l6 ;\
ld [%sp + 28], %l7 ;\
ld [%sp + 32], %i0 ;\
ld [%sp + 36], %i1 ;\
ld [%sp + 40], %i2 ;\
ld [%sp + 44], %i3 ;\
ld [%sp + 48], %i4 ;\
ld [%sp + 52], %i5 ;\
ld [%sp + 56], %i6 ;\
ld [%sp + 60], %i7 ;\
restored ;\
retry ;\
SKIP(31-19-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_/**/tail ;\
.empty
/*
* FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
* wide address space via the designated asi. It is used to fill
* non-kernel windows. The stack pointer is required to be eight-byte
* aligned even though this code only needs it to be four-byte
* aligned.
*/
#define FILL_32bit_asi(asi_num, tail) \
srl %sp, 0, %sp ;\
1: TT_TRACE_L(trace_win) ;\
mov 4, %g1 ;\
lda [%sp + %g0]asi_num, %l0 ;\
mov 8, %g2 ;\
lda [%sp + %g1]asi_num, %l1 ;\
mov 12, %g3 ;\
lda [%sp + %g2]asi_num, %l2 ;\
lda [%sp + %g3]asi_num, %l3 ;\
add %sp, 16, %g4 ;\
lda [%g4 + %g0]asi_num, %l4 ;\
lda [%g4 + %g1]asi_num, %l5 ;\
lda [%g4 + %g2]asi_num, %l6 ;\
lda [%g4 + %g3]asi_num, %l7 ;\
add %g4, 16, %g4 ;\
lda [%g4 + %g0]asi_num, %i0 ;\
lda [%g4 + %g1]asi_num, %i1 ;\
lda [%g4 + %g2]asi_num, %i2 ;\
lda [%g4 + %g3]asi_num, %i3 ;\
add %g4, 16, %g4 ;\
lda [%g4 + %g0]asi_num, %i4 ;\
lda [%g4 + %g1]asi_num, %i5 ;\
lda [%g4 + %g2]asi_num, %i6 ;\
lda [%g4 + %g3]asi_num, %i7 ;\
restored ;\
retry ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_/**/tail ;\
.empty
/*
* FILL_32bit_tt1 fills a 32-bit-wide register window from a 32-bit
* wide address space via the designated asi. It is used to fill
* windows at tl>1 where performance isn't the primary concern and
* where we don't want to use unnecessary registers. The stack
* pointer is required to be eight-byte aligned even though this code
* only needs it to be four-byte aligned.
*/
#define FILL_32bit_tt1(asi_num, tail) \
mov asi_num, %asi ;\
1: srl %sp, 0, %sp ;\
TT_TRACE_L(trace_win) ;\
lda [%sp + 0]%asi, %l0 ;\
lda [%sp + 4]%asi, %l1 ;\
lda [%sp + 8]%asi, %l2 ;\
lda [%sp + 12]%asi, %l3 ;\
lda [%sp + 16]%asi, %l4 ;\
lda [%sp + 20]%asi, %l5 ;\
lda [%sp + 24]%asi, %l6 ;\
lda [%sp + 28]%asi, %l7 ;\
lda [%sp + 32]%asi, %i0 ;\
lda [%sp + 36]%asi, %i1 ;\
lda [%sp + 40]%asi, %i2 ;\
lda [%sp + 44]%asi, %i3 ;\
lda [%sp + 48]%asi, %i4 ;\
lda [%sp + 52]%asi, %i5 ;\
lda [%sp + 56]%asi, %i6 ;\
lda [%sp + 60]%asi, %i7 ;\
restored ;\
retry ;\
SKIP(31-20-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_/**/tail ;\
.empty
/*
* SPILL_64bit spills a 64-bit-wide kernel register window. It
* assumes that the kernel context and the nucleus context are the
* same. The stack pointer is required to be eight-byte aligned.
*/
#define SPILL_64bit(tail) \
2: stx %l0, [%sp + V9BIAS64 + 0] ;\
stx %l1, [%sp + V9BIAS64 + 8] ;\
stx %l2, [%sp + V9BIAS64 + 16] ;\
stx %l3, [%sp + V9BIAS64 + 24] ;\
stx %l4, [%sp + V9BIAS64 + 32] ;\
stx %l5, [%sp + V9BIAS64 + 40] ;\
stx %l6, [%sp + V9BIAS64 + 48] ;\
stx %l7, [%sp + V9BIAS64 + 56] ;\
stx %i0, [%sp + V9BIAS64 + 64] ;\
stx %i1, [%sp + V9BIAS64 + 72] ;\
stx %i2, [%sp + V9BIAS64 + 80] ;\
stx %i3, [%sp + V9BIAS64 + 88] ;\
stx %i4, [%sp + V9BIAS64 + 96] ;\
stx %i5, [%sp + V9BIAS64 + 104] ;\
stx %i6, [%sp + V9BIAS64 + 112] ;\
stx %i7, [%sp + V9BIAS64 + 120] ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-18-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_/**/tail ;\
.empty
/*
* SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
* wide address space via the designated asi. It is used to spill
* non-kernel windows. The stack pointer is required to be eight-byte
* aligned.
*/
#define SPILL_64bit_asi(asi_num, tail) \
mov 0 + V9BIAS64, %g1 ;\
2: stxa %l0, [%sp + %g1]asi_num ;\
mov 8 + V9BIAS64, %g2 ;\
stxa %l1, [%sp + %g2]asi_num ;\
mov 16 + V9BIAS64, %g3 ;\
stxa %l2, [%sp + %g3]asi_num ;\
mov 24 + V9BIAS64, %g4 ;\
stxa %l3, [%sp + %g4]asi_num ;\
add %sp, 32, %g5 ;\
stxa %l4, [%g5 + %g1]asi_num ;\
stxa %l5, [%g5 + %g2]asi_num ;\
stxa %l6, [%g5 + %g3]asi_num ;\
stxa %l7, [%g5 + %g4]asi_num ;\
add %g5, 32, %g5 ;\
stxa %i0, [%g5 + %g1]asi_num ;\
stxa %i1, [%g5 + %g2]asi_num ;\
stxa %i2, [%g5 + %g3]asi_num ;\
stxa %i3, [%g5 + %g4]asi_num ;\
add %g5, 32, %g5 ;\
stxa %i4, [%g5 + %g1]asi_num ;\
stxa %i5, [%g5 + %g2]asi_num ;\
stxa %i6, [%g5 + %g3]asi_num ;\
stxa %i7, [%g5 + %g4]asi_num ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_/**/tail ;\
.empty
/*
* SPILL_64bit_tt1 spills a 64-bit-wide register window into a 64-bit
* wide address space via the designated asi. It is used to spill
* windows at tl>1 where performance isn't the primary concern and
* where we don't want to use unnecessary registers. The stack
* pointer is required to be eight-byte aligned.
*/
#define SPILL_64bit_tt1(asi_num, tail) \
mov asi_num, %asi ;\
2: stxa %l0, [%sp + V9BIAS64 + 0]%asi ;\
stxa %l1, [%sp + V9BIAS64 + 8]%asi ;\
stxa %l2, [%sp + V9BIAS64 + 16]%asi ;\
stxa %l3, [%sp + V9BIAS64 + 24]%asi ;\
stxa %l4, [%sp + V9BIAS64 + 32]%asi ;\
stxa %l5, [%sp + V9BIAS64 + 40]%asi ;\
stxa %l6, [%sp + V9BIAS64 + 48]%asi ;\
stxa %l7, [%sp + V9BIAS64 + 56]%asi ;\
stxa %i0, [%sp + V9BIAS64 + 64]%asi ;\
stxa %i1, [%sp + V9BIAS64 + 72]%asi ;\
stxa %i2, [%sp + V9BIAS64 + 80]%asi ;\
stxa %i3, [%sp + V9BIAS64 + 88]%asi ;\
stxa %i4, [%sp + V9BIAS64 + 96]%asi ;\
stxa %i5, [%sp + V9BIAS64 + 104]%asi ;\
stxa %i6, [%sp + V9BIAS64 + 112]%asi ;\
stxa %i7, [%sp + V9BIAS64 + 120]%asi ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-19-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_/**/tail ;\
.empty
/*
* FILL_64bit fills a 64-bit-wide kernel register window. It assumes
* that the kernel context and the nucleus context are the same. The
* stack pointer is required to be eight-byte aligned.
*/
#define FILL_64bit(tail) \
2: TT_TRACE_L(trace_win) ;\
ldx [%sp + V9BIAS64 + 0], %l0 ;\
ldx [%sp + V9BIAS64 + 8], %l1 ;\
ldx [%sp + V9BIAS64 + 16], %l2 ;\
ldx [%sp + V9BIAS64 + 24], %l3 ;\
ldx [%sp + V9BIAS64 + 32], %l4 ;\
ldx [%sp + V9BIAS64 + 40], %l5 ;\
ldx [%sp + V9BIAS64 + 48], %l6 ;\
ldx [%sp + V9BIAS64 + 56], %l7 ;\
ldx [%sp + V9BIAS64 + 64], %i0 ;\
ldx [%sp + V9BIAS64 + 72], %i1 ;\
ldx [%sp + V9BIAS64 + 80], %i2 ;\
ldx [%sp + V9BIAS64 + 88], %i3 ;\
ldx [%sp + V9BIAS64 + 96], %i4 ;\
ldx [%sp + V9BIAS64 + 104], %i5 ;\
ldx [%sp + V9BIAS64 + 112], %i6 ;\
ldx [%sp + V9BIAS64 + 120], %i7 ;\
restored ;\
retry ;\
SKIP(31-18-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_/**/tail ;\
.empty
/*
* FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
* wide address space via the designated asi. It is used to fill
* non-kernel windows. The stack pointer is required to be eight-byte
* aligned.
*/
#define FILL_64bit_asi(asi_num, tail) \
mov V9BIAS64 + 0, %g1 ;\
2: TT_TRACE_L(trace_win) ;\
ldxa [%sp + %g1]asi_num, %l0 ;\
mov V9BIAS64 + 8, %g2 ;\
ldxa [%sp + %g2]asi_num, %l1 ;\
mov V9BIAS64 + 16, %g3 ;\
ldxa [%sp + %g3]asi_num, %l2 ;\
mov V9BIAS64 + 24, %g4 ;\
ldxa [%sp + %g4]asi_num, %l3 ;\
add %sp, 32, %g5 ;\
ldxa [%g5 + %g1]asi_num, %l4 ;\
ldxa [%g5 + %g2]asi_num, %l5 ;\
ldxa [%g5 + %g3]asi_num, %l6 ;\
ldxa [%g5 + %g4]asi_num, %l7 ;\
add %g5, 32, %g5 ;\
ldxa [%g5 + %g1]asi_num, %i0 ;\
ldxa [%g5 + %g2]asi_num, %i1 ;\
ldxa [%g5 + %g3]asi_num, %i2 ;\
ldxa [%g5 + %g4]asi_num, %i3 ;\
add %g5, 32, %g5 ;\
ldxa [%g5 + %g1]asi_num, %i4 ;\
ldxa [%g5 + %g2]asi_num, %i5 ;\
ldxa [%g5 + %g3]asi_num, %i6 ;\
ldxa [%g5 + %g4]asi_num, %i7 ;\
restored ;\
retry ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_/**/tail ;\
.empty
/*
* FILL_64bit_tt1 fills a 64-bit-wide register window from a 64-bit
* wide address space via the designated asi. It is used to fill
* windows at tl>1 where performance isn't the primary concern and
* where we don't want to use unnecessary registers. The stack
* pointer is required to be eight-byte aligned.
*/
#define FILL_64bit_tt1(asi_num, tail) \
mov asi_num, %asi ;\
TT_TRACE_L(trace_win) ;\
ldxa [%sp + V9BIAS64 + 0]%asi, %l0 ;\
ldxa [%sp + V9BIAS64 + 8]%asi, %l1 ;\
ldxa [%sp + V9BIAS64 + 16]%asi, %l2 ;\
ldxa [%sp + V9BIAS64 + 24]%asi, %l3 ;\
ldxa [%sp + V9BIAS64 + 32]%asi, %l4 ;\
ldxa [%sp + V9BIAS64 + 40]%asi, %l5 ;\
ldxa [%sp + V9BIAS64 + 48]%asi, %l6 ;\
ldxa [%sp + V9BIAS64 + 56]%asi, %l7 ;\
ldxa [%sp + V9BIAS64 + 64]%asi, %i0 ;\
ldxa [%sp + V9BIAS64 + 72]%asi, %i1 ;\
ldxa [%sp + V9BIAS64 + 80]%asi, %i2 ;\
ldxa [%sp + V9BIAS64 + 88]%asi, %i3 ;\
ldxa [%sp + V9BIAS64 + 96]%asi, %i4 ;\
ldxa [%sp + V9BIAS64 + 104]%asi, %i5 ;\
ldxa [%sp + V9BIAS64 + 112]%asi, %i6 ;\
ldxa [%sp + V9BIAS64 + 120]%asi, %i7 ;\
restored ;\
retry ;\
SKIP(31-19-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_/**/tail ;\
.empty
#endif /* !lint */
/*
* SPILL_mixed spills either size window, depending on
* whether %sp is even or odd, to a 32-bit address space.
* This may only be used in conjunction with SPILL_32bit/
* SPILL_64bit. New versions of SPILL_mixed_{tt1,asi} would be
* needed for use with SPILL_{32,64}bit_{tt1,asi}. Particular
* attention should be paid to the instructions that belong
* in the delay slots of the branches depending on the type
* of spill handler being branched to.
* Clear upper 32 bits of %sp if it is odd.
* We won't need to clear them in 64 bit kernel.
*/
#define SPILL_mixed \
btst 1, %sp ;\
bz,a,pt %xcc, 1b ;\
srl %sp, 0, %sp ;\
ba,pt %xcc, 2b ;\
nop ;\
.align 128
/*
* FILL_mixed(ASI) fills either size window, depending on
* whether %sp is even or odd, from a 32-bit address space.
* This may only be used in conjunction with FILL_32bit/
* FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
* needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
* attention should be paid to the instructions that belong
* in the delay slots of the branches depending on the type
* of fill handler being branched to.
* Clear upper 32 bits of %sp if it is odd.
* We won't need to clear them in 64 bit kernel.
*/
#define FILL_mixed \
btst 1, %sp ;\
bz,a,pt %xcc, 1b ;\
srl %sp, 0, %sp ;\
ba,pt %xcc, 2b ;\
nop ;\
.align 128
/*
* SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
* respectively, into the address space via the designated asi. The
* unbiased stack pointer is required to be eight-byte aligned (even for
* the 32-bit case even though this code does not require such strict
* alignment).
*
* With SPARC v9 the spill trap takes precedence over the cleanwin trap
* so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
* will cause cwp + 2 to be spilled but will not clean cwp + 1. That
* window may contain kernel data so in user_rtt we set wstate to call
* these spill handlers on the first user spill trap. These handler then
* spill the appropriate window but also back up a window and clean the
* window that didn't get a cleanwin trap.
*/
#define SPILL_32clean(asi_num, tail) \
srl %sp, 0, %sp ;\
sta %l0, [%sp + %g0]asi_num ;\
mov 4, %g1 ;\
sta %l1, [%sp + %g1]asi_num ;\
mov 8, %g2 ;\
sta %l2, [%sp + %g2]asi_num ;\
mov 12, %g3 ;\
sta %l3, [%sp + %g3]asi_num ;\
add %sp, 16, %g4 ;\
sta %l4, [%g4 + %g0]asi_num ;\
sta %l5, [%g4 + %g1]asi_num ;\
sta %l6, [%g4 + %g2]asi_num ;\
sta %l7, [%g4 + %g3]asi_num ;\
add %g4, 16, %g4 ;\
sta %i0, [%g4 + %g0]asi_num ;\
sta %i1, [%g4 + %g1]asi_num ;\
sta %i2, [%g4 + %g2]asi_num ;\
sta %i3, [%g4 + %g3]asi_num ;\
add %g4, 16, %g4 ;\
sta %i4, [%g4 + %g0]asi_num ;\
sta %i5, [%g4 + %g1]asi_num ;\
sta %i6, [%g4 + %g2]asi_num ;\
sta %i7, [%g4 + %g3]asi_num ;\
TT_TRACE_L(trace_win) ;\
b .spill_clean ;\
mov WSTATE_USER32, %g7 ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_/**/tail ;\
.empty
#define SPILL_64clean(asi_num, tail) \
mov 0 + V9BIAS64, %g1 ;\
stxa %l0, [%sp + %g1]asi_num ;\
mov 8 + V9BIAS64, %g2 ;\
stxa %l1, [%sp + %g2]asi_num ;\
mov 16 + V9BIAS64, %g3 ;\
stxa %l2, [%sp + %g3]asi_num ;\
mov 24 + V9BIAS64, %g4 ;\
stxa %l3, [%sp + %g4]asi_num ;\
add %sp, 32, %g5 ;\
stxa %l4, [%g5 + %g1]asi_num ;\
stxa %l5, [%g5 + %g2]asi_num ;\
stxa %l6, [%g5 + %g3]asi_num ;\
stxa %l7, [%g5 + %g4]asi_num ;\
add %g5, 32, %g5 ;\
stxa %i0, [%g5 + %g1]asi_num ;\
stxa %i1, [%g5 + %g2]asi_num ;\
stxa %i2, [%g5 + %g3]asi_num ;\
stxa %i3, [%g5 + %g4]asi_num ;\
add %g5, 32, %g5 ;\
stxa %i4, [%g5 + %g1]asi_num ;\
stxa %i5, [%g5 + %g2]asi_num ;\
stxa %i6, [%g5 + %g3]asi_num ;\
stxa %i7, [%g5 + %g4]asi_num ;\
TT_TRACE_L(trace_win) ;\
b .spill_clean ;\
mov WSTATE_USER64, %g7 ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_/**/tail ;\
.empty
/*
* Floating point disabled.
*/
#define FP_DISABLED_TRAP \
TT_TRACE(trace_gen) ;\
ba,pt %xcc,.fp_disabled ;\
nop ;\
.align 32
/*
* Floating point exceptions.
*/
#define FP_IEEE_TRAP \
TT_TRACE(trace_gen) ;\
ba,pt %xcc,.fp_ieee_exception ;\
nop ;\
.align 32
#define FP_TRAP \
TT_TRACE(trace_gen) ;\
ba,pt %xcc,.fp_exception ;\
nop ;\
.align 32
#if !defined(lint)
/*
* asynchronous traps at level 0 and level 1
*
* The first instruction must be a membar for UltraSPARC-III
* to stop RED state entry if the store queue has many
* pending bad stores (PRM, Chapter 11).
*/
#define ASYNC_TRAP(ttype, ttlabel)\
membar #Sync ;\
TT_TRACE(ttlabel) ;\
ba async_err ;\
mov ttype, %g5 ;\
.align 32
/*
* Defaults to BAD entry, but establishes label to be used for
* architecture-specific overwrite of trap table entry.
*/
#define LABELED_BAD(table_name) \
.global table_name ;\
table_name: ;\
BAD
#endif /* !lint */
/*
* illegal instruction trap
*/
#define ILLTRAP_INSTR \
membar #Sync ;\
TT_TRACE(trace_gen) ;\
or %g0, P_UTRAP4, %g2 ;\
or %g0, T_UNIMP_INSTR, %g3 ;\
sethi %hi(.check_v9utrap), %g4 ;\
jmp %g4 + %lo(.check_v9utrap) ;\
nop ;\
.align 32
/*
* tag overflow trap
*/
#define TAG_OVERFLOW \
TT_TRACE(trace_gen) ;\
or %g0, P_UTRAP10, %g2 ;\
or %g0, T_TAG_OVERFLOW, %g3 ;\
sethi %hi(.check_v9utrap), %g4 ;\
jmp %g4 + %lo(.check_v9utrap) ;\
nop ;\
.align 32
/*
* divide by zero trap
*/
#define DIV_BY_ZERO \
TT_TRACE(trace_gen) ;\
or %g0, P_UTRAP11, %g2 ;\
or %g0, T_IDIV0, %g3 ;\
sethi %hi(.check_v9utrap), %g4 ;\
jmp %g4 + %lo(.check_v9utrap) ;\
nop ;\
.align 32
/*
* trap instruction for V9 user trap handlers
*/
#define TRAP_INSTR \
TT_TRACE(trace_gen) ;\
or %g0, T_SOFTWARE_TRAP, %g3 ;\
sethi %hi(.check_v9utrap), %g4 ;\
jmp %g4 + %lo(.check_v9utrap) ;\
nop ;\
.align 32
#define TRP4 TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
/*
* LEVEL_INTERRUPT is for level N interrupts.
* VECTOR_INTERRUPT is for the vector trap.
*/
#define LEVEL_INTERRUPT(level) \
.global tt_pil/**/level ;\
tt_pil/**/level: ;\
ba,pt %xcc, pil_interrupt ;\
mov level, %g4 ;\
.align 32
#define LEVEL14_INTERRUPT \
ba pil14_interrupt ;\
mov PIL_14, %g4 ;\
.align 32
#define VECTOR_INTERRUPT \
ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g1 ;\
btst IRSR_BUSY, %g1 ;\
bnz,pt %xcc, vec_interrupt ;\
nop ;\
ba,a,pt %xcc, vec_intr_spurious ;\
.empty ;\
.align 32
/*
* MMU Trap Handlers.
*/
#define SWITCH_GLOBALS /* mmu->alt, alt->mmu */ \
rdpr %pstate, %g5 ;\
wrpr %g5, PSTATE_MG | PSTATE_AG, %pstate
#define IMMU_EXCEPTION \
membar #Sync ;\
SWITCH_GLOBALS ;\
wr %g0, ASI_IMMU, %asi ;\
rdpr %tpc, %g2 ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
ba,pt %xcc, .mmu_exception_end ;\
mov T_INSTR_EXCEPTION, %g1 ;\
.align 32
#define DMMU_EXCEPTION \
SWITCH_GLOBALS ;\
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_TAG_ACCESS]%asi, %g2 ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
ba,pt %xcc, .mmu_exception_end ;\
mov T_DATA_EXCEPTION, %g1 ;\
.align 32
#define DMMU_EXC_AG_PRIV \
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_SFAR]%asi, %g2 ;\
ba,pt %xcc, .mmu_priv_exception ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
.align 32
#define DMMU_EXC_AG_NOT_ALIGNED \
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_SFAR]%asi, %g2 ;\
ba,pt %xcc, .mmu_exception_not_aligned ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
.align 32
/*
* SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
*/
#define DMMU_EXC_LDDF_NOT_ALIGNED \
btst 1, %sp ;\
bnz,pt %xcc, .lddf_exception_not_aligned ;\
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_SFAR]%asi, %g2 ;\
ba,pt %xcc, .mmu_exception_not_aligned ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
.align 32
#define DMMU_EXC_STDF_NOT_ALIGNED \
btst 1, %sp ;\
bnz,pt %xcc, .stdf_exception_not_aligned ;\
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_SFAR]%asi, %g2 ;\
ba,pt %xcc, .mmu_exception_not_aligned ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
.align 32
/*
* Flush the TLB using either the primary, secondary, or nucleus flush
* operation based on whether the ctx from the tag access register matches
* the primary or secondary context (flush the nucleus if neither matches).
*
* Requires a membar #Sync before next ld/st.
* exits with:
* g2 = tag access register
* g3 = ctx number
*/
#if TAGACC_CTX_MASK != CTXREG_CTX_MASK
#error "TAGACC_CTX_MASK != CTXREG_CTX_MASK"
#endif
#define DTLB_DEMAP_ENTRY \
mov MMU_TAG_ACCESS, %g1 ;\
mov MMU_PCONTEXT, %g5 ;\
ldxa [%g1]ASI_DMMU, %g2 ;\
sethi %hi(TAGACC_CTX_MASK), %g4 ;\
or %g4, %lo(TAGACC_CTX_MASK), %g4 ;\
and %g2, %g4, %g3 /* g3 = ctx */ ;\
ldxa [%g5]ASI_DMMU, %g6 /* g6 = primary ctx */ ;\
and %g6, %g4, %g6 /* &= CTXREG_CTX_MASK */ ;\
cmp %g3, %g6 ;\
be,pt %xcc, 1f ;\
andn %g2, %g4, %g1 /* ctx = primary */ ;\
mov MMU_SCONTEXT, %g5 ;\
ldxa [%g5]ASI_DMMU, %g6 /* g6 = secondary ctx */ ;\
and %g6, %g4, %g6 /* &= CTXREG_CTX_MASK */ ;\
cmp %g3, %g6 ;\
be,a,pt %xcc, 1f ;\
or %g1, DEMAP_SECOND, %g1 ;\
or %g1, DEMAP_NUCLEUS, %g1 ;\
1: stxa %g0, [%g1]ASI_DTLB_DEMAP /* MMU_DEMAP_PAGE */ ;\
membar #Sync
#if defined(cscope)
/*
* Define labels to direct cscope quickly to labels that
* are generated by macro expansion of DTLB_MISS().
*/
.global tt0_dtlbmiss
tt0_dtlbmiss:
.global tt1_dtlbmiss
tt1_dtlbmiss:
nop
#endif
/*
* Needs to be exactly 32 instructions
*
* UTLB NOTE: If we don't hit on the 8k pointer then we branch
* to a special 4M tsb handler. It would be nice if that handler
* could live in this file but currently it seems better to allow
* it to fall thru to sfmmu_tsb_miss.
*/
#define DTLB_MISS(table_name) ;\
.global table_name/**/_dtlbmiss ;\
table_name/**/_dtlbmiss: ;\
HAT_PERCPU_DBSTAT(TSBMISS_DTLBMISS) /* 3 instr ifdef DEBUG */ ;\
mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\
ldxa [%g0]ASI_DMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\
ldxa [%g6]ASI_DMMU, %g2 /* g2 = tag access */ ;\
sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\
cmp %g3, INVALID_CONTEXT ;\
ble,pn %xcc, sfmmu_kdtlb_miss ;\
srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
brlz,pn %g1, sfmmu_udtlb_slowpath ;\
nop ;\
ldda [%g1]ASI_NQUAD_LD, %g4 /* g4 = tag, %g5 data */ ;\
cmp %g4, %g7 ;\
bne,pn %xcc, sfmmu_tsb_miss_tt /* no 4M TSB, miss */ ;\
mov %g0, %g3 /* clear 4M tsbe ptr */ ;\
TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\
stxa %g5, [%g0]ASI_DTLB_IN /* trapstat expects TTE */ ;\
retry /* in %g5 */ ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
.align 128
#if defined(cscope)
/*
* Define labels to direct cscope quickly to labels that
* are generated by macro expansion of ITLB_MISS().
*/
.global tt0_itlbmiss
tt0_itlbmiss:
.global tt1_itlbmiss
tt1_itlbmiss:
nop
#endif
/*
* Instruction miss handler.
* ldda instructions will have their ASI patched
* by sfmmu_patch_ktsb at runtime.
* MUST be EXACTLY 32 instructions or we'll break.
*/
#define ITLB_MISS(table_name) \
.global table_name/**/_itlbmiss ;\
table_name/**/_itlbmiss: ;\
HAT_PERCPU_DBSTAT(TSBMISS_ITLBMISS) /* 3 instr ifdef DEBUG */ ;\
mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\
ldxa [%g0]ASI_IMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\
ldxa [%g6]ASI_IMMU, %g2 /* g2 = tag access */ ;\
sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\
cmp %g3, INVALID_CONTEXT ;\
ble,pn %xcc, sfmmu_kitlb_miss ;\
srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
brlz,pn %g1, sfmmu_uitlb_slowpath /* if >1 TSB branch */ ;\
nop ;\
ldda [%g1]ASI_NQUAD_LD, %g4 /* g4 = tag, g5 = data */ ;\
cmp %g4, %g7 ;\
bne,pn %xcc, sfmmu_tsb_miss_tt /* br if 8k ptr miss */ ;\
mov %g0, %g3 /* no 4M TSB */ ;\
andcc %g5, TTE_EXECPRM_INT, %g0 /* check execute bit */ ;\
bz,pn %icc, exec_fault ;\
nop ;\
TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\
stxa %g5, [%g0]ASI_ITLB_IN /* trapstat expects %g5 */ ;\
retry ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
.align 128
/*
* This macro is the first level handler for fast protection faults.
* It first demaps the tlb entry which generated the fault and then
* attempts to set the modify bit on the hash. It needs to be
* exactly 32 instructions.
*/
#define DTLB_PROT \
DTLB_DEMAP_ENTRY /* 20 instructions */ ;\
/* ;\
* At this point: ;\
* g1 = ???? ;\
* g2 = tag access register ;\
* g3 = ctx number ;\
* g4 = ???? ;\
*/ ;\
TT_TRACE(trace_dataprot) /* 2 instr ifdef TRAPTRACE */ ;\
/* clobbers g1 and g6 */ ;\
ldxa [%g0]ASI_DMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\
brnz,pt %g3, sfmmu_uprot_trap /* user trap */ ;\
nop ;\
ba,a,pt %xcc, sfmmu_kprot_trap /* kernel trap */ ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
.align 128
#define DMMU_EXCEPTION_TL1 ;\
SWITCH_GLOBALS ;\
ba,a,pt %xcc, mmu_trap_tl1 ;\
nop ;\
.align 32
#define MISALIGN_ADDR_TL1 ;\
ba,a,pt %xcc, mmu_trap_tl1 ;\
nop ;\
.align 32
/*
* Trace a tsb hit
* g1 = tsbe pointer (in/clobbered)
* g2 = tag access register (in)
* g3 - g4 = scratch (clobbered)
* g5 = tsbe data (in)
* g6 = scratch (clobbered)
* g7 = pc we jumped here from (in)
* ttextra = value to OR in to trap type (%tt) (in)
*/
#ifdef TRAPTRACE
#define TRACE_TSBHIT(ttextra) \
membar #Sync ;\
sethi %hi(FLUSH_ADDR), %g6 ;\
flush %g6 ;\
TRACE_PTR(%g3, %g6) ;\
GET_TRACE_TICK(%g6) ;\
stxa %g6, [%g3 + TRAP_ENT_TICK]%asi ;\
stxa %g2, [%g3 + TRAP_ENT_SP]%asi /* tag access */ ;\
stxa %g5, [%g3 + TRAP_ENT_F1]%asi /* tsb data */ ;\
rdpr %tnpc, %g6 ;\
stxa %g6, [%g3 + TRAP_ENT_F2]%asi ;\
stxa %g1, [%g3 + TRAP_ENT_F3]%asi /* tsb pointer */ ;\
stxa %g0, [%g3 + TRAP_ENT_F4]%asi ;\
rdpr %tpc, %g6 ;\
stxa %g6, [%g3 + TRAP_ENT_TPC]%asi ;\
rdpr %tl, %g6 ;\
stha %g6, [%g3 + TRAP_ENT_TL]%asi ;\
rdpr %tt, %g6 ;\
or %g6, (ttextra), %g6 ;\
stha %g6, [%g3 + TRAP_ENT_TT]%asi ;\
ldxa [%g0]ASI_IMMU, %g1 /* tag target */ ;\
ldxa [%g0]ASI_DMMU, %g4 ;\
cmp %g6, FAST_IMMU_MISS_TT ;\
movne %icc, %g4, %g1 ;\
stxa %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* tsb tag */ ;\
stxa %g0, [%g3 + TRAP_ENT_TR]%asi ;\
TRACE_NEXT(%g3, %g4, %g6)
#else
#define TRACE_TSBHIT(ttextra)
#endif
#if defined(lint)
struct scb trap_table;
struct scb scb; /* trap_table/scb are the same object */
#else /* lint */
/*
* =======================================================================
* SPARC V9 TRAP TABLE
*
* The trap table is divided into two halves: the first half is used when
* taking traps when TL=0; the second half is used when taking traps from
* TL>0. Note that handlers in the second half of the table might not be able
* to make the same assumptions as handlers in the first half of the table.
*
* Worst case trap nesting so far:
*
* at TL=0 client issues software trap requesting service
* at TL=1 nucleus wants a register window
* at TL=2 register window clean/spill/fill takes a TLB miss
* at TL=3 processing TLB miss
* at TL=4 handle asynchronous error
*
* Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
*
* =======================================================================
*/
.section ".text"
.align 4
.global trap_table, scb, trap_table0, trap_table1, etrap_table
.type trap_table, #function
.type scb, #function
trap_table:
scb:
trap_table0:
/* hardware traps */
NOT; /* 000 reserved */
RED; /* 001 power on reset */
RED; /* 002 watchdog reset */
RED; /* 003 externally initiated reset */
RED; /* 004 software initiated reset */
RED; /* 005 red mode exception */
NOT; NOT; /* 006 - 007 reserved */
IMMU_EXCEPTION; /* 008 instruction access exception */
NOT; /* 009 instruction access MMU miss */
ASYNC_TRAP(T_INSTR_ERROR, trace_gen);
/* 00A instruction access error */
NOT; NOT4; /* 00B - 00F reserved */
ILLTRAP_INSTR; /* 010 illegal instruction */
TRAP(T_PRIV_INSTR); /* 011 privileged opcode */
NOT; /* 012 unimplemented LDD */
NOT; /* 013 unimplemented STD */
NOT4; NOT4; NOT4; /* 014 - 01F reserved */
FP_DISABLED_TRAP; /* 020 fp disabled */
FP_IEEE_TRAP; /* 021 fp exception ieee 754 */
FP_TRAP; /* 022 fp exception other */
TAG_OVERFLOW; /* 023 tag overflow */
CLEAN_WINDOW; /* 024 - 027 clean window */
DIV_BY_ZERO; /* 028 division by zero */
NOT; /* 029 internal processor error */
NOT; NOT; NOT4; /* 02A - 02F reserved */
DMMU_EXCEPTION; /* 030 data access exception */
NOT; /* 031 data access MMU miss */
ASYNC_TRAP(T_DATA_ERROR, trace_gen);
/* 032 data access error */
NOT; /* 033 data access protection */
DMMU_EXC_AG_NOT_ALIGNED; /* 034 mem address not aligned */
DMMU_EXC_LDDF_NOT_ALIGNED; /* 035 LDDF mem address not aligned */
DMMU_EXC_STDF_NOT_ALIGNED; /* 036 STDF mem address not aligned */
DMMU_EXC_AG_PRIV; /* 037 privileged action */
NOT; /* 038 LDQF mem address not aligned */
NOT; /* 039 STQF mem address not aligned */
NOT; NOT; NOT4; /* 03A - 03F reserved */
NOT; /* 040 async data error */
LEVEL_INTERRUPT(1); /* 041 interrupt level 1 */
LEVEL_INTERRUPT(2); /* 042 interrupt level 2 */
LEVEL_INTERRUPT(3); /* 043 interrupt level 3 */
LEVEL_INTERRUPT(4); /* 044 interrupt level 4 */
LEVEL_INTERRUPT(5); /* 045 interrupt level 5 */
LEVEL_INTERRUPT(6); /* 046 interrupt level 6 */
LEVEL_INTERRUPT(7); /* 047 interrupt level 7 */
LEVEL_INTERRUPT(8); /* 048 interrupt level 8 */
LEVEL_INTERRUPT(9); /* 049 interrupt level 9 */
LEVEL_INTERRUPT(10); /* 04A interrupt level 10 */
LEVEL_INTERRUPT(11); /* 04B interrupt level 11 */
LEVEL_INTERRUPT(12); /* 04C interrupt level 12 */
LEVEL_INTERRUPT(13); /* 04D interrupt level 13 */
LEVEL14_INTERRUPT; /* 04E interrupt level 14 */
LEVEL_INTERRUPT(15); /* 04F interrupt level 15 */
NOT4; NOT4; NOT4; NOT4; /* 050 - 05F reserved */
VECTOR_INTERRUPT; /* 060 interrupt vector */
GOTO(kmdb_trap); /* 061 PA watchpoint */
GOTO(kmdb_trap); /* 062 VA watchpoint */
GOTO_TT(ce_err, trace_gen); /* 063 corrected ECC error */
ITLB_MISS(tt0); /* 064 instruction access MMU miss */
DTLB_MISS(tt0); /* 068 data access MMU miss */
DTLB_PROT; /* 06C data access protection */
LABELED_BAD(tt0_fecc); /* 070 fast ecache ECC error */
LABELED_BAD(tt0_dperr); /* 071 Cheetah+ dcache parity error */
LABELED_BAD(tt0_iperr); /* 072 Cheetah+ icache parity error */
NOT; /* 073 reserved */
NOT4; NOT4; NOT4; /* 074 - 07F reserved */
NOT4; /* 080 spill 0 normal */
SPILL_32bit_asi(ASI_AIUP,sn0); /* 084 spill 1 normal */
SPILL_64bit_asi(ASI_AIUP,sn0); /* 088 spill 2 normal */
SPILL_32clean(ASI_AIUP,sn0); /* 08C spill 3 normal */
SPILL_64clean(ASI_AIUP,sn0); /* 090 spill 4 normal */
SPILL_32bit(not); /* 094 spill 5 normal */
SPILL_64bit(not); /* 098 spill 6 normal */
SPILL_mixed; /* 09C spill 7 normal */
NOT4; /* 0A0 spill 0 other */
SPILL_32bit_asi(ASI_AIUS,so0); /* 0A4 spill 1 other */
SPILL_64bit_asi(ASI_AIUS,so0); /* 0A8 spill 2 other */
SPILL_32bit_asi(ASI_AIUS,so0); /* 0AC spill 3 other */
SPILL_64bit_asi(ASI_AIUS,so0); /* 0B0 spill 4 other */
NOT4; /* 0B4 spill 5 other */
NOT4; /* 0B8 spill 6 other */
NOT4; /* 0BC spill 7 other */
NOT4; /* 0C0 fill 0 normal */
FILL_32bit_asi(ASI_AIUP,fn0); /* 0C4 fill 1 normal */
FILL_64bit_asi(ASI_AIUP,fn0); /* 0C8 fill 2 normal */
FILL_32bit_asi(ASI_AIUP,fn0); /* 0CC fill 3 normal */
FILL_64bit_asi(ASI_AIUP,fn0); /* 0D0 fill 4 normal */
FILL_32bit(not); /* 0D4 fill 5 normal */
FILL_64bit(not); /* 0D8 fill 6 normal */
FILL_mixed; /* 0DC fill 7 normal */
NOT4; /* 0E0 fill 0 other */
NOT4; /* 0E4 fill 1 other */
NOT4; /* 0E8 fill 2 other */
NOT4; /* 0EC fill 3 other */
NOT4; /* 0F0 fill 4 other */
NOT4; /* 0F4 fill 5 other */
NOT4; /* 0F8 fill 6 other */
NOT4; /* 0FC fill 7 other */
/* user traps */
GOTO(syscall_trap_4x); /* 100 old system call */
TRAP(T_BREAKPOINT); /* 101 user breakpoint */
TRAP(T_DIV0); /* 102 user divide by zero */
FLUSHW(); /* 103 flush windows */
GOTO(.clean_windows); /* 104 clean windows */
BAD; /* 105 range check ?? */
GOTO(.fix_alignment); /* 106 do unaligned references */
BAD; /* 107 unused */
SYSCALL(syscall_trap32); /* 108 ILP32 system call on LP64 */
GOTO(set_trap0_addr); /* 109 set trap0 address */
BAD; BAD; BAD4; /* 10A - 10F unused */
TRP4; TRP4; TRP4; TRP4; /* 110 - 11F V9 user trap handlers */
GOTO(.getcc); /* 120 get condition codes */
GOTO(.setcc); /* 121 set condition codes */
GOTO(.getpsr); /* 122 get psr */
GOTO(.setpsr); /* 123 set psr (some fields) */
GOTO(get_timestamp); /* 124 get timestamp */
GOTO(get_virtime); /* 125 get lwp virtual time */
PRIV(self_xcall); /* 126 self xcall */
GOTO(get_hrestime); /* 127 get hrestime */
BAD; /* 128 ST_SETV9STACK */
GOTO(.getlgrp); /* 129 get lgrpid */
BAD; BAD; BAD4; /* 12A - 12F unused */
BAD4; BAD4; /* 130 - 137 unused */
DTRACE_PID; /* 138 dtrace pid tracing provider */
DTRACE_FASTTRAP; /* 139 dtrace fasttrap provider */
DTRACE_RETURN; /* 13A dtrace pid return probe */
BAD; BAD4; /* 13B - 13F unused */
SYSCALL(syscall_trap) /* 140 LP64 system call */
BAD; /* 141 unused */
#ifdef DEBUG_USER_TRAPTRACECTL
GOTO(.traptrace_freeze); /* 142 freeze traptrace */
GOTO(.traptrace_unfreeze); /* 143 unfreeze traptrace */
#else
BAD; BAD; /* 142 - 143 unused */
#endif
BAD4; BAD4; BAD4; /* 144 - 14F unused */
BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */
BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */
BAD; /* 170 - unused */
BAD; /* 171 - unused */
BAD; BAD; /* 172 - 173 unused */
BAD4; BAD4; /* 174 - 17B unused */
#ifdef PTL1_PANIC_DEBUG
mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
/* 17C test ptl1_panic */
#else
BAD; /* 17C unused */
#endif /* PTL1_PANIC_DEBUG */
PRIV(kmdb_trap); /* 17D kmdb enter (L1-A) */
PRIV(kmdb_trap); /* 17E kmdb breakpoint */
PRIV(kctx_obp_bpt); /* 17F obp breakpoint */
/* reserved */
NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */
NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */
NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */
NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */
NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */
NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */
NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */
NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */
trap_table1:
NOT4; NOT4; NOT; NOT; /* 000 - 009 unused */
ASYNC_TRAP(T_INSTR_ERROR + T_TL1, trace_gen);
/* 00A instruction access error */
NOT; NOT4; /* 00B - 00F unused */
NOT4; NOT4; NOT4; NOT4; /* 010 - 01F unused */
NOT4; /* 020 - 023 unused */
CLEAN_WINDOW; /* 024 - 027 clean window */
NOT4; NOT4; /* 028 - 02F unused */
DMMU_EXCEPTION_TL1; /* 030 data access exception */
NOT; /* 031 unused */
ASYNC_TRAP(T_DATA_ERROR + T_TL1, trace_gen);
/* 032 data access error */
NOT; /* 033 unused */
MISALIGN_ADDR_TL1; /* 034 mem address not aligned */
NOT; NOT; NOT; NOT4; NOT4 /* 035 - 03F unused */
NOT4; NOT4; NOT4; NOT4; /* 040 - 04F unused */
NOT4; NOT4; NOT4; NOT4; /* 050 - 05F unused */
NOT; /* 060 unused */
GOTO(kmdb_trap_tl1); /* 061 PA watchpoint */
GOTO(kmdb_trap_tl1); /* 062 VA watchpoint */
GOTO_TT(ce_err_tl1, trace_gen); /* 063 corrected ECC error */
ITLB_MISS(tt1); /* 064 instruction access MMU miss */
DTLB_MISS(tt1); /* 068 data access MMU miss */
DTLB_PROT; /* 06C data access protection */
LABELED_BAD(tt1_fecc); /* 070 fast ecache ECC error */
LABELED_BAD(tt1_dperr); /* 071 Cheetah+ dcache parity error */
LABELED_BAD(tt1_iperr); /* 072 Cheetah+ icache parity error */
NOT; /* 073 reserved */
NOT4; NOT4; NOT4; /* 074 - 07F reserved */
NOT4; /* 080 spill 0 normal */
SPILL_32bit_tt1(ASI_AIUP,sn1); /* 084 spill 1 normal */
SPILL_64bit_tt1(ASI_AIUP,sn1); /* 088 spill 2 normal */
SPILL_32bit_tt1(ASI_AIUP,sn1); /* 08C spill 3 normal */
SPILL_64bit_tt1(ASI_AIUP,sn1); /* 090 spill 4 normal */
SPILL_32bit(not); /* 094 spill 5 normal */
SPILL_64bit(not); /* 098 spill 6 normal */
SPILL_mixed; /* 09C spill 7 normal */
NOT4; /* 0A0 spill 0 other */
SPILL_32bit_tt1(ASI_AIUS,so1); /* 0A4 spill 1 other */
SPILL_64bit_tt1(ASI_AIUS,so1); /* 0A8 spill 2 other */
SPILL_32bit_tt1(ASI_AIUS,so1); /* 0AC spill 3 other */
SPILL_64bit_tt1(ASI_AIUS,so1); /* 0B0 spill 4 other */
NOT4; /* 0B4 spill 5 other */
NOT4; /* 0B8 spill 6 other */
NOT4; /* 0BC spill 7 other */
NOT4; /* 0C0 fill 0 normal */
FILL_32bit_tt1(ASI_AIUP,fn1); /* 0C4 fill 1 normal */
FILL_64bit_tt1(ASI_AIUP,fn1); /* 0C8 fill 2 normal */
FILL_32bit_tt1(ASI_AIUP,fn1); /* 0CC fill 3 normal */
FILL_64bit_tt1(ASI_AIUP,fn1); /* 0D0 fill 4 normal */
FILL_32bit(not); /* 0D4 fill 5 normal */
FILL_64bit(not); /* 0D8 fill 6 normal */
FILL_mixed; /* 0DC fill 7 normal */
NOT4; NOT4; NOT4; NOT4; /* 0E0 - 0EF unused */
NOT4; NOT4; NOT4; NOT4; /* 0F0 - 0FF unused */
LABELED_BAD(tt1_swtrap0); /* 100 fast ecache ECC error (cont) */
LABELED_BAD(tt1_swtrap1); /* 101 Ch+ D$ parity error (cont) */
LABELED_BAD(tt1_swtrap2); /* 102 Ch+ I$ parity error (cont) */
NOT; /* 103 reserved */
/*
* We only reserve the above four special case soft traps for code running
* at TL>0, so we can truncate the trap table here.
*/
etrap_table:
.size trap_table, (.-trap_table)
.size scb, (.-scb)
/*
* We get to exec_fault in the case of an instruction miss and tte
* has no execute bit set. We go to tl0 to handle it.
*
* g1 = tsbe pointer (in/clobbered)
* g2 = tag access register (in)
* g3 - g4 = scratch (clobbered)
* g5 = tsbe data (in)
* g6 = scratch (clobbered)
*/
ALTENTRY(exec_fault)
TRACE_TSBHIT(0x200)
SWITCH_GLOBALS
mov MMU_TAG_ACCESS, %g4
ldxa [%g4]ASI_IMMU, %g2 ! arg1 = addr
mov T_INSTR_MMU_MISS, %g3 ! arg2 = traptype
set trap, %g1
ba,pt %xcc, sys_trap
mov -1, %g4
.mmu_exception_not_aligned:
rdpr %tstate, %g1
btst TSTATE_PRIV, %g1
bnz,pn %icc, 2f
nop
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldn [%g1 + T_PROCP], %g1 ! load proc pointer
ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
brz,pt %g5, 2f
nop
ldn [%g5 + P_UTRAP15], %g5 ! unaligned utrap?
brz,pn %g5, 2f
nop
btst 1, %sp
bz,pt %xcc, 1f ! 32 bit user program
nop
ba,pt %xcc, .setup_v9utrap ! 64 bit user program
nop
1:
ba,pt %xcc, .setup_utrap
or %g2, %g0, %g7
2:
ba,pt %xcc, .mmu_exception_end
mov T_ALIGNMENT, %g1
.mmu_priv_exception:
rdpr %tstate, %g1
btst TSTATE_PRIV, %g1
bnz,pn %icc, 1f
nop
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldn [%g1 + T_PROCP], %g1 ! load proc pointer
ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
brz,pt %g5, 1f
nop
ldn [%g5 + P_UTRAP16], %g5
brnz,pt %g5, .setup_v9utrap
nop
1:
mov T_PRIV_INSTR, %g1
.mmu_exception_end:
CPU_INDEX(%g4, %g5)
set cpu_core, %g5
sllx %g4, CPU_CORE_SHIFT, %g4
add %g4, %g5, %g4
lduh [%g4 + CPUC_DTRACE_FLAGS], %g5
andcc %g5, CPU_DTRACE_NOFAULT, %g0
bz %xcc, .mmu_exception_tlb_chk
or %g5, CPU_DTRACE_BADADDR, %g5
stuh %g5, [%g4 + CPUC_DTRACE_FLAGS]
done
.mmu_exception_tlb_chk:
GET_CPU_IMPL(%g5) ! check SFSR.FT to see if this
cmp %g5, PANTHER_IMPL ! is a TLB parity error. But
bne 2f ! we only do this check while
mov 1, %g4 ! running on Panther CPUs
sllx %g4, PN_SFSR_PARITY_SHIFT, %g4 ! since US-I/II use the same
andcc %g3, %g4, %g0 ! bit for something else which
bz 2f ! will be handled later.
nop
.mmu_exception_is_tlb_parity:
.weak itlb_parity_trap
.weak dtlb_parity_trap
set itlb_parity_trap, %g4
cmp %g1, T_INSTR_EXCEPTION ! branch to the itlb or
be 3f ! dtlb parity handler
nop ! if this trap is due
set dtlb_parity_trap, %g4
cmp %g1, T_DATA_EXCEPTION ! to a IMMU exception
be 3f ! or DMMU exception.
nop
2:
sllx %g3, 32, %g3
or %g3, %g1, %g3
set trap, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
3:
jmp %g4 ! off to the appropriate
nop ! TLB parity handler
.fp_disabled:
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
#ifdef SF_ERRATA_30 /* call causes fp-disabled */
brz,a,pn %g1, 2f
nop
#endif
rdpr %tstate, %g4
btst TSTATE_PRIV, %g4
#ifdef SF_ERRATA_30 /* call causes fp-disabled */
bnz,pn %icc, 2f
nop
#else
bnz,a,pn %icc, ptl1_panic
mov PTL1_BAD_FPTRAP, %g1
#endif
ldn [%g1 + T_PROCP], %g1 ! load proc pointer
ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
brz,a,pt %g5, 2f
nop
ldn [%g5 + P_UTRAP7], %g5 ! fp_disabled utrap?
brz,a,pn %g5, 2f
nop
btst 1, %sp
bz,a,pt %xcc, 1f ! 32 bit user program
nop
ba,a,pt %xcc, .setup_v9utrap ! 64 bit user program
nop
1:
ba,pt %xcc, .setup_utrap
or %g0, %g0, %g7
2:
set fp_disabled, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
.fp_ieee_exception:
rdpr %tstate, %g1
btst TSTATE_PRIV, %g1
bnz,a,pn %icc, ptl1_panic
mov PTL1_BAD_FPTRAP, %g1
CPU_ADDR(%g1, %g4) ! load CPU struct addr
stx %fsr, [%g1 + CPU_TMP1]
ldx [%g1 + CPU_TMP1], %g2
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldn [%g1 + T_PROCP], %g1 ! load proc pointer
ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
brz,a,pt %g5, 1f
nop
ldn [%g5 + P_UTRAP8], %g5
brnz,a,pt %g5, .setup_v9utrap
nop
1:
set _fp_ieee_exception, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
/*
* Register Inputs:
* %g5 user trap handler
* %g7 misaligned addr - for alignment traps only
*/
.setup_utrap:
set trap, %g1 ! setup in case we go
mov T_FLUSH_PCB, %g3 ! through sys_trap on
sub %g0, 1, %g4 ! the save instruction below
/*
* If the DTrace pid provider is single stepping a copied-out
* instruction, t->t_dtrace_step will be set. In that case we need
* to abort the single-stepping (since execution of the instruction
* was interrupted) and use the value of t->t_dtrace_npc as the %npc.
*/
save %sp, -SA(MINFRAME32), %sp ! window for trap handler
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
rdpr %tnpc, %l2 ! arg1 == tnpc
brz,pt %g2, 1f
rdpr %tpc, %l1 ! arg0 == tpc
ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
ldn [%g1 + T_DTRACE_NPC], %l2 ! arg1 = t->t_dtrace_npc (step)
brz,pt %g2, 1f
st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1:
mov %g7, %l3 ! arg2 == misaligned address
rdpr %tstate, %g1 ! cwp for trap handler
rdpr %cwp, %g4
bclr TSTATE_CWP_MASK, %g1
wrpr %g1, %g4, %tstate
wrpr %g0, %g5, %tnpc ! trap handler address
FAST_TRAP_DONE
/* NOTREACHED */
.check_v9utrap:
rdpr %tstate, %g1
btst TSTATE_PRIV, %g1
bnz,a,pn %icc, 3f
nop
CPU_ADDR(%g4, %g1) ! load CPU struct addr
ldn [%g4 + CPU_THREAD], %g5 ! load thread pointer
ldn [%g5 + T_PROCP], %g5 ! load proc pointer
ldn [%g5 + P_UTRAPS], %g5 ! are there utraps?
cmp %g3, T_SOFTWARE_TRAP
bne,a,pt %icc, 1f
nop
brz,pt %g5, 3f ! if p_utraps == NULL goto trap()
rdpr %tt, %g3 ! delay - get actual hw trap type
sub %g3, 254, %g1 ! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
ba,pt %icc, 2f
smul %g1, CPTRSIZE, %g2
1:
brz,a,pt %g5, 3f ! if p_utraps == NULL goto trap()
nop
cmp %g3, T_UNIMP_INSTR
bne,a,pt %icc, 2f
nop
mov 1, %g1
st %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
rdpr %tpc, %g1 ! ld trapping instruction using
lduwa [%g1]ASI_AIUP, %g1 ! "AS IF USER" ASI which could fault
st %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
sethi %hi(0xc1c00000), %g4 ! setup mask for illtrap instruction
andcc %g1, %g4, %g4 ! and instruction with mask
bnz,a,pt %icc, 3f ! if %g4 == zero, %g1 is an ILLTRAP
nop ! fall thru to setup
2:
ldn [%g5 + %g2], %g5
brnz,a,pt %g5, .setup_v9utrap
nop
3:
set trap, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
/* NOTREACHED */
/*
* Register Inputs:
* %g5 user trap handler
*/
.setup_v9utrap:
set trap, %g1 ! setup in case we go
mov T_FLUSH_PCB, %g3 ! through sys_trap on
sub %g0, 1, %g4 ! the save instruction below
/*
* If the DTrace pid provider is single stepping a copied-out
* instruction, t->t_dtrace_step will be set. In that case we need
* to abort the single-stepping (since execution of the instruction
* was interrupted) and use the value of t->t_dtrace_npc as the %npc.
*/
save %sp, -SA(MINFRAME64), %sp ! window for trap handler
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
rdpr %tnpc, %l7 ! arg1 == tnpc
brz,pt %g2, 1f
rdpr %tpc, %l6 ! arg0 == tpc
ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
ldn [%g1 + T_DTRACE_NPC], %l7 ! arg1 == t->t_dtrace_npc (step)
brz,pt %g2, 1f
st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1:
rdpr %tstate, %g2 ! cwp for trap handler
rdpr %cwp, %g4
bclr TSTATE_CWP_MASK, %g2
wrpr %g2, %g4, %tstate
ldn [%g1 + T_PROCP], %g4 ! load proc pointer
ldn [%g4 + P_AS], %g4 ! load as pointer
ldn [%g4 + A_USERLIMIT], %g4 ! load as userlimit
cmp %l7, %g4 ! check for single-step set
bne,pt %xcc, 4f
nop
ldn [%g1 + T_LWP], %g1 ! load klwp pointer
ld [%g1 + PCB_STEP], %g4 ! load single-step flag
cmp %g4, STEP_ACTIVE ! step flags set in pcb?
bne,pt %icc, 4f
nop
stn %g5, [%g1 + PCB_TRACEPC] ! save trap handler addr in pcb
mov %l7, %g4 ! on entry to precise user trap
add %l6, 4, %l7 ! handler, %l6 == pc, %l7 == npc
! at time of trap
wrpr %g0, %g4, %tnpc ! generate FLTBOUNDS,
! %g4 == userlimit
FAST_TRAP_DONE
/* NOTREACHED */
4:
wrpr %g0, %g5, %tnpc ! trap handler address
FAST_TRAP_DONE_CHK_INTR
/* NOTREACHED */
.fp_exception:
CPU_ADDR(%g1, %g4)
stx %fsr, [%g1 + CPU_TMP1]
ldx [%g1 + CPU_TMP1], %g2
/*
* Cheetah takes unfinished_FPop trap for certain range of operands
* to the "fitos" instruction. Instead of going through the slow
* software emulation path, we try to simulate the "fitos" instruction
* via "fitod" and "fdtos" provided the following conditions are met:
*
* fpu_exists is set (if DEBUG)
* not in privileged mode
* ftt is unfinished_FPop
* NXM IEEE trap is not enabled
* instruction at %tpc is "fitos"
*
* Usage:
* %g1 per cpu address
* %g2 %fsr
* %g6 user instruction
*
* Note that we can take a memory access related trap while trying
* to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
* flag to catch those traps and let the SFMMU code deal with page
* fault and data access exception.
*/
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
sethi %hi(fpu_exists), %g7
ld [%g7 + %lo(fpu_exists)], %g7
brz,pn %g7, .fp_exception_cont
nop
#endif
rdpr %tstate, %g7 ! branch if in privileged mode
btst TSTATE_PRIV, %g7
bnz,pn %xcc, .fp_exception_cont
srl %g2, FSR_FTT_SHIFT, %g7 ! extract ftt from %fsr
and %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
cmp %g7, FTT_UNFIN
set FSR_TEM_NX, %g5
bne,pn %xcc, .fp_exception_cont ! branch if NOT unfinished_FPop
andcc %g2, %g5, %g0
bne,pn %xcc, .fp_exception_cont ! branch if FSR_TEM_NX enabled
rdpr %tpc, %g5 ! get faulting PC
or %g0, 1, %g7
st %g7, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
lda [%g5]ASI_USER, %g6 ! get user's instruction
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
set FITOS_INSTR_MASK, %g7
and %g6, %g7, %g7
set FITOS_INSTR, %g5
cmp %g7, %g5
bne,pn %xcc, .fp_exception_cont ! branch if not FITOS_INSTR
nop
/*
* This is unfinished FPops trap for "fitos" instruction. We
* need to simulate "fitos" via "fitod" and "fdtos" instruction
* sequence.
*
* We need a temporary FP register to do the conversion. Since
* both source and destination operands for the "fitos" instruction
* have to be within %f0-%f31, we use an FP register from the upper
* half to guarantee that it won't collide with the source or the
* dest operand. However, we do have to save and restore its value.
*
* We use %d62 as a temporary FP register for the conversion and
* branch to appropriate instruction within the conversion tables
* based upon the rs2 and rd values.
*/
std %d62, [%g1 + CPU_TMP1] ! save original value
srl %g6, FITOS_RS2_SHIFT, %g7
and %g7, FITOS_REG_MASK, %g7
set _fitos_fitod_table, %g4
sllx %g7, 2, %g7
jmp %g4 + %g7
ba,pt %xcc, _fitos_fitod_done
.empty
_fitos_fitod_table:
fitod %f0, %d62
fitod %f1, %d62
fitod %f2, %d62
fitod %f3, %d62
fitod %f4, %d62
fitod %f5, %d62
fitod %f6, %d62
fitod %f7, %d62
fitod %f8, %d62
fitod %f9, %d62
fitod %f10, %d62
fitod %f11, %d62
fitod %f12, %d62
fitod %f13, %d62
fitod %f14, %d62
fitod %f15, %d62
fitod %f16, %d62
fitod %f17, %d62
fitod %f18, %d62
fitod %f19, %d62
fitod %f20, %d62
fitod %f21, %d62
fitod %f22, %d62
fitod %f23, %d62
fitod %f24, %d62
fitod %f25, %d62
fitod %f26, %d62
fitod %f27, %d62
fitod %f28, %d62
fitod %f29, %d62
fitod %f30, %d62
fitod %f31, %d62
_fitos_fitod_done:
/*
* Now convert data back into single precision
*/
srl %g6, FITOS_RD_SHIFT, %g7
and %g7, FITOS_REG_MASK, %g7
set _fitos_fdtos_table, %g4
sllx %g7, 2, %g7
jmp %g4 + %g7
ba,pt %xcc, _fitos_fdtos_done
.empty
_fitos_fdtos_table:
fdtos %d62, %f0
fdtos %d62, %f1
fdtos %d62, %f2
fdtos %d62, %f3
fdtos %d62, %f4
fdtos %d62, %f5
fdtos %d62, %f6
fdtos %d62, %f7
fdtos %d62, %f8
fdtos %d62, %f9
fdtos %d62, %f10
fdtos %d62, %f11
fdtos %d62, %f12
fdtos %d62, %f13
fdtos %d62, %f14
fdtos %d62, %f15
fdtos %d62, %f16
fdtos %d62, %f17
fdtos %d62, %f18
fdtos %d62, %f19
fdtos %d62, %f20
fdtos %d62, %f21
fdtos %d62, %f22
fdtos %d62, %f23
fdtos %d62, %f24
fdtos %d62, %f25
fdtos %d62, %f26
fdtos %d62, %f27
fdtos %d62, %f28
fdtos %d62, %f29
fdtos %d62, %f30
fdtos %d62, %f31
_fitos_fdtos_done:
ldd [%g1 + CPU_TMP1], %d62 ! restore %d62
#if DEBUG
/*
* Update FPop_unfinished trap kstat
*/
set fpustat+FPUSTAT_UNFIN_KSTAT, %g7
ldx [%g7], %g5
1:
add %g5, 1, %g6
casxa [%g7] ASI_N, %g5, %g6
cmp %g5, %g6
bne,a,pn %xcc, 1b
or %g0, %g6, %g5
/*
* Update fpu_sim_fitos kstat
*/
set fpuinfo+FPUINFO_FITOS_KSTAT, %g7
ldx [%g7], %g5
1:
add %g5, 1, %g6
casxa [%g7] ASI_N, %g5, %g6
cmp %g5, %g6
bne,a,pn %xcc, 1b
or %g0, %g6, %g5
#endif /* DEBUG */
FAST_TRAP_DONE
.fp_exception_cont:
/*
* Let _fp_exception deal with simulating FPop instruction.
* Note that we need to pass %fsr in %g2 (already read above).
*/
set _fp_exception, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
.clean_windows:
set trap, %g1
mov T_FLUSH_PCB, %g3
sub %g0, 1, %g4
save
flushw
restore
wrpr %g0, %g0, %cleanwin ! no clean windows
CPU_ADDR(%g4, %g5)
ldn [%g4 + CPU_MPCB], %g4
brz,a,pn %g4, 1f
nop
ld [%g4 + MPCB_WSTATE], %g5
add %g5, WSTATE_CLEAN_OFFSET, %g5
wrpr %g0, %g5, %wstate
1: FAST_TRAP_DONE
/*
* .spill_clean: clean the previous window, restore the wstate, and
* "done".
*
* Entry: %g7 contains new wstate
*/
.spill_clean:
sethi %hi(nwin_minus_one), %g5
ld [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
rdpr %cwp, %g6 ! %g6 = %cwp
deccc %g6 ! %g6--
movneg %xcc, %g5, %g6 ! if (%g6<0) %g6 = nwin-1
wrpr %g6, %cwp
TT_TRACE_L(trace_win)
clr %l0
clr %l1
clr %l2
clr %l3
clr %l4
clr %l5
clr %l6
clr %l7
wrpr %g0, %g7, %wstate
saved
retry ! restores correct %cwp
.fix_alignment:
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldn [%g1 + T_PROCP], %g1
mov 1, %g2
stb %g2, [%g1 + P_FIXALIGNMENT]
FAST_TRAP_DONE
#define STDF_REG(REG, ADDR, TMP) \
sll REG, 3, REG ;\
mark1: set start1, TMP ;\
jmp REG + TMP ;\
nop ;\
start1: ba,pt %xcc, done1 ;\
std %f0, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f32, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f2, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f34, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f4, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f36, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f6, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f38, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f8, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f40, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f10, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f42, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f12, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f44, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f14, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f46, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f16, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f48, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f18, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f50, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f20, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f52, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f22, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f54, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f24, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f56, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f26, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f58, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f28, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f60, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f30, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f62, [ADDR + CPU_TMP1] ;\
done1:
#define LDDF_REG(REG, ADDR, TMP) \
sll REG, 3, REG ;\
mark2: set start2, TMP ;\
jmp REG + TMP ;\
nop ;\
start2: ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f0 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f32 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f2 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f34 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f4 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f36 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f6 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f38 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f8 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f40 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f10 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f42 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f12 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f44 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f14 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f46 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f16 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f48 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f18 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f50 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f20 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f52 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f22 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f54 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f24 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f56 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f26 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f58 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f28 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f60 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f30 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f62 ;\
done2:
.lddf_exception_not_aligned:
/*
* Cheetah overwrites SFAR on a DTLB miss, hence read it now.
*/
ldxa [MMU_SFAR]%asi, %g5 ! misaligned vaddr in %g5
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
sethi %hi(fpu_exists), %g2 ! check fpu_exists
ld [%g2 + %lo(fpu_exists)], %g2
brz,a,pn %g2, 4f
nop
#endif
CPU_ADDR(%g1, %g4)
or %g0, 1, %g4
st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
rdpr %tpc, %g2
lda [%g2]ASI_AIUP, %g6 ! get the user's lddf instruction
srl %g6, 23, %g1 ! using ldda or not?
and %g1, 1, %g1
brz,a,pt %g1, 2f ! check for ldda instruction
nop
srl %g6, 13, %g1 ! check immflag
and %g1, 1, %g1
rdpr %tstate, %g2 ! %tstate in %g2
brnz,a,pn %g1, 1f
srl %g2, 31, %g1 ! get asi from %tstate
srl %g6, 5, %g1 ! get asi from instruction
and %g1, 0xFF, %g1 ! imm_asi field
1:
cmp %g1, ASI_P ! primary address space
be,a,pt %icc, 2f
nop
cmp %g1, ASI_PNF ! primary no fault address space
be,a,pt %icc, 2f
nop
cmp %g1, ASI_S ! secondary address space
be,a,pt %icc, 2f
nop
cmp %g1, ASI_SNF ! secondary no fault address space
bne,a,pn %icc, 3f
nop
2:
lduwa [%g5]ASI_USER, %g7 ! get first half of misaligned data
add %g5, 4, %g5 ! increment misaligned data address
lduwa [%g5]ASI_USER, %g5 ! get second half of misaligned data
sllx %g7, 32, %g7
or %g5, %g7, %g5 ! combine data
CPU_ADDR(%g7, %g1) ! save data on a per-cpu basis
stx %g5, [%g7 + CPU_TMP1] ! save in cpu_tmp1
srl %g6, 25, %g3 ! %g6 has the instruction
and %g3, 0x1F, %g3 ! %g3 has rd
LDDF_REG(%g3, %g7, %g4)
CPU_ADDR(%g1, %g4)
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
FAST_TRAP_DONE
3:
CPU_ADDR(%g1, %g4)
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
4:
set T_USER, %g3 ! trap type in %g3
or %g3, T_LDDF_ALIGN, %g3
mov %g5, %g2 ! misaligned vaddr in %g2
set fpu_trap, %g1 ! goto C for the little and
ba,pt %xcc, sys_trap ! no fault little asi's
sub %g0, 1, %g4
.stdf_exception_not_aligned:
/*
* Cheetah overwrites SFAR on a DTLB miss, hence read it now.
*/
ldxa [MMU_SFAR]%asi, %g5 ! misaligned vaddr in %g5
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
sethi %hi(fpu_exists), %g7 ! check fpu_exists
ld [%g7 + %lo(fpu_exists)], %g3
brz,a,pn %g3, 4f
nop
#endif
CPU_ADDR(%g1, %g4)
or %g0, 1, %g4
st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
rdpr %tpc, %g2
lda [%g2]ASI_AIUP, %g6 ! get the user's stdf instruction
srl %g6, 23, %g1 ! using stda or not?
and %g1, 1, %g1
brz,a,pt %g1, 2f ! check for stda instruction
nop
srl %g6, 13, %g1 ! check immflag
and %g1, 1, %g1
rdpr %tstate, %g2 ! %tstate in %g2
brnz,a,pn %g1, 1f
srl %g2, 31, %g1 ! get asi from %tstate
srl %g6, 5, %g1 ! get asi from instruction
and %g1, 0xFF, %g1 ! imm_asi field
1:
cmp %g1, ASI_P ! primary address space
be,a,pt %icc, 2f
nop
cmp %g1, ASI_S ! secondary address space
bne,a,pn %icc, 3f
nop
2:
srl %g6, 25, %g6
and %g6, 0x1F, %g6 ! %g6 has rd
CPU_ADDR(%g7, %g1)
STDF_REG(%g6, %g7, %g4) ! STDF_REG(REG, ADDR, TMP)
ldx [%g7 + CPU_TMP1], %g6
srlx %g6, 32, %g7
stuwa %g7, [%g5]ASI_USER ! first half
add %g5, 4, %g5 ! increment misaligned data address
stuwa %g6, [%g5]ASI_USER ! second half
CPU_ADDR(%g1, %g4)
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
FAST_TRAP_DONE
3:
CPU_ADDR(%g1, %g4)
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
4:
set T_USER, %g3 ! trap type in %g3
or %g3, T_STDF_ALIGN, %g3
mov %g5, %g2 ! misaligned vaddr in %g2
set fpu_trap, %g1 ! goto C for the little and
ba,pt %xcc, sys_trap ! nofault little asi's
sub %g0, 1, %g4
#ifdef DEBUG_USER_TRAPTRACECTL
.traptrace_freeze:
mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
TT_TRACE_L(trace_win)
mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
set trap_freeze, %g1
mov 1, %g2
st %g2, [%g1]
FAST_TRAP_DONE
.traptrace_unfreeze:
set trap_freeze, %g1
st %g0, [%g1]
mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
TT_TRACE_L(trace_win)
mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
FAST_TRAP_DONE
#endif /* DEBUG_USER_TRAPTRACECTL */
.getcc:
CPU_ADDR(%g1, %g2)
stx %o0, [%g1 + CPU_TMP1] ! save %o0
stx %o1, [%g1 + CPU_TMP2] ! save %o1
rdpr %tstate, %g3 ! get tstate
srlx %g3, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
set PSR_ICC, %g2
and %o0, %g2, %o0 ! mask out the rest
srl %o0, PSR_ICC_SHIFT, %o0 ! right justify
rdpr %pstate, %o1
wrpr %o1, PSTATE_AG, %pstate ! get into normal globals
mov %o0, %g1 ! move ccr to normal %g1
wrpr %g0, %o1, %pstate ! back into alternate globals
ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
ldx [%g1 + CPU_TMP2], %o1 ! restore %o1
FAST_TRAP_DONE
.setcc:
CPU_ADDR(%g1, %g2)
stx %o0, [%g1 + CPU_TMP1] ! save %o0
stx %o1, [%g1 + CPU_TMP2] ! save %o1
rdpr %pstate, %o0
wrpr %o0, PSTATE_AG, %pstate ! get into normal globals
mov %g1, %o1
wrpr %g0, %o0, %pstate ! back to alternates
sll %o1, PSR_ICC_SHIFT, %g2
set PSR_ICC, %g3
and %g2, %g3, %g2 ! mask out rest
sllx %g2, PSR_TSTATE_CC_SHIFT, %g2
rdpr %tstate, %g3 ! get tstate
srl %g3, 0, %g3 ! clear upper word
or %g3, %g2, %g3 ! or in new bits
wrpr %g3, %tstate
ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
ldx [%g1 + CPU_TMP2], %o1 ! restore %o1
FAST_TRAP_DONE
/*
* getpsr(void)
* Note that the xcc part of the ccr is not provided.
* The V8 code shows why the V9 trap is not faster:
* #define GETPSR_TRAP() \
* mov %psr, %i0; jmp %l2; rett %l2+4; nop;
*/
.type .getpsr, #function
.getpsr:
rdpr %tstate, %g1 ! get tstate
srlx %g1, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
set PSR_ICC, %g2
and %o0, %g2, %o0 ! mask out the rest
rd %fprs, %g1 ! get fprs
and %g1, FPRS_FEF, %g2 ! mask out dirty upper/lower
sllx %g2, PSR_FPRS_FEF_SHIFT, %g2 ! shift fef to V8 psr.ef
or %o0, %g2, %o0 ! or result into psr.ef
set V9_PSR_IMPLVER, %g2 ! SI assigned impl/ver: 0xef
or %o0, %g2, %o0 ! or psr.impl/ver
FAST_TRAP_DONE
SET_SIZE(.getpsr)
/*
* setpsr(newpsr)
* Note that there is no support for ccr.xcc in the V9 code.
*/
.type .setpsr, #function
.setpsr:
rdpr %tstate, %g1 ! get tstate
! setx TSTATE_V8_UBITS, %g2
or %g0, CCR_ICC, %g3
sllx %g3, TSTATE_CCR_SHIFT, %g2
andn %g1, %g2, %g1 ! zero current user bits
set PSR_ICC, %g2
and %g2, %o0, %g2 ! clear all but psr.icc bits
sllx %g2, PSR_TSTATE_CC_SHIFT, %g3 ! shift to tstate.ccr.icc
wrpr %g1, %g3, %tstate ! write tstate
set PSR_EF, %g2
and %g2, %o0, %g2 ! clear all but fp enable bit
srlx %g2, PSR_FPRS_FEF_SHIFT, %g4 ! shift ef to V9 fprs.fef
wr %g0, %g4, %fprs ! write fprs
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1
ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
ldn [%g2 + T_LWP], %g3 ! load klwp pointer
ldn [%g3 + LWP_FPU], %g2 ! get lwp_fpu pointer
stuw %g4, [%g2 + FPU_FPRS] ! write fef value to fpu_fprs
srlx %g4, 2, %g4 ! shift fef value to bit 0
stub %g4, [%g2 + FPU_EN] ! write fef value to fpu_en
FAST_TRAP_DONE
SET_SIZE(.setpsr)
/*
* getlgrp
* get home lgrpid on which the calling thread is currently executing.
*/
.type .getlgrp, #function
.getlgrp:
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
ld [%g1 + CPU_ID], %o0 ! load cpu_id
ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
ldn [%g2 + T_LPL], %g2 ! load lpl pointer
ld [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid
sra %g1, 0, %o1
FAST_TRAP_DONE
SET_SIZE(.getlgrp)
/*
* Entry for old 4.x trap (trap 0).
*/
ENTRY_NP(syscall_trap_4x)
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
ldn [%g2 + T_LWP], %g2 ! load klwp pointer
ld [%g2 + PCB_TRAP0], %g2 ! lwp->lwp_pcb.pcb_trap0addr
brz,pn %g2, 1f ! has it been set?
st %l0, [%g1 + CPU_TMP1] ! delay - save some locals
st %l1, [%g1 + CPU_TMP2]
rdpr %tnpc, %l1 ! save old tnpc
wrpr %g0, %g2, %tnpc ! setup tnpc
rdpr %pstate, %l0
wrpr %l0, PSTATE_AG, %pstate ! switch to normal globals
mov %l1, %g6 ! pass tnpc to user code in %g6
wrpr %l0, %g0, %pstate ! switch back to alternate globals
! Note that %g1 still contains CPU struct addr
ld [%g1 + CPU_TMP2], %l1 ! restore locals
ld [%g1 + CPU_TMP1], %l0
FAST_TRAP_DONE_CHK_INTR
1:
mov %g1, %l0
st %l1, [%g1 + CPU_TMP2]
rdpr %pstate, %l1
wrpr %l1, PSTATE_AG, %pstate
!
! check for old syscall mmap which is the only different one which
! must be the same. Others are handled in the compatibility library.
!
cmp %g1, OSYS_mmap ! compare to old 4.x mmap
movz %icc, SYS_mmap, %g1
wrpr %g0, %l1, %pstate
ld [%l0 + CPU_TMP2], %l1 ! restore locals
ld [%l0 + CPU_TMP1], %l0
SYSCALL(syscall_trap32)
SET_SIZE(syscall_trap_4x)
/*
* Handler for software trap 9.
* Set trap0 emulation address for old 4.x system call trap.
* XXX - this should be a system call.
*/
ENTRY_NP(set_trap0_addr)
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
ldn [%g2 + T_LWP], %g2 ! load klwp pointer
st %l0, [%g1 + CPU_TMP1] ! save some locals
st %l1, [%g1 + CPU_TMP2]
rdpr %pstate, %l0
wrpr %l0, PSTATE_AG, %pstate
mov %g1, %l1
wrpr %g0, %l0, %pstate
andn %l1, 3, %l1 ! force alignment
st %l1, [%g2 + PCB_TRAP0] ! lwp->lwp_pcb.pcb_trap0addr
ld [%g1 + CPU_TMP1], %l0 ! restore locals
ld [%g1 + CPU_TMP2], %l1
FAST_TRAP_DONE
SET_SIZE(set_trap0_addr)
/*
* mmu_trap_tl1
* trap handler for unexpected mmu traps.
* simply checks if the trap was a user lddf/stdf alignment trap, in which
* case we go to fpu_trap or a user trap from the window handler, in which
* case we go save the state on the pcb. Otherwise, we go to ptl1_panic.
*/
.type mmu_trap_tl1, #function
mmu_trap_tl1:
#ifdef TRAPTRACE
TRACE_PTR(%g5, %g6)
GET_TRACE_TICK(%g6)
stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
rdpr %tl, %g6
stha %g6, [%g5 + TRAP_ENT_TL]%asi
rdpr %tt, %g6
stha %g6, [%g5 + TRAP_ENT_TT]%asi
rdpr %tstate, %g6
stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi
stna %sp, [%g5 + TRAP_ENT_SP]%asi
stna %g0, [%g5 + TRAP_ENT_TR]%asi
rdpr %tpc, %g6
stna %g6, [%g5 + TRAP_ENT_TPC]%asi
set MMU_SFAR, %g6
ldxa [%g6]ASI_DMMU, %g6
stxa %g6, [%g5 + TRAP_ENT_F1]%asi
CPU_PADDR(%g7, %g6);
add %g7, CPU_TL1_HDLR, %g7
lda [%g7]ASI_MEM, %g6
stxa %g6, [%g5 + TRAP_ENT_F2]%asi
set 0xdeadbeef, %g6
stna %g6, [%g5 + TRAP_ENT_F3]%asi
stna %g6, [%g5 + TRAP_ENT_F4]%asi
TRACE_NEXT(%g5, %g6, %g7)
#endif /* TRAPTRACE */
GET_CPU_IMPL(%g5)
cmp %g5, PANTHER_IMPL
bne mmu_trap_tl1_4
nop
rdpr %tt, %g5
cmp %g5, T_DATA_EXCEPTION
bne mmu_trap_tl1_4
nop
wr %g0, ASI_DMMU, %asi
ldxa [MMU_SFSR]%asi, %g5
mov 1, %g6
sllx %g6, PN_SFSR_PARITY_SHIFT, %g6
andcc %g5, %g6, %g0
bz mmu_trap_tl1_4
/*
* We are running on a Panther and have hit a DTLB parity error.
*/
ldxa [MMU_TAG_ACCESS]%asi, %g2
mov %g5, %g3
ba,pt %xcc, .mmu_exception_is_tlb_parity
mov T_DATA_EXCEPTION, %g1
mmu_trap_tl1_4:
CPU_PADDR(%g7, %g6);
add %g7, CPU_TL1_HDLR, %g7 ! %g7 = &cpu_m.tl1_hdlr (PA)
/*
* AM is cleared on trap, so addresses are 64 bit
*/
lda [%g7]ASI_MEM, %g6
brz,a,pt %g6, 1f
nop
/*
* We are going to update cpu_m.tl1_hdlr using physical address.
* Flush the D$ line, so that stale data won't be accessed later.
*/
CPU_ADDR(%g6, %g5)
add %g6, CPU_TL1_HDLR, %g6 ! %g6 = &cpu_m.tl1_hdlr (VA)
GET_CPU_IMPL(%g5)
cmp %g5, CHEETAH_IMPL
bl,pn %icc, 3f
sethi %hi(dcache_line_mask), %g5
stxa %g0, [%g7]ASI_DC_INVAL
membar #Sync
ba,pt %xcc, 2f
nop
3:
ld [%g5 + %lo(dcache_line_mask)], %g5
and %g6, %g5, %g5
stxa %g0, [%g5]ASI_DC_TAG
membar #Sync
2:
sta %g0, [%g7]ASI_MEM
SWITCH_GLOBALS ! back to mmu globals
ba,a,pt %xcc, sfmmu_mmu_trap ! handle page faults
1:
rdpr %tt, %g5
rdpr %tl, %g7
sub %g7, 1, %g6
wrpr %g6, %tl
rdpr %tt, %g6
wrpr %g7, %tl
and %g6, WTRAP_TTMASK, %g6
cmp %g6, WTRAP_TYPE
bne,a,pn %xcc, ptl1_panic
mov PTL1_BAD_MMUTRAP, %g1
rdpr %tpc, %g7
/* tpc should be in the trap table */
set trap_table, %g6
cmp %g7, %g6
blt,a,pn %xcc, ptl1_panic
mov PTL1_BAD_MMUTRAP, %g1
set etrap_table, %g6
cmp %g7, %g6
bge,a,pn %xcc, ptl1_panic
mov PTL1_BAD_MMUTRAP, %g1
cmp %g5, T_ALIGNMENT
move %icc, MMU_SFAR, %g6
movne %icc, MMU_TAG_ACCESS, %g6
ldxa [%g6]ASI_DMMU, %g6
andn %g7, WTRAP_ALIGN, %g7 /* 128 byte aligned */
add %g7, WTRAP_FAULTOFF, %g7
wrpr %g0, %g7, %tnpc
done
SET_SIZE(mmu_trap_tl1)
/*
* Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers. These
* traps are valid only when kmdb is loaded. When the debugger is active,
* the code below is rewritten to transfer control to the appropriate
* debugger entry points.
*/
.global kmdb_trap
.align 8
kmdb_trap:
ba,a trap_table0
jmp %g1 + 0
nop
.global kmdb_trap_tl1
.align 8
kmdb_trap_tl1:
ba,a trap_table0
jmp %g1 + 0
nop
/*
* This entry is copied from OBP's trap table during boot.
*/
.global obp_bpt
.align 8
obp_bpt:
NOT
/*
* if kernel, set PCONTEXT to 0 for debuggers
* if user, clear nucleus page sizes
*/
.global kctx_obp_bpt
kctx_obp_bpt:
set obp_bpt, %g2
1:
mov MMU_PCONTEXT, %g1
ldxa [%g1]ASI_DMMU, %g1
srlx %g1, CTXREG_NEXT_SHIFT, %g3
brz,pt %g3, 3f ! nucleus pgsz is 0, no problem
sllx %g3, CTXREG_NEXT_SHIFT, %g3
set CTXREG_CTX_MASK, %g4 ! check Pcontext
btst %g4, %g1
bz,a,pt %xcc, 2f
clr %g3 ! kernel: PCONTEXT=0
xor %g3, %g1, %g3 ! user: clr N_pgsz0/1 bits
2:
set DEMAP_ALL_TYPE, %g1
stxa %g0, [%g1]ASI_DTLB_DEMAP
stxa %g0, [%g1]ASI_ITLB_DEMAP
mov MMU_PCONTEXT, %g1
stxa %g3, [%g1]ASI_DMMU
membar #Sync
sethi %hi(FLUSH_ADDR), %g1
flush %g1 ! flush required by immu
3:
jmp %g2
nop
#ifdef TRAPTRACE
/*
* TRAPTRACE support.
* labels here are branched to with "rd %pc, %g7" in the delay slot.
* Return is done by "jmp %g7 + 4".
*/
trace_gen:
TRACE_PTR(%g3, %g6)
GET_TRACE_TICK(%g6)
stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
rdpr %tl, %g6
stha %g6, [%g3 + TRAP_ENT_TL]%asi
rdpr %tt, %g6
stha %g6, [%g3 + TRAP_ENT_TT]%asi
rdpr %tstate, %g6
stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
stna %sp, [%g3 + TRAP_ENT_SP]%asi
rdpr %tpc, %g6
stna %g6, [%g3 + TRAP_ENT_TPC]%asi
TRACE_NEXT(%g3, %g4, %g5)
jmp %g7 + 4
nop
trace_win:
TRACE_WIN_INFO(0, %l0, %l1, %l2)
! Keep the locals as clean as possible, caller cleans %l4
clr %l2
clr %l1
jmp %l4 + 4
clr %l0
/*
* Trace a tsb hit
* g1 = tsbe pointer (in/clobbered)
* g2 = tag access register (in)
* g3 - g4 = scratch (clobbered)
* g5 = tsbe data (in)
* g6 = scratch (clobbered)
* g7 = pc we jumped here from (in)
*/
! Do not disturb %g5, it will be used after the trace
ALTENTRY(trace_tsbhit)
TRACE_TSBHIT(0)
jmp %g7 + 4
nop
/*
* Trace a TSB miss
*
* g1 = tsb8k pointer (in)
* g2 = tag access register (in)
* g3 = tsb4m pointer (in)
* g4 = tsbe tag (in/clobbered)
* g5 - g6 = scratch (clobbered)
* g7 = pc we jumped here from (in)
*/
.global trace_tsbmiss
trace_tsbmiss:
membar #Sync
sethi %hi(FLUSH_ADDR), %g6
flush %g6
TRACE_PTR(%g5, %g6)
GET_TRACE_TICK(%g6)
stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
stxa %g2, [%g5 + TRAP_ENT_SP]%asi ! tag access
stxa %g4, [%g5 + TRAP_ENT_F1]%asi ! tsb tag
rdpr %tnpc, %g6
stxa %g6, [%g5 + TRAP_ENT_F2]%asi
stna %g1, [%g5 + TRAP_ENT_F3]%asi ! tsb8k pointer
srlx %g1, 32, %g6
stna %g6, [%g5 + TRAP_ENT_F4]%asi ! huh?
rdpr %tpc, %g6
stna %g6, [%g5 + TRAP_ENT_TPC]%asi
rdpr %tl, %g6
stha %g6, [%g5 + TRAP_ENT_TL]%asi
rdpr %tt, %g6
or %g6, TT_MMU_MISS, %g4
stha %g4, [%g5 + TRAP_ENT_TT]%asi
cmp %g6, FAST_IMMU_MISS_TT
be,a %icc, 1f
ldxa [%g0]ASI_IMMU, %g6
ldxa [%g0]ASI_DMMU, %g6
1: stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! tag target
stxa %g3, [%g5 + TRAP_ENT_TR]%asi ! tsb4m pointer
TRACE_NEXT(%g5, %g4, %g6)
jmp %g7 + 4
nop
/*
* g2 = tag access register (in)
* g3 = ctx number (in)
*/
trace_dataprot:
membar #Sync
sethi %hi(FLUSH_ADDR), %g6
flush %g6
TRACE_PTR(%g1, %g6)
GET_TRACE_TICK(%g6)
stxa %g6, [%g1 + TRAP_ENT_TICK]%asi
rdpr %tpc, %g6
stna %g6, [%g1 + TRAP_ENT_TPC]%asi
rdpr %tstate, %g6
stxa %g6, [%g1 + TRAP_ENT_TSTATE]%asi
stxa %g2, [%g1 + TRAP_ENT_SP]%asi ! tag access reg
stxa %g0, [%g1 + TRAP_ENT_TR]%asi
stxa %g0, [%g1 + TRAP_ENT_F1]%asi
stxa %g0, [%g1 + TRAP_ENT_F2]%asi
stxa %g0, [%g1 + TRAP_ENT_F3]%asi
stxa %g0, [%g1 + TRAP_ENT_F4]%asi
rdpr %tl, %g6
stha %g6, [%g1 + TRAP_ENT_TL]%asi
rdpr %tt, %g6
stha %g6, [%g1 + TRAP_ENT_TT]%asi
TRACE_NEXT(%g1, %g4, %g5)
jmp %g7 + 4
nop
#endif /* TRAPTRACE */
/*
* expects offset into tsbmiss area in %g1 and return pc in %g7
*/
stat_mmu:
CPU_INDEX(%g5, %g6)
sethi %hi(tsbmiss_area), %g6
sllx %g5, TSBMISS_SHIFT, %g5
or %g6, %lo(tsbmiss_area), %g6
add %g6, %g5, %g6 /* g6 = tsbmiss area */
ld [%g6 + %g1], %g5
add %g5, 1, %g5
jmp %g7 + 4
st %g5, [%g6 + %g1]
/*
* fast_trap_done, fast_trap_done_chk_intr:
*
* Due to the design of UltraSPARC pipeline, pending interrupts are not
* taken immediately after a RETRY or DONE instruction which causes IE to
* go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
* to execute first before taking any interrupts. If that instruction
* results in other traps, and if the corresponding trap handler runs
* entirely at TL=1 with interrupts disabled, then pending interrupts
* won't be taken until after yet another instruction following the %tpc
* or %tnpc.
*
* A malicious user program can use this feature to block out interrupts
* for extended durations, which can result in send_mondo_timeout kernel
* panic.
*
* This problem is addressed by servicing any pending interrupts via
* sys_trap before returning back to the user mode from a fast trap
* handler. The "done" instruction within a fast trap handler, which
* runs entirely at TL=1 with interrupts disabled, is replaced with the
* FAST_TRAP_DONE macro, which branches control to this fast_trap_done
* entry point.
*
* We check for any pending interrupts here and force a sys_trap to
* service those interrupts, if any. To minimize overhead, pending
* interrupts are checked if the %tpc happens to be at 16K boundary,
* which allows a malicious program to execute at most 4K consecutive
* instructions before we service any pending interrupts. If a worst
* case fast trap handler takes about 2 usec, then interrupts will be
* blocked for at most 8 msec, less than a clock tick.
*
* For the cases where we don't know if the %tpc will cross a 16K
* boundary, we can't use the above optimization and always process
* any pending interrupts via fast_frap_done_chk_intr entry point.
*
* Entry Conditions:
* %pstate am:0 priv:1 ie:0
* globals are AG (not normal globals)
*/
.global fast_trap_done, fast_trap_done_chk_intr
fast_trap_done:
rdpr %tpc, %g5
sethi %hi(0xffffc000), %g6 ! 1's complement of 0x3fff
andncc %g5, %g6, %g0 ! check lower 14 bits of %tpc
bz,a,pn %icc, 1f ! branch if zero (lower 32 bits only)
ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g5
done
fast_trap_done_chk_intr:
ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g5
1: rd SOFTINT, %g6
and %g5, IRSR_BUSY, %g5
orcc %g5, %g6, %g0
bnz,pn %xcc, 2f ! branch if any pending intr
nop
done
2:
/*
* We get here if there are any pending interrupts.
* Adjust %tpc/%tnpc as we'll be resuming via "retry"
* instruction.
*/
rdpr %tnpc, %g5
wrpr %g0, %g5, %tpc
add %g5, 4, %g5
wrpr %g0, %g5, %tnpc
/*
* Force a dummy sys_trap call so that interrupts can be serviced.
*/
set fast_trap_dummy_call, %g1
ba,pt %xcc, sys_trap
mov -1, %g4
fast_trap_dummy_call:
retl
nop
#endif /* lint */