fasttrap_isa.c revision f498645a3eecf2ddd304b4ea9c7f1b4c155ff79e
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
#include <sys/fasttrap_isa.h>
#include <sys/fasttrap_impl.h>
#include <sys/dtrace_impl.h>
#include <sys/sysmacros.h>
/*
* Lossless User-Land Tracing on SPARC
* -----------------------------------
*
* The Basic Idea
*
* The most important design constraint is, of course, correct execution of
* the user thread above all else. The next most important goal is rapid
* execution. We combine execution of instructions in user-land with
* emulation of certain instructions in the kernel to aim for complete
* correctness and maximal performance.
*
* single-stepping; when we copy an instruction out to the scratch space in
* the ulwp_t structure (held in the %g7 register on SPARC), we can
* effectively single step by setting the PC to our scratch space and leaving
* the NPC alone. This executes the replaced instruction and then continues
* on without having to reenter the kernel as with single- stepping. The
* obvious caveat is for instructions whose execution is PC dependant --
* branches, call and link instructions (call and jmpl), and the rdpc
* instruction. These instructions cannot be executed in the manner described
* so they must be emulated in the kernel.
*
* Emulation for this small set of instructions if fairly simple; the most
* difficult part being emulating branch conditions.
*
*
* A Cache Heavy Portfolio
*
* It's important to note at this time that copying an instruction out to the
* ulwp_t scratch space in user-land is rather complicated. SPARC has
* separate data and instruction caches so any writes to the D$ (using a
* store instruction for example) aren't necessarily reflected in the I$.
* The flush instruction can be used to synchronize the two and must be used
* for any self-modifying code, but the flush instruction only applies to the
* primary address space (the absence of a flusha analogue to the flush
* instruction that accepts an ASI argument is an obvious omission from SPARC
* v9 where the notion of the alternate address space was introduced on
* SPARC). To correctly copy out the instruction we must use a block store
* that doesn't allocate in the D$ and ensures synchronization with the I$;
* see dtrace_blksuword32() for the implementation (this function uses
* ASI_BLK_COMMIT_S to write a block through the secondary ASI in the manner
* ASI_BLK_COMMIT_S ASI.
*
*
* Return Subtleties
*
* When we're firing a return probe we need to expose the value returned by
* the function being traced. Since the function can set the return value
* in its last instruction, we need to fire the return probe only _after_
* the effects of the instruction are apparent. For instructions that we
* emulate, we can call dtrace_probe() after we've performed the emulation;
* for instructions that we execute after we return to user-land, we set
* %pc to the instruction we copied out (as described above) and set %npc
* to a trap instruction stashed in the ulwp_t structure. After the traced
* instruction is executed, the trap instruction returns control to the
* kernel where we can fire the return probe.
*
* This need for a second trap in cases where we execute the traced
* instruction makes it all the more important to emulate the most common
* instructions to avoid the second trip in and out of the kernel.
*
*
* Making it Fast
*
* Since copying out an instruction is neither simple nor inexpensive for the
* CPU, we should attempt to avoid doing it in as many cases as possible.
* Since function entry and return are usually the most interesting probe
* sites, we attempt to tune the performance of the fasttrap provider around
* instructions typically in those places.
*
* Looking at a bunch of functions in libraries and executables reveals that
* most functions begin with either a save or a sethi (to setup a larger
* argument to the save) and end with a restore or an or (in the case of leaf
* functions). To try to improve performance, we emulate all of these
* instructions in the kernel.
*
* The save and restore instructions are a little tricky since they perform
* register window maniplulation. Rather than trying to tinker with the
* register windows from the kernel, we emulate the implicit add that takes
* place as part of those instructions and set the %pc to point to a simple
* save or restore we've hidden in the ulwp_t structure. If we're in a return
* probe so want to make it seem as though the tracepoint has been completely
* executed we need to remember that we've pulled this trick with restore and
* pull registers from the previous window (the one that we'll switch to once
* the simple store instruction is executed) rather than the current one. This
* is why in the case of emulating a restore we set the DTrace CPU flag
* CPU_DTRACE_FAKERESTORE before calling dtrace_probe() for the return probes
* (see fasttrap_return_common()).
*/
#define OP(x) ((x) >> 30)
#define A(x) (((x) >> 29) & 0x01)
#define I(x) (((x) >> 13) & 0x01)
#define RS2(x) (((x) >> 0) & 0x1f)
#define DISP22(x) ((x) & 0x3fffff)
#define DISP19(x) ((x) & 0x7ffff)
#define DISP30(x) ((x) & 0x3fffffff)
#define SW_TRAP(x) ((x) & 0x7f)
#define OP3_OR 0x02
#define OP3_RD 0x28
#define OP3_JMPL 0x38
#define OP3_RETURN 0x39
#define OP3_TCC 0x3a
#define OP3_SAVE 0x3c
#define OP3_RESTORE 0x3d
#define OP3_PREFETCH 0x2d
#define OP3_CASA 0x3c
#define OP3_PREFETCHA 0x3d
#define OP3_CASXA 0x3e
#define OP2_ILLTRAP 0x0
#define OP2_BPcc 0x1
#define OP2_Bicc 0x2
#define OP2_BPr 0x3
#define OP2_SETHI 0x4
#define OP2_FBPfcc 0x5
#define OP2_FBfcc 0x6
#define R_G0 0
#define R_O0 8
#define R_SP 14
#define R_I0 24
#define R_I1 25
#define R_I2 26
#define R_I3 27
/*
* Check the comment in fasttrap.h when changing these offsets or adding
* new instructions.
*/
#define FASTTRAP_OFF_SAVE 64
#define FASTTRAP_OFF_RESTORE 68
#define FASTTRAP_OFF_FTRET 72
#define FASTTRAP_OFF_RETURN 76
/*
* Tunable to let users turn off the fancy save instruction optimization.
* If a program is non-ABI compliant, there's a possibility that the save
* instruction optimization could cause an error.
*/
int fasttrap_optimize_save = 1;
static uint64_t
{
if (argno < 6)
} else {
}
return (value);
}
static void
{
uintptr_t v;
for (i = 0; i < cap; i++) {
x = probe->ftp_argmap[i];
if (x < 6)
argv[i] = 0;
}
} else {
uint32_t v;
for (i = 0; i < cap; i++) {
x = probe->ftp_argmap[i];
if (x < 6)
argv[i] = 0;
}
}
for (; i < argc; i++) {
argv[i] = 0;
}
}
static void
{
break;
}
/*
* Don't sweat it if we can't find the tracepoint again; unlike
* when we're in fasttrap_pid_probe(), finding the tracepoint here
* is not essential to the correct execution of the process.
*/
return;
}
} else {
uintptr_t t[5];
sizeof (t) / sizeof (t[0]), t);
t[2], t[3], t[4]);
}
continue;
}
/*
* If this is only a possible return point, we must
* be looking at a potential tail call in leaf context.
* If the %npc is still within this function, then we
* must have misidentified a jmpl as a tail-call when it
* is, in fact, part of a jump table. It would be nice to
* remove this tracepoint, but this is neither the time
* nor the place.
*/
continue;
/*
* It's possible for a function to branch to the delay slot
* of an instruction that we've identified as a return site.
* We can dectect this spurious return probe activation by
* observing that in this case %npc will be %pc + 4 and %npc
* will be inside the current function (unless the user is
* doing _crazy_ instruction picking in which case there's
* very little we can do). The second check is important
* in case the last instructions of a function make a tail-
* call to the function located immediately subsequent.
*/
continue;
/*
* The first argument is the offset of return tracepoint
* in the function; the remaining arguments are the return
* values.
*
* If fake_restore is set, we need to pull the return values
* out of the %i's rather than the %o's -- a little trickier.
*/
if (!fake_restore) {
} else {
}
}
}
int
{
/*
* It's possible that a user (in a veritable orgy of bad planning)
* could redirect this thread's flow of control before it reached the
* return probe fasttrap. In this case we need to kill the process
* since it's in a unrecoverable state.
*/
if (curthread->t_dtrace_step) {
return (0);
}
/*
* Clear all user tracing flags.
*/
curthread->t_dtrace_ft = 0;
curthread->t_dtrace_pc = 0;
curthread->t_dtrace_npc = 0;
curthread->t_dtrace_scrpc = 0;
curthread->t_dtrace_astpc = 0;
/*
* Treat a child created by a call to vfork(2) as if it were its
* parent. We know that there's only one thread of control in such a
* process: this one.
*/
p = p->p_parent;
}
/*
* Lookup the tracepoint that the process just hit.
*/
break;
}
/*
* If we couldn't find a matching tracepoint, either a tracepoint has
* been inserted without using the pid<pid> ioctl interface (see
* fasttrap_ioctl), or somehow we have mislaid this tracepoint.
*/
return (-1);
}
is_enabled = 1;
continue;
}
/*
* We note that this was an entry probe to help ustack() find
* the first caller.
*/
if (isentry) {
}
if (isentry) {
}
}
/*
* We're about to do a bunch of work so we cache a local copy of
* the tracepoint to emulate the instruction, and then find the
* tracepoint again later if we need to light up any return probes.
*/
/*
* If there's an is-enabled probe conntected to this tracepoint it
* means that there was a 'mov %g0, %o0' instruction that was placed
* there by DTrace when the binary was linked. As this probe is, in
* fact, enabled, we need to stuff 1 into %o0. Accordingly, we can
* bypass all the instruction emulation logic since we know the
* inevitable result. It's possible that a user could construct a
* scenario where the 'is-enabled' probe was on some other
* instruction, but that would be a rather exotic way to shoot oneself
* in the foot.
*/
if (is_enabled) {
goto done;
}
/*
* We emulate certain types of instructions to ensure correctness
* (in the case of position dependent instructions) or optimize
* common cases. The rest we have the thread execute back in user-
* land.
*/
case FASTTRAP_T_SAVE:
{
/*
* This an optimization to let us handle function entry
* probes more efficiently. Many functions begin with a save
* instruction that follows the pattern:
* save %sp, <imm>, %sp
*
* Meanwhile, we've stashed the instruction:
* save %g1, %g0, %sp
*
* off of %g7, so all we have to do is stick the right value
* into %g1 and reset %pc to point to the instruction we've
* cleverly hidden (%npc should not be touched).
*/
imm >>= 19;
break;
}
case FASTTRAP_T_RESTORE:
{
/*
* This is an optimization to let us handle function
* return probes more efficiently. Most non-leaf functions
* end with the sequence:
* ret
* restore <reg>, <reg_or_imm>, %oX
*
* We've stashed the instruction:
* restore %g0, %g0, %g0
*
* off of %g7 so we just need to place the correct value
* in the right %i register (since after our fake-o
* restore, the %i's will become the %o's) and set the %pc
* to point to our hidden restore. We also set fake_restore to
* let fasttrap_return_common() know that it will find the
* return values in the %i's rather than the %o's.
*/
imm >>= 19;
} else {
}
/*
* Convert %o's to %i's; leave %g's as they are.
*/
fake_restore = 1;
break;
}
case FASTTRAP_T_RETURN:
{
/*
* A return instruction is like a jmpl (without the link
* part) that executes an implicit restore. We've stashed
* the instruction:
* return %o0
*
* off of %g7 so we just need to place the target in %o0
* and set the %pc to point to the stashed return instruction.
* We use %o0 since that register disappears after the return
* executes, erasing any evidence of this tampering.
*/
imm >>= 19;
} else {
}
fake_restore = 1;
break;
}
case FASTTRAP_T_OR:
{
imm >>= 19;
} else {
}
break;
}
case FASTTRAP_T_SETHI:
}
break;
case FASTTRAP_T_CCR:
{
ccr >>= 4;
c = (ccr >> 0) & 1;
case 0x0: /* BN */
taken = 0; break;
case 0x1: /* BE */
taken = z; break;
case 0x2: /* BLE */
taken = z | (n ^ v); break;
case 0x3: /* BL */
taken = n ^ v; break;
case 0x4: /* BLEU */
taken = c | z; break;
case 0x5: /* BCS (BLU) */
taken = c; break;
case 0x6: /* BNEG */
taken = n; break;
case 0x7: /* BVS */
taken = v; break;
case 0x8: /* BA */
/*
* We handle the BA case differently since the annul
* bit means something slightly different.
*/
panic("fasttrap: mishandled a branch");
taken = 1; break;
case 0x9: /* BNE */
taken = ~z; break;
case 0xa: /* BG */
taken = ~(z | (n ^ v)); break;
case 0xb: /* BGE */
taken = ~(n ^ v); break;
case 0xc: /* BGU */
taken = ~(c | z); break;
case 0xd: /* BCC (BGEU) */
taken = ~c; break;
case 0xe: /* BPOS */
taken = ~n; break;
case 0xf: /* BVC */
taken = ~v; break;
}
if (taken & 1) {
/*
* Untaken annulled branches don't execute the
* instruction in the delay slot.
*/
} else {
}
break;
}
case FASTTRAP_T_FCC:
{
dtrace_getfsr(&fsr);
} else {
}
case 0x0: /* FBN */
case 0x1: /* FBNE */
case 0x2: /* FBLG */
case 0x3: /* FBUL */
case 0x4: /* FBL */
case 0x5: /* FBUG */
case 0x6: /* FBG */
case 0x7: /* FBU */
case 0x8: /* FBA */
/*
* We handle the FBA case differently since the annul
* bit means something slightly different.
*/
panic("fasttrap: mishandled a branch");
case 0x9: /* FBE */
case 0xa: /* FBUE */
case 0xb: /* FBGE */
case 0xc: /* FBUGE */
case 0xd: /* FBLE */
case 0xe: /* FBULE */
case 0xf: /* FBO */
}
if (taken) {
/*
* Untaken annulled branches don't execute the
* instruction in the delay slot.
*/
} else {
}
break;
}
case FASTTRAP_T_REG:
{
/*
* An ILP32 process shouldn't be using a branch predicated on
* an %i or an %l since it would violate the ABI. It's a
* violation of the ABI because we can't ensure deterministic
* behavior. We should have identified this case when we
* enabled the probe.
*/
case 0x1: /* BRZ */
case 0x2: /* BRLEZ */
case 0x3: /* BRLZ */
case 0x5: /* BRNZ */
case 0x6: /* BRGZ */
case 0x7: /* BRGEZ */
default:
case 0x0:
case 0x4:
panic("fasttrap: mishandled a branch");
}
if (taken) {
/*
* Untaken annulled branches don't execute the
* instruction in the delay slot.
*/
} else {
}
break;
}
case FASTTRAP_T_ALWAYS:
/*
* BAs, BA,As...
*/
/*
* Annulled branch always instructions never execute
* the instruction in the delay slot.
*/
} else {
}
break;
case FASTTRAP_T_RDPC:
break;
case FASTTRAP_T_CALL:
/*
* It's a call _and_ link remember...
*/
break;
case FASTTRAP_T_JMPL:
imm >>= 19;
} else {
}
/*
* Do the link part of the jump-and-link instruction.
*/
break;
case FASTTRAP_T_COMMON:
{
/*
* Copy the instruction to a reserved location in the
* user-land thread structure, then set the PC to that
* location and leave the NPC alone. We take pains to ensure
* consistency in the instruction stream (See SPARC
* Architecture Manual Version 9, sections 8.4.7, A.20, and
* and 13.6.4) by using the ASI ASI_BLK_COMMIT_S to copy the
* instruction into the user's address space without
* bypassing the I$. There's no AS_USER version of this ASI
* (as exist for other ASIs) so we use the lofault
* mechanism to catch faults.
*/
/*
* If the copyout fails, then the process's state
* is not consistent (the effects of the traced
* instruction will never be seen). This process
* cannot be allowed to continue execution.
*/
return (0);
}
}
break;
}
default:
panic("fasttrap: mishandled an instruction");
}
/*
* This bit me in the ass a couple of times, so lets toss this
* in as a cursory sanity check.
*/
done:
/*
* If there were no return probes when we first found the tracepoint,
* we should feel no obligation to honor any return probes that were
* subsequently enabled -- they'll just have to wait until the next
* time around.
*/
/*
* We need to wait until the results of the instruction are
* apparent before invoking any return probes. If this
* instruction was emulated we can just call
* fasttrap_return_common(); if it needs to be executed, we
* need to wait until we return to the kernel.
*/
} else {
}
}
return (0);
}
int
{
curthread->t_dtrace_pc = 0;
curthread->t_dtrace_npc = 0;
curthread->t_dtrace_scrpc = 0;
curthread->t_dtrace_astpc = 0;
/*
* Treat a child created by a call to vfork(2) as if it were its
* parent. We know there's only one thread of control in such a
* process: this one.
*/
p = p->p_parent;
}
/*
* We set the %pc and %npc to their values when the traced
* instruction was initially executed so that it appears to
* dtrace_probe() that we're on the original instruction, and so that
* the user can't easily detect our complex web of lies.
* dtrace_return_probe() (our caller) will correctly set %pc and %npc
* after we return.
*/
return (0);
}
int
{
return (-1);
return (0);
}
int
{
/*
* Distinguish between read or write failures and a changed
* instruction.
*/
return (0);
return (0);
return (-1);
return (0);
}
int
{
/*
* Read the instruction at the given address out of the process's
* address space. We don't have to worry about a debugger
* changing this instruction before we overwrite it with our trap
* instruction since P_PR_LOCK is set.
*/
return (-1);
/*
* Decode the instruction to fill in the probe flags. We can have
* trick, but pc-relative control transfer present a problem since
* we're relocating the instruction. We emulate these instructions
* in the kernel. We assume a default type and over-write that as
* needed.
*
* pc-relative instructions must be emulated for correctness;
* other instructions (which represent a large set of commonly traced
* instructions) are emulated or otherwise optimized for performance.
*/
/*
* Call instructions.
*/
/*
* Branch instructions.
*
* Unconditional branches need careful attention when they're
* annulled: annulled unconditional branches never execute
* the instruction in the delay slot.
*/
case OP2_ILLTRAP:
case 0x7:
/*
* The compiler may place an illtrap after a call to
* a function that returns a structure. In the case of
* a returned structure, the compiler places an illtrap
* whose const22 field is the size of the returned
* structure immediately following the delay slot of
* the call. To stay out of the way, we refuse to
* place tracepoints on top of illtrap instructions.
*
* This is one of the dumbest architectural decisions
* I've ever had to work around.
*
* We also identify the only illegal op2 value (See
* SPARC Architecture Manual Version 9, E.2 table 31).
*/
return (-1);
case OP2_BPcc:
} else {
/*
* Check for an illegal instruction.
*/
return (-1);
}
if (A(instr) != 0)
disp <<= 13;
disp >>= 11;
break;
case OP2_Bicc:
} else {
}
if (A(instr) != 0)
disp <<= 10;
disp >>= 8;
break;
case OP2_BPr:
/*
* Check for an illegal instruction.
*/
return (-1);
/*
* It's a violation of the v8plus ABI to use a
* register-predicated branch in a 32-bit app if
* the register used is an %l or an %i (%gs and %os
* are legit because they're not saved to the stack
* in 32-bit words when we take a trap).
*/
return (-1);
if (A(instr) != 0)
disp <<= 16;
disp >>= 14;
break;
case OP2_SETHI:
break;
case OP2_FBPfcc:
} else {
}
if (A(instr) != 0)
disp <<= 13;
disp >>= 11;
break;
case OP2_FBfcc:
} else {
}
if (A(instr) != 0)
disp <<= 10;
disp >>= 8;
break;
}
case OP3_RETURN:
break;
case OP3_JMPL:
break;
case OP3_RD:
break;
case OP3_SAVE:
/*
* We optimize for save instructions at function
* entry; see the comment in fasttrap_pid_probe()
* (near FASTTRAP_T_SAVE) for details.
*/
if (fasttrap_optimize_save != 0 &&
type == DTFTP_ENTRY &&
break;
case OP3_RESTORE:
/*
* We optimize restore instructions at function
* return; see the comment in fasttrap_pid_probe()
* (near FASTTRAP_T_RESTORE) for details.
*
* rd must be an %o or %g register.
*/
break;
case OP3_OR:
/*
* A large proportion of instructions in the delay
* slot of retl instructions are or's so we emulate
* these downstairs as an optimization.
*/
break;
case OP3_TCC:
/*
* Breakpoint instructions are effectively position-
* dependent since the debugger uses the %pc value
* to lookup which breakpoint was executed. As a
* result, we can't actually instrument breakpoints.
*/
return (-1);
break;
case 0x19:
case 0x1d:
case 0x29:
case 0x33:
case 0x3f:
/*
* Identify illegal instructions (See SPARC
* Architecture Manual Version 9, E.2 table 32).
*/
return (-1);
}
/*
* Identify illegal instructions (See SPARC Architecture
* Manual Version 9, E.2 table 33).
*/
return (-1);
} else {
return (-1);
}
}
/*
* We don't know how this tracepoint is going to be used, but in case
* it's used as part of a function return probe, we need to indicate
* whether it's always a return site or only potentially a return
* site. If it's part of a return probe, it's always going to be a
* return from that function if it's a restore instruction or if
* the previous instruction was a return. If we could reliably
* distinguish jump tables from return sites, this wouldn't be
* necessary.
*/
return (0);
}
/*ARGSUSED*/
int aframes)
{
}
/*ARGSUSED*/
int aframes)
{
}
static uint64_t fasttrap_getreg_fast_cnt;
static uint64_t fasttrap_getreg_mpcb_cnt;
static uint64_t fasttrap_getreg_slow_cnt;
static ulong_t
{
/*
* We have the %os and %gs in our struct regs, but if we need to
* snag a %l or %i we need to go scrounging around in the process's
* address space.
*/
if (reg == 0)
return (0);
if (reg < 16)
/*
* Before we look at the user's stack, we'll check the register
* windows to see if the information we want is in there.
*/
if (dtrace_getotherwin() > 0) {
return (value);
}
/*
* First check the machpcb structure to see if we've already read
* in the register window we're looking for; if we haven't, (and
* we probably haven't) try to copy in the value of the register.
*/
if (get_udatamodel() == DATAMODEL_NATIVE) {
if (mpcb->mpcb_wbcnt > 0) {
int i = mpcb->mpcb_wbcnt;
do {
i--;
continue;
} while (i > 0);
}
goto err;
} else {
if (mpcb->mpcb_wbcnt > 0) {
int i = mpcb->mpcb_wbcnt;
do {
i--;
continue;
} while (i > 0);
}
goto err;
v32[0] = 0;
}
return (value);
err:
/*
* If the copy in failed, the process will be in a irrecoverable
* state, and we have no choice but to kill it.
*/
return (0);
}
static uint64_t fasttrap_putreg_fast_cnt;
static uint64_t fasttrap_putreg_mpcb_cnt;
static uint64_t fasttrap_putreg_slow_cnt;
static void
{
if (reg == 0)
return;
if (reg < 16) {
return;
}
/*
* If the user process is still using some register windows, we
* can just place the value in the correct window.
*/
if (dtrace_getotherwin() > 0) {
return;
}
/*
* First see if there's a copy of the register window in the
* machpcb structure that we can modify; if there isn't try to
* copy out the value. If that fails, we try to create a new
* register window in the machpcb structure. While this isn't
* _precisely_ the intended use of the machpcb structure, it
* can't cause any problems since we know at this point in the
* code that all of the user's data have been flushed out of the
* register file (since %otherwin is 0).
*/
if (get_udatamodel() == DATAMODEL_NATIVE) {
if (mpcb->mpcb_wbcnt > 0) {
int i = mpcb->mpcb_wbcnt;
do {
i--;
continue;
return;
} while (i > 0);
}
goto err;
mpcb->mpcb_wbcnt++;
return;
}
} else {
if (mpcb->mpcb_wbcnt > 0) {
int i = mpcb->mpcb_wbcnt;
do {
i--;
continue;
return;
} while (i > 0);
}
goto err;
mpcb->mpcb_wbcnt++;
return;
}
}
return;
err:
/*
* If we couldn't record this register's value, the process is in an
* irrecoverable state and we have no choice but to euthanize it.
*/
}