sparc/dtrace/fbt.c

	fbt.c revision b9e93c10c0a2a4bb069d38bb311021a9478c4711
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


#include <sys/errno.h>
#include <sys/stat.h>
#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/systm.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/cpuvar.h>
#include <sys/kmem.h>
#include <sys/strsubr.h>
#include <sys/dtrace.h>
#include <sys/kobj.h>
#include <sys/modctl.h>
#include <sys/atomic.h>
#include <vm/seg_kmem.h>
#include <sys/stack.h>
#include <sys/ctf_api.h>
#include <sys/sysmacros.h>

static dev_info_t       *fbt_devi;
static dtrace_provider_id_t fbt_id;
static uintptr_t        fbt_trampoline;
static caddr_t          fbt_trampoline_window;
static size_t           fbt_trampoline_size;
static int          fbt_verbose = 0;

/*
 * Various interesting bean counters.
 */
static int          fbt_entry;
static int          fbt_ret;
static int          fbt_retl;
static int          fbt_retl_jmptab;
static int          fbt_retl_twoinstr;
static int          fbt_retl_tailcall;
static int          fbt_retl_tailjmpl;
static int          fbt_leaf_functions;

extern char         stubs_base[];
extern char         stubs_end[];

#define FBT_REG_G0      0
#define FBT_REG_G1      1
#define FBT_REG_O0      8
#define FBT_REG_O1      9
#define FBT_REG_O2      10
#define FBT_REG_O3      11
#define FBT_REG_O4      12
#define FBT_REG_O5      13
#define FBT_REG_O6      14
#define FBT_REG_O7      15
#define FBT_REG_I0      24
#define FBT_REG_I1      25
#define FBT_REG_I2      26
#define FBT_REG_I3      27
#define FBT_REG_I4      28
#define FBT_REG_I7      31
#define FBT_REG_L0      16
#define FBT_REG_L1      17
#define FBT_REG_L2      18
#define FBT_REG_L3      19
#define FBT_REG_PC      5

#define FBT_REG_ISGLOBAL(r) ((r) < 8)
#define FBT_REG_ISOUTPUT(r) ((r) >= 8 && (r) < 16)
#define FBT_REG_ISLOCAL(r)  ((r) >= 16 && (r) < 24)
#define FBT_REG_ISVOLATILE(r)   \
    ((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0)
#define FBT_REG_NLOCALS     8

#define FBT_REG_MARKLOCAL(locals, r)    \
    if (FBT_REG_ISLOCAL(r)) \
        (locals)[(r) - FBT_REG_L0] = 1;

#define FBT_REG_INITLOCALS(local, locals)   \
    for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++)  \
        (locals)[(local)] = 0; \
    (local) = FBT_REG_L0

#define FBT_REG_ALLOCLOCAL(local, locals)   \
    while ((locals)[(local) - FBT_REG_L0]) \
        (local)++; \
    (locals)[(local) - FBT_REG_L0] = 1;

#define FBT_OP_MASK     0xc0000000
#define FBT_OP_SHIFT        30
#define FBT_OP(val)     ((val) & FBT_FMT1_MASK)

#define FBT_SIMM13_MASK     0x1fff
#define FBT_SIMM13_MAX      ((int32_t)0xfff)
#define FBT_IMM22_MASK      0x3fffff
#define FBT_IMM22_SHIFT     10
#define FBT_IMM10_MASK      0x3ff

#define FBT_DISP30_MASK     0x3fffffff
#define FBT_DISP30(from, to)    \
    (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK)

#define FBT_DISP22_MASK     0x3fffff
#define FBT_DISP22(from, to)    \
    (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK)

#define FBT_DISP19_MASK     0x7ffff
#define FBT_DISP19(from, to)    \
    (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK)

#define FBT_DISP16_HISHIFT  20
#define FBT_DISP16_HIMASK   (0x3 << FBT_DISP16_HISHIFT)
#define FBT_DISP16_LOMASK   (0x3fff)
#define FBT_DISP16_MASK     (FBT_DISP16_HIMASK | FBT_DISP16_LOMASK)
#define FBT_DISP16(val) \
    ((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK))

#define FBT_DISP14_MASK     0x3fff
#define FBT_DISP14(from, to)    \
    (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK)

#define FBT_OP0         (((uint32_t)0) << FBT_OP_SHIFT)
#define FBT_OP1         (((uint32_t)1) << FBT_OP_SHIFT)
#define FBT_OP2         (((uint32_t)2) << FBT_OP_SHIFT)
#define FBT_ILLTRAP     0

#define FBT_ANNUL_SHIFT     29
#define FBT_ANNUL       (1 << FBT_ANNUL_SHIFT)

#define FBT_FMT3_OP3_SHIFT  19
#define FBT_FMT3_OP_MASK    0xc1f80000
#define FBT_FMT3_OP(val)    ((val) & FBT_FMT3_OP_MASK)

#define FBT_FMT3_RD_SHIFT   25
#define FBT_FMT3_RD_MASK    (0x1f << FBT_FMT3_RD_SHIFT)
#define FBT_FMT3_RD(val)    \
    (((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT)

#define FBT_FMT3_RS1_SHIFT  14
#define FBT_FMT3_RS1_MASK   (0x1f << FBT_FMT3_RS1_SHIFT)
#define FBT_FMT3_RS1(val)   \
    (((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT)
#define FBT_FMT3_RS1_SET(val, rs1) \
    (val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT)

#define FBT_FMT3_RS2_SHIFT  0
#define FBT_FMT3_RS2_MASK   (0x1f << FBT_FMT3_RS2_SHIFT)
#define FBT_FMT3_RS2(val)   \
    (((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT)
#define FBT_FMT3_RS2_SET(val, rs2) \
    (val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT)

#define FBT_FMT3_IMM_SHIFT  13
#define FBT_FMT3_IMM        (1 << FBT_FMT3_IMM_SHIFT)
#define FBT_FMT3_SIMM13_MASK    FBT_SIMM13_MASK

#define FBT_FMT3_ISIMM(val) ((val) & FBT_FMT3_IMM)
#define FBT_FMT3_SIMM13(val)    ((val) & FBT_FMT3_SIMM13_MASK)

#define FBT_FMT2_OP2_SHIFT  22
#define FBT_FMT2_OP2_MASK   (0x7 << FBT_FMT2_OP2_SHIFT)
#define FBT_FMT2_RD_SHIFT   25

#define FBT_FMT1_OP(val)    ((val) & FBT_OP_MASK)
#define FBT_FMT1_DISP30(val)    ((val) & FBT_DISP30_MASK)

#define FBT_FMT2_OP2_BPCC   (0x01 << FBT_FMT2_OP2_SHIFT)
#define FBT_FMT2_OP2_BCC    (0x02 << FBT_FMT2_OP2_SHIFT)
#define FBT_FMT2_OP2_BPR    (0x03 << FBT_FMT2_OP2_SHIFT)
#define FBT_FMT2_OP2_SETHI  (0x04 << FBT_FMT2_OP2_SHIFT)

#define FBT_FMT2_COND_SHIFT 25
#define FBT_FMT2_COND_BA    (0x8 << FBT_FMT2_COND_SHIFT)
#define FBT_FMT2_COND_BL    (0x3 << FBT_FMT2_COND_SHIFT)
#define FBT_FMT2_COND_BGE   (0xb << FBT_FMT2_COND_SHIFT)

#define FBT_OP_RESTORE      (FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_SAVE     (FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_JMPL     (FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_RETURN       (FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_CALL     FBT_OP1
#define FBT_OP_SETHI        (FBT_OP0 | FBT_FMT2_OP2_SETHI)
#define FBT_OP_ADD      (FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_OR       (FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_SUB      (FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_CC       (FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_BA       (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA)
#define FBT_OP_BL       (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL)
#define FBT_OP_BGE      (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE)
#define FBT_OP_BAPCC        (FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA)
#define FBT_OP_RD       (FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT))

#define FBT_ORLO(rs, val, rd) \
    (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
    ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK))

#define FBT_ORSIMM13(rs, val, rd) \
    (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
    ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))

#define FBT_ADDSIMM13(rs, val, rd) \
    (FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \
    ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))

#define FBT_ADD(rs1, rs2, rd) \
    (FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
    ((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))

#define FBT_CMP(rs1, rs2) \
    (FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
    ((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT))

#define FBT_MOV(rs, rd) \
    (FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \
    ((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))

#define FBT_SETHI(val, reg) \
    (FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \
    ((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK))

#define FBT_CALL(orig, dest)    (FBT_OP_CALL | FBT_DISP30(orig, dest))

#define FBT_RET \
    (FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \
    (FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1))

#define FBT_SAVEIMM(rd, val, rs1)   \
    (FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
    ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))

#define FBT_RESTORE(rd, rs1, rs2)   \
    (FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
    ((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT))

#define FBT_RETURN(rs1, val)        \
    (FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
    FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))

#define FBT_BA(orig, dest)  (FBT_OP_BA | FBT_DISP22(orig, dest))
#define FBT_BAA(orig, dest) (FBT_BA(orig, dest) | FBT_ANNUL)
#define FBT_BL(orig, dest)  (FBT_OP_BL | FBT_DISP22(orig, dest))
#define FBT_BGE(orig, dest) (FBT_OP_BGE | FBT_DISP22(orig, dest))
#define FBT_BDEST(va, instr)    ((uintptr_t)(va) + \
    (((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8))
#define FBT_BPCCDEST(va, instr) ((uintptr_t)(va) + \
    (((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11))
#define FBT_BPRDEST(va, instr)  ((uintptr_t)(va) + \
    (((int32_t)((FBT_DISP16(instr)) << 16)) >> 14))

/*
 * We're only going to treat a save as safe if (a) both rs1 and rd are
 * %sp and (b) if the instruction has a simm, the value isn't 0.
 */
#define FBT_IS_SAVE(instr)  \
    (FBT_FMT3_OP(instr) == FBT_OP_SAVE && \
    FBT_FMT3_RD(instr) == FBT_REG_O6 && \
    FBT_FMT3_RS1(instr) == FBT_REG_O6 && \
    !(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0))

#define FBT_IS_BA(instr)    (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA)
#define FBT_IS_BAPCC(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC)

#define FBT_IS_RDPC(instr)  ((FBT_FMT3_OP(instr) == FBT_OP_RD) && \
    (FBT_FMT3_RD(instr) == FBT_REG_PC))

#define FBT_IS_PCRELATIVE(instr)    \
    ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
    ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
    ((instr) & FBT_OP_MASK) == FBT_OP1 || \
    FBT_IS_RDPC(instr))

#define FBT_IS_CTI(instr)   \
    ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
    ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
    ((instr) & FBT_OP_MASK) == FBT_OP1 || \
    (FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \
    (FBT_FMT3_OP(instr) == FBT_OP_RETURN))

#define FBT_PROBENAME_ENTRY "entry"
#define FBT_PROBENAME_RETURN    "return"
#define FBT_ESTIMATE_ID     (UINT32_MAX)
#define FBT_COUNTER(id, count)  if ((id) != FBT_ESTIMATE_ID) (count)++

#define FBT_ENTENT_MAXSIZE  (16 * sizeof (uint32_t))
#define FBT_RETENT_MAXSIZE  (11 * sizeof (uint32_t))
#define FBT_RETLENT_MAXSIZE (23 * sizeof (uint32_t))
#define FBT_ENT_MAXSIZE     \
    MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE)

typedef struct fbt_probe {
    char        *fbtp_name;
    dtrace_id_t fbtp_id;
    uintptr_t   fbtp_addr;
    struct modctl   *fbtp_ctl;
    int     fbtp_loadcnt;
    int     fbtp_symndx;
    int     fbtp_primary;
    int     fbtp_return;
    uint32_t    *fbtp_patchpoint;
    uint32_t    fbtp_patchval;
    uint32_t    fbtp_savedval;
    struct fbt_probe *fbtp_next;
} fbt_probe_t;

typedef struct fbt_trampoline {
    uintptr_t   fbtt_va;
    uintptr_t   fbtt_limit;
    uintptr_t   fbtt_next;
} fbt_trampoline_t;

static caddr_t
fbt_trampoline_map(uintptr_t tramp, size_t size)
{
    uintptr_t offs;
    page_t **ppl;

    ASSERT(fbt_trampoline_window == NULL);
    ASSERT(fbt_trampoline_size == 0);
    ASSERT(fbt_trampoline == NULL);

    size += tramp & PAGEOFFSET;
    fbt_trampoline = tramp & PAGEMASK;
    fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK;
    fbt_trampoline_window =
        vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP);

    (void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline,
        fbt_trampoline_size, S_WRITE);

    for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) {
        hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE,
            hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs),
            PROT_READ | PROT_WRITE,
            HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
    }

    as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size,
        S_WRITE);

    return (fbt_trampoline_window + (tramp & PAGEOFFSET));
}

static void
fbt_trampoline_unmap()
{
    ASSERT(fbt_trampoline_window != NULL);
    ASSERT(fbt_trampoline_size != 0);
    ASSERT(fbt_trampoline != NULL);

    membar_enter();
    sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size);
    sync_icache(fbt_trampoline_window, fbt_trampoline_size);

    hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size,
        HAT_UNLOAD_UNLOCK);

    vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size);

    fbt_trampoline_window = NULL;
    fbt_trampoline = NULL;
    fbt_trampoline_size = 0;
}

static uintptr_t
fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp,
    int nargs)
{
    uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
    uint32_t first = *instr;
    uintptr_t va = tramp->fbtt_va;
    uintptr_t base = tramp->fbtt_next;

    if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) {
        /*
         * There isn't sufficient room for this entry; return failure.
         */
        return (0);
    }

    FBT_COUNTER(id, fbt_entry);

    if (FBT_IS_SAVE(first)) {
        *tinstr++ = first;
    } else {
        *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
    }

    if (id > (uint32_t)FBT_SIMM13_MAX) {
        *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
        *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
    } else {
        *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
    }

    if (nargs >= 1)
        *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1);

    if (nargs >= 2)
        *tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2);

    if (nargs >= 3)
        *tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3);

    if (nargs >= 4)
        *tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4);

    if (nargs >= 5)
        *tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5);

    if (FBT_IS_SAVE(first)) {
        uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t);

        *tinstr++ = FBT_SETHI(ret, FBT_REG_G1);
        *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
        tinstr++;
        *tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7);
    } else {
        uintptr_t slot = *--tinstr;
        uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t);
        uint32_t delay = first;

        *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
        tinstr++;
        *tinstr++ = slot;
        *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);

        if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) {
            /*
             * This is a special case:  we are instrumenting a
             * a non-annulled branch-always (or variant).  We'll
             * return directly to the destination of the branch,
             * copying the instruction in the delay slot here,
             * and then executing it in the slot of a ba.
             */
            if (FBT_IS_BA(first)) {
                ret = FBT_BDEST(instr, *instr);
            } else {
                ret = FBT_BPCCDEST(instr, *instr);
            }

            delay = *(instr + 1);
        }

        if ((first & FBT_OP_MASK) != FBT_OP0 ||
            (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
            *tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret);
            tinstr++;
            *tinstr++ = delay;
        } else {
            /*
             * If this is a branch-on-register, we have a little
             * more work to do:  because the displacement is only
             * sixteen bits, we're going to thunk the branch into
             * the trampoline, and then ba,a to the appropriate
             * destination in the branch targets.  That is, we're
             * constructing this sequence in the trampoline:
             *
             *      br[cc]  %[rs], 1f
             *      <delay-instruction>
             *      ba,a    <not-taken-destination>
             *  1:  ba,a    <taken-destination>
             *
             */
            uintptr_t targ = FBT_BPRDEST(instr, first);

            *tinstr = first & ~(FBT_DISP16_MASK);
            *tinstr |= FBT_DISP14(tinstr, &tinstr[3]);
            tinstr++;
            *tinstr++ = *(instr + 1);
            *tinstr = FBT_BAA((uintptr_t)tinstr - base + va,
                ret + sizeof (uint32_t));
            tinstr++;
            *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ);
            tinstr++;
        }
    }

    tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
    tramp->fbtt_next = (uintptr_t)tinstr;

    return (1);
}

/*
 * We are patching control-transfer/restore couplets.  There are three
 * variants of couplet:
 *
 * (a)  return      rs1 + imm
 *  delay
 *
 * (b)  jmpl        rs1 + (rs2 | offset), rd
 *  restore     rs1, rs2 | imm, rd
 *
 * (c)  call        displacement
 *  restore     rs1, rs2 | imm, rd
 *
 * If rs1 in (a) is anything other than %i7, or imm is anything other than 8,
 * or delay is a DCTI, we fail.  If rd from the jmpl in (b) is something other
 * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call
 * through a register), we fail.
 *
 * Note that rs1 and rs2 in the restore instructions in (b) and (c) are
 * potentially outputs and/or globals.  Because these registers cannot be
 * relied upon across the call to dtrace_probe(), we move rs1 into an unused
 * local, ls0, and rs2 into an unused local, ls1, and restructure the restore
 * to be:
 *
 *  restore     ls0, ls1, rd
 *
 * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals.
 * If the jmpl uses outputs or globals, we restructure it to be:
 *
 *  jmpl        ls2 + (ls3 | offset), (%g0 | %o7)
 *
 */
/*ARGSUSED*/
static int
fbt_canpatch_return(uint32_t *instr, int offset, const char *name)
{
    int rd;

    if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
        uint32_t delay = *(instr + 1);

        if (*instr != FBT_RETURN(FBT_REG_I7, 8)) {
            /*
             * It's unclear if we should warn about this or not.
             * We really wouldn't expect the compiler to generate
             * return instructions with something other than %i7
             * as rs1 and 8 as the simm13 -- it would just be
             * mean-spirited.  That said, such a construct isn't
             * necessarily incorrect.  Sill, we err on the side of
             * caution and warn about it...
             */
            cmn_err(CE_NOTE, "cannot instrument return of %s at "
                "%p: non-canonical return instruction", name,
                (void *)instr);
            return (0);
        }

        if (FBT_IS_CTI(delay)) {
            /*
             * This is even weirder -- a DCTI coupled with a
             * return instruction.  Similar constructs are used to
             * return from utraps, but these typically have the
             * return in the slot -- and we wouldn't expect to see
             * it in the kernel regardless.  At any rate, we don't
             * want to try to instrument this construct, whatever
             * it may be.
             */
            cmn_err(CE_NOTE, "cannot instrument return of %s at "
                "%p: CTI in delay slot of return instruction",
                name, (void *)instr);
            return (0);
        }

        if (FBT_IS_PCRELATIVE(delay)) {
            /*
             * This is also very weird, but might be correct code
             * if the function is (for example) returning the
             * address of the delay instruction of the return as
             * its return value (e.g. "rd %pc, %o0" in the slot).
             * Perhaps correct, but still too weird to not warn
             * about it...
             */
            cmn_err(CE_NOTE, "cannot instrument return of %s at "
                "%p: PC-relative instruction in delay slot of "
                "return instruction", name, (void *)instr);
            return (0);
        }

        return (1);
    }

    if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE)
        return (0);

    if (FBT_FMT1_OP(*instr) == FBT_OP_CALL)
        return (1);

    if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
        return (0);

    rd = FBT_FMT3_RD(*instr);

    if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0)
        return (1);

    /*
     * We have encountered a jmpl that is storing the calling %pc in
     * some register besides %i7, %o7 or %g0.  This is strange; emit
     * a warning and fail.
     */
    cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected "
        "jmpl destination register", name, (void *)instr);
    return (0);
}

static int
fbt_canpatch_retl(uint32_t *instr, int offset, const char *name)
{
    if (FBT_FMT1_OP(*instr) == FBT_OP_CALL ||
        (FBT_FMT3_OP(*instr) == FBT_OP_JMPL &&
        FBT_FMT3_RD(*instr) == FBT_REG_O7)) {
        /*
         * If this is a call (or a jmpl that links into %o7), we can
         * patch it iff the next instruction uses %o7 as a destination
         * register.  Because there is an ABI responsibility to
         * restore %o7 to the value before the call/jmpl, we don't
         * particularly care how this routine is managing to restore
         * it (mov, add, ld or divx for all we care).  If it doesn't
         * seem to be restoring it at all, however, we'll refuse
         * to patch it.
         */
        uint32_t delay = *(instr + 1);
        uint32_t op, rd;

        op = FBT_FMT1_OP(delay);
        rd = FBT_FMT3_RD(delay);

        if (op != FBT_OP2 || rd != FBT_REG_O7) {
            /*
             * This is odd.  Before we assume that we're looking
             * at something bizarre (and warn accordingly), we'll
             * check to see if it's obviously a jump table entry.
             */
            if (*instr < (uintptr_t)instr &&
                *instr >= (uintptr_t)instr - offset)
                return (0);

            cmn_err(CE_NOTE, "cannot instrument return of %s at "
                "%p: leaf jmpl/call delay isn't restoring %%o7",
                name, (void *)instr);
            return (0);
        }

        return (1);
    }

    if (offset == sizeof (uint32_t)) {
        /*
         * If this is the second instruction in the function, we're
         * going to allow it to be patched if the first instruction
         * is a patchable return-from-leaf instruction.
         */
        if (fbt_canpatch_retl(instr - 1, 0, name))
            return (1);
    }

    if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
        return (0);

    if (FBT_FMT3_RD(*instr) != FBT_REG_G0)
        return (0);

    return (1);
}

/*ARGSUSED*/
static uint32_t
fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
    int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
{
    uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
    uint32_t cti = *instr, restore = *(instr + 1), rs1, dest;
    uintptr_t va = tramp->fbtt_va;
    uintptr_t base = tramp->fbtt_next;
    uint32_t locals[FBT_REG_NLOCALS], local;

    if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) {
        /*
         * There isn't sufficient room for this entry; return failure.
         */
        return (FBT_ILLTRAP);
    }

    FBT_COUNTER(id, fbt_ret);

    if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
        /*
         * To handle the case of the return instruction, we'll emit a
         * restore, followed by the instruction in the slot (which
         * we'll transplant here), and then another save.  While it
         * may seem intellectually unsatisfying to emit the additional
         * restore/save couplet, one can take solace in the fact that
         * we don't do this if the instruction in the return delay
         * slot is a nop -- which it is nearly 90% of the time with
         * gcc.  (And besides, this couplet can't induce unnecessary
         * spill/fill traps; rewriting the delay instruction to be
         * in terms of the current window hardly seems worth the
         * trouble -- let alone the risk.)
         */
        uint32_t delay = *(instr + 1);
        ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8));

        cti = FBT_RET;
        restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);

        if (delay != FBT_SETHI(0, FBT_REG_G0)) {
            *tinstr++ = restore;
            *tinstr++ = delay;
            *tinstr++ = FBT_SAVEIMM(FBT_REG_O6,
                -SA(MINFRAME), FBT_REG_O6);
        }
    }

    FBT_REG_INITLOCALS(local, locals);

    /*
     * Mark the locals used in the jmpl.
     */
    if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
        uint32_t rs1 = FBT_FMT3_RS1(cti);
        FBT_REG_MARKLOCAL(locals, rs1);

        if (!FBT_FMT3_ISIMM(cti)) {
            uint32_t rs2 = FBT_FMT3_RS2(cti);
            FBT_REG_MARKLOCAL(locals, rs2);
        }
    }

    /*
     * And mark the locals used in the restore.
     */
    rs1 = FBT_FMT3_RS1(restore);
    FBT_REG_MARKLOCAL(locals, rs1);

    if (!FBT_FMT3_ISIMM(restore)) {
        uint32_t rs2 = FBT_FMT3_RS2(restore);
        FBT_REG_MARKLOCAL(locals, rs2);
    }

    if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
        uint32_t rs1 = FBT_FMT3_RS1(cti);

        if (FBT_REG_ISVOLATILE(rs1)) {
            FBT_REG_ALLOCLOCAL(local, locals);
            FBT_FMT3_RS1_SET(cti, local);
            *tinstr++ = FBT_MOV(rs1, local);
        }

        if (!FBT_FMT3_ISIMM(cti)) {
            uint32_t rs2 = FBT_FMT3_RS2(cti);

            if (FBT_REG_ISVOLATILE(rs2)) {
                FBT_REG_ALLOCLOCAL(local, locals);
                FBT_FMT3_RS2_SET(cti, local);
                *tinstr++ = FBT_MOV(rs2, local);
            }
        }
    }

    rs1 = FBT_FMT3_RS1(restore);

    if (FBT_REG_ISVOLATILE(rs1)) {
        FBT_REG_ALLOCLOCAL(local, locals);
        FBT_FMT3_RS1_SET(restore, local);
        *tinstr++ = FBT_MOV(rs1, local);
    }

    if (!FBT_FMT3_ISIMM(restore)) {
        uint32_t rs2 = FBT_FMT3_RS2(restore);

        if (FBT_REG_ISVOLATILE(rs2)) {
            FBT_REG_ALLOCLOCAL(local, locals);
            FBT_FMT3_RS2_SET(restore, local);
            *tinstr++ = FBT_MOV(rs2, local);
        }
    }

    if (id > (uint32_t)FBT_SIMM13_MAX) {
        *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
        *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
    } else {
        *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
    }

    if (offset > (uint32_t)FBT_SIMM13_MAX) {
        *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
        *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
    } else {
        *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
    }

    *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
    tinstr++;

    if (FBT_FMT3_RD(restore) == FBT_REG_O0) {
        /*
         * If the destination register of the restore is %o0, we
         * need to perform the implied calculation to derive the
         * return value.
         */
        uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD;
        add &= ~FBT_FMT3_RD_MASK;
        *tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT);
    } else {
        *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
    }

    /*
     * If the control transfer instruction is %pc-relative (i.e. a
     * call), we need to reset it appropriately.
     */
    if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
        dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
        *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
        tinstr++;
    } else {
        *tinstr++ = cti;
    }

    *tinstr++ = restore;
    tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
    tramp->fbtt_next = (uintptr_t)tinstr;

    return (FBT_BAA(instr, va));
}

static uint32_t
fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
    int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
{
    uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
    uintptr_t va = tramp->fbtt_va;
    uintptr_t base = tramp->fbtt_next;
    uint32_t cti = *instr, dest;
    int annul = 0;

    FBT_COUNTER(id, fbt_retl);

    if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) {
        /*
         * There isn't sufficient room for this entry; return failure.
         */
        return (FBT_ILLTRAP);
    }

    if (offset == sizeof (uint32_t) &&
        fbt_canpatch_retl(instr - 1, 0, name)) {
        *tinstr++ = *instr;
        annul = 1;
        FBT_COUNTER(id, fbt_retl_twoinstr);
    } else {
        if (FBT_FMT3_OP(cti) == FBT_OP_JMPL &&
            FBT_FMT3_RD(cti) != FBT_REG_O7 &&
            FBT_FMT3_RS1(cti) != FBT_REG_O7) {
            annul = 1;
            *tinstr++ = *(instr + 1);
        }
    }

    *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);

    if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
        uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0;

        /*
         * If we have a jmpl and it's in terms of output registers, we
         * need to rewrite it to be in terms of the corresponding input
         * registers.  If it's in terms of the globals, we'll rewrite
         * it to be in terms of locals.
         */
        rs1 = FBT_FMT3_RS1(cti);

        if (FBT_REG_ISOUTPUT(rs1))
            rs1 += o2i;

        if (FBT_REG_ISGLOBAL(rs1)) {
            *tinstr++ = FBT_MOV(rs1, FBT_REG_L0);
            rs1 = FBT_REG_L0;
        }

        FBT_FMT3_RS1_SET(cti, rs1);

        if (!FBT_FMT3_ISIMM(cti)) {
            rs2 = FBT_FMT3_RS2(cti);

            if (FBT_REG_ISOUTPUT(rs2))
                rs2 += o2i;

            if (FBT_REG_ISGLOBAL(rs2)) {
                *tinstr++ = FBT_MOV(rs2, FBT_REG_L1);
                rs2 = FBT_REG_L1;
            }

            FBT_FMT3_RS2_SET(cti, rs2);
        }

        /*
         * Now we need to check the rd and source register for the jmpl;
         * If neither rd nor the source register is %o7, then we might
         * have a jmp that is actually part of a jump table.  We need
         * to generate the code to compare it to the base and limit of
         * the function.
         */
        if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) {
            uintptr_t base = (uintptr_t)funcbase;
            uintptr_t limit = (uintptr_t)funclim;

            FBT_COUNTER(id, fbt_retl_jmptab);

            if (FBT_FMT3_ISIMM(cti)) {
                *tinstr++ = FBT_ADDSIMM13(rs1,
                    FBT_FMT3_SIMM13(cti), FBT_REG_L2);
            } else {
                *tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2);
            }

            *tinstr++ = FBT_SETHI(base, FBT_REG_L3);
            *tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3);
            *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
            *tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t));
            *tinstr++ = FBT_SETHI(limit, FBT_REG_L3);
            *tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3);
            *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
            *tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t));
            *tinstr++ = FBT_SETHI(0, FBT_REG_G0);
            *tinstr++ = cti;
            *tinstr++ = FBT_RESTORE(FBT_REG_G0,
                FBT_REG_G0, FBT_REG_G0);
        }
    }

    if (id > (uint32_t)FBT_SIMM13_MAX) {
        *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
        *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
    } else {
        *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
    }

    if (offset > (uint32_t)FBT_SIMM13_MAX) {
        *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
        *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
    } else {
        *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
    }

    *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
    tinstr++;
    *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);

    /*
     * If the control transfer instruction is %pc-relative (i.e. a
     * call), we need to reset it appropriately.
     */
    if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
        FBT_COUNTER(id, fbt_retl_tailcall);
        dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
        *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
        tinstr++;
        annul = 1;
    } else {
        if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
            *tinstr++ = cti;

            if (FBT_FMT3_RD(cti) == FBT_REG_O7) {
                FBT_COUNTER(id, fbt_retl_tailjmpl);
                annul = 1;
            }
        } else {
            *tinstr++ = FBT_RET;
        }
    }

    *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);

    tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
    tramp->fbtt_next = (uintptr_t)tinstr;

    return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va));
}

/*ARGSUSED*/
static void
fbt_provide_module(void *arg, struct modctl *ctl)
{
    struct module *mp = ctl->mod_mp;
    char *modname = ctl->mod_modname;
    char *str = mp->strings;
    int nsyms = mp->nsyms;
    Shdr *symhdr = mp->symhdr;
    size_t symsize;
    char *name;
    int i;
    fbt_probe_t *fbt, *retfbt;
    fbt_trampoline_t tramp;
    uintptr_t offset;
    int primary = 0;
    ctf_file_t *fp = NULL;
    int error;
    int estimate = 1;
    uint32_t faketramp[50];
    size_t fbt_size = 0;

    /*
     * Employees of dtrace and their families are ineligible.  Void
     * where prohibited.
     */
    if (strcmp(modname, "dtrace") == 0)
        return;

    if (ctl->mod_requisites != NULL) {
        struct modctl_list *list;

        list = (struct modctl_list *)ctl->mod_requisites;

        for (; list != NULL; list = list->modl_next) {
            if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0)
                return;
        }
    }

    /*
     * KMDB is ineligible for instrumentation -- it may execute in
     * any context, including probe context.
     */
    if (strcmp(modname, "kmdbmod") == 0)
        return;

    if (str == NULL || symhdr == NULL || symhdr->sh_addr == NULL) {
        /*
         * If this module doesn't (yet) have its string or symbol
         * table allocated, clear out.
         */
        return;
    }

    symsize = symhdr->sh_entsize;

    if (mp->fbt_nentries) {
        /*
         * This module has some FBT entries allocated; we're afraid
         * to screw with it.
         */
        return;
    }

    if (mp->fbt_tab != NULL)
        estimate = 0;

    /*
     * This is a hack for unix/genunix/krtld.
     */
    primary = vmem_contains(heap_arena, (void *)ctl,
        sizeof (struct modctl)) == 0;
    kobj_textwin_alloc(mp);

    /*
     * Open the CTF data for the module.  We'll use this to determine the
     * functions that can be instrumented.  Note that this call can fail,
     * in which case we'll use heuristics to determine the functions that
     * can be instrumented.  (But in particular, leaf functions will not be
     * instrumented.)
     */
    fp = ctf_modopen(mp, &error);

forreal:
    if (!estimate) {
        tramp.fbtt_next =
            (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab,
            mp->fbt_size);
        tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size;
        tramp.fbtt_va = (uintptr_t)mp->fbt_tab;
    }

    for (i = 1; i < nsyms; i++) {
        ctf_funcinfo_t f;
        uint32_t *instr, *base, *limit;
        Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize);
        int have_ctf = 0, is_leaf = 0, nargs, cti = 0;
        int (*canpatch)(uint32_t *, int, const char *);
        uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int,
            uint32_t, fbt_trampoline_t *, const char *);

        if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
            continue;

        /*
         * Weak symbols are not candidates.  This could be made to
         * work (where weak functions and their underlying function
         * appear as two disjoint probes), but it's not simple.
         */
        if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
            continue;

        name = str + sym->st_name;

        if (strstr(name, "dtrace_") == name &&
            strstr(name, "dtrace_safe_") != name) {
            /*
             * Anything beginning with "dtrace_" may be called
             * from probe context unless it explitly indicates
             * that it won't be called from probe context by
             * using the prefix "dtrace_safe_".
             */
            continue;
        }

        if (strstr(name, "kdi_") == name ||
            strstr(name, "_kdi_") != NULL) {
            /*
             * Any function name beginning with "kdi_" or
             * containing the string "_kdi_" is a part of the
             * kernel debugger interface and may be called in
             * arbitrary context -- including probe context.
             */
            continue;
        }

        if (strstr(name, "__relocatable") != NULL) {
            /*
             * Anything with the string "__relocatable" anywhere
             * in the function name is considered to be a function
             * that may be manually relocated before execution.
             * Because FBT uses a PC-relative technique for
             * instrumentation, these functions cannot safely
             * be instrumented by us.
             */
            continue;
        }

        if (strstr(name, "ip_ocsum") == name) {
            /*
             * The ip_ocsum_* family of routines are all ABI
             * violators.  (They expect incoming arguments in the
             * globals!)  Break the ABI?  No soup for you!
             */
            continue;
        }

        /*
         * We want to scan the function for one (and only one) save.
         * Any more indicates that something fancy is going on.
         */
        base = (uint32_t *)sym->st_value;
        limit = (uint32_t *)(sym->st_value + sym->st_size);

        /*
         * We don't want to interpose on the module stubs.
         */
        if (base >= (uint32_t *)stubs_base &&
            base <= (uint32_t *)stubs_end)
            continue;

        /*
         * We can't safely trace a zero-length function...
         */
        if (base == limit)
            continue;

        /*
         * Due to 4524008, _init and _fini may have a bloated st_size.
         * While this bug was fixed quite some time ago, old drivers
         * may be lurking.  We need to develop a better solution to
         * this problem, such that correct _init and _fini functions
         * (the vast majority) may be correctly traced.  One solution
         * may be to scan through the entire symbol table to see if
         * any symbol overlaps with _init.  If none does, set a bit in
         * the module structure that this module has correct _init and
         * _fini sizes.  This will cause some pain the first time a
         * module is scanned, but at least it would be O(N) instead of
         * O(N log N)...
         */
        if (strcmp(name, "_init") == 0)
            continue;

        if (strcmp(name, "_fini") == 0)
            continue;

        instr = base;

        /*
         * While we try hard to only trace safe functions (that is,
         * functions at TL=0), one unsafe function manages to otherwise
         * appear safe:  prom_trap().  We could discover prom_trap()
         * if we added an additional rule:  in order to trace a
         * function, we must either (a) discover a restore or (b)
         * determine that the function does not have any unlinked
         * control transfers to another function (i.e., the function
         * never returns).  Unfortunately, as of this writing, one
         * legitimate function (resume_from_zombie()) transfers
         * control to a different function (_resume_from_idle())
         * without executing a restore.  Barring a rule to figure out
         * that resume_from_zombie() is safe while prom_trap() is not,
         * we resort to hard-coding prom_trap() here.
         */
        if (strcmp(name, "prom_trap") == 0)
            continue;

        if (fp != NULL && ctf_func_info(fp, i, &f) != CTF_ERR) {
            nargs = f.ctc_argc;
            have_ctf = 1;
        } else {
            nargs = 32;
        }

        /*
         * If the first instruction of the function is a branch and
         * it's not a branch-always-not-annulled, we're going to refuse
         * to patch it.
         */
        if ((*instr & FBT_OP_MASK) == FBT_OP0 &&
            (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI &&
            (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
            if (!FBT_IS_BA(*instr) && !FBT_IS_BAPCC(*instr)) {
                if (have_ctf) {
                    cmn_err(CE_NOTE, "cannot instrument %s:"
                        " begins with non-ba, "
                        "non-br CTI", name);
                }
                continue;
            }
        }

        while (!FBT_IS_SAVE(*instr)) {
            /*
             * Before we assume that this is a leaf routine, check
             * forward in the basic block for a save.
             */
            int op = *instr & FBT_OP_MASK;
            int op2 = *instr & FBT_FMT2_OP2_MASK;

            if (op == FBT_OP0 && op2 != FBT_FMT2_OP2_SETHI) {
                /*
                 * This is a CTI.  If we see a subsequent
                 * save, we will refuse to process this
                 * routine unless both of the following are
                 * true:
                 *
                 *  (a) The branch is not annulled
                 *
                 *  (b) The subsequent save is in the delay
                 *  slot of the branch
                 */
                if ((*instr & FBT_ANNUL) ||
                    !FBT_IS_SAVE(*(instr + 1))) {
                    cti = 1;
                } else {
                    instr++;
                    break;
                }
            }

            if (op == FBT_OP1)
                cti = 1;

            if (++instr == limit)
                break;
        }

        if (instr < limit && cti) {
            /*
             * If we found a CTI before the save, we need to not
             * do anything.  But if we have CTF information, this
             * is weird enough that it merits a message.
             */
            if (!have_ctf)
                continue;

            cmn_err(CE_NOTE, "cannot instrument %s: "
                "save not in first basic block", name);
            continue;
        }

        if (instr == limit) {
            if (!have_ctf)
                continue;
            is_leaf = 1;

            if (!estimate)
                fbt_leaf_functions++;

            canpatch = fbt_canpatch_retl;
            patch = fbt_patch_retl;
        } else {
            canpatch = fbt_canpatch_return;
            patch = fbt_patch_return;
        }

        if (!have_ctf && !is_leaf) {
            /*
             * Before we assume that this isn't something tricky,
             * look for other saves.  If we find them, there are
             * multiple entry points here (or something), and we'll
             * leave it alone.
             */
            while (++instr < limit) {
                if (FBT_IS_SAVE(*instr))
                    break;
            }

            if (instr != limit)
                continue;
        }

        instr = base;

        if (FBT_IS_CTI(*instr)) {
            /*
             * If we have a CTI, we want to be sure that we don't
             * have a CTI or a PC-relative instruction in the
             * delay slot -- we want to be able to thunk the
             * instruction into the trampoline without worrying
             * about either DCTIs or relocations.  It would be
             * very odd for the compiler to generate this kind of
             * code, so we warn about it if we have CTF
             * information.
             */
            if (FBT_IS_CTI(*(instr + 1))) {
                if (!have_ctf)
                    continue;

                cmn_err(CE_NOTE, "cannot instrument %s: "
                    "CTI in delay slot of first instruction",
                    name);
                continue;
            }

            if (FBT_IS_PCRELATIVE(*(instr + 1))) {
                if (!have_ctf)
                    continue;

                cmn_err(CE_NOTE, "cannot instrument %s: "
                    "PC-relative instruction in delay slot of"
                    " first instruction", name);
                continue;
            }
        }

        if (estimate) {
            tramp.fbtt_next = (uintptr_t)faketramp;
            tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
            (void) fbt_patch_entry(instr, FBT_ESTIMATE_ID,
                &tramp, nargs);
            fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
        } else {
            fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
            fbt->fbtp_name = name;
            fbt->fbtp_ctl = ctl;
            fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
                name, FBT_PROBENAME_ENTRY, 1, fbt);
            fbt->fbtp_patchval = FBT_BAA(instr, tramp.fbtt_va);

            if (!fbt_patch_entry(instr, fbt->fbtp_id,
                &tramp, nargs)) {
                cmn_err(CE_WARN, "unexpectedly short FBT table "
                    "in module %s (sym %d of %d)", modname,
                    i, nsyms);
                break;
            }

            fbt->fbtp_patchpoint =
                (uint32_t *)((uintptr_t)mp->textwin +
                ((uintptr_t)instr - (uintptr_t)mp->text));
            fbt->fbtp_savedval = *instr;

            fbt->fbtp_loadcnt = ctl->mod_loadcnt;
            fbt->fbtp_primary = primary;
            fbt->fbtp_symndx = i;
            mp->fbt_nentries++;
        }

        retfbt = NULL;
again:
        if (++instr == limit)
            continue;

        offset = (uintptr_t)instr - (uintptr_t)base;

        if (!(*canpatch)(instr, offset, name))
            goto again;

        if (estimate) {
            tramp.fbtt_next = (uintptr_t)faketramp;
            tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
            (void) (*patch)(instr, base, limit,
                offset, FBT_ESTIMATE_ID, &tramp, name);
            fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;

            goto again;
        }

        fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
        fbt->fbtp_name = name;
        fbt->fbtp_ctl = ctl;

        if (retfbt == NULL) {
            fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
                name, FBT_PROBENAME_RETURN, 1, fbt);
        } else {
            retfbt->fbtp_next = fbt;
            fbt->fbtp_id = retfbt->fbtp_id;
        }

        fbt->fbtp_return = 1;
        retfbt = fbt;

        if ((fbt->fbtp_patchval = (*patch)(instr, base, limit, offset,
            fbt->fbtp_id, &tramp, name)) == FBT_ILLTRAP) {
            cmn_err(CE_WARN, "unexpectedly short FBT table "
                "in module %s (sym %d of %d)", modname, i, nsyms);
            break;
        }

        fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin +
            ((uintptr_t)instr - (uintptr_t)mp->text));
        fbt->fbtp_savedval = *instr;
        fbt->fbtp_loadcnt = ctl->mod_loadcnt;
        fbt->fbtp_primary = primary;
        fbt->fbtp_symndx = i;
        mp->fbt_nentries++;

        goto again;
    }

    if (estimate) {
        /*
         * Slosh on another entry's worth...
         */
        fbt_size += FBT_ENT_MAXSIZE;
        mp->fbt_size = fbt_size;
        mp->fbt_tab = kobj_texthole_alloc(mp->text, fbt_size);

        if (mp->fbt_tab == NULL) {
            cmn_err(CE_WARN, "couldn't allocate FBT table "
                "for module %s", modname);
        } else {
            estimate = 0;
            goto forreal;
        }
    } else {
        fbt_trampoline_unmap();
    }

error:
    if (fp != NULL)
        ctf_close(fp);
}

/*ARGSUSED*/
static void
fbt_destroy(void *arg, dtrace_id_t id, void *parg)
{
    fbt_probe_t *fbt = parg, *next;
    struct modctl *ctl = fbt->fbtp_ctl;

    do {
        if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) {
            if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt &&
                ctl->mod_loaded) || fbt->fbtp_primary) {
                ((struct module *)
                    (ctl->mod_mp))->fbt_nentries--;
            }
        }

        next = fbt->fbtp_next;
        kmem_free(fbt, sizeof (fbt_probe_t));
        fbt = next;
    } while (fbt != NULL);
}

/*ARGSUSED*/
static int
fbt_enable(void *arg, dtrace_id_t id, void *parg)
{
    fbt_probe_t *fbt = parg, *f;
    struct modctl *ctl = fbt->fbtp_ctl;

    ctl->mod_nenabled++;

    for (f = fbt; f != NULL; f = f->fbtp_next) {
        if (f->fbtp_patchpoint == NULL) {
            /*
             * Due to a shortened FBT table, this entry was never
             * completed; refuse to enable it.
             */
            if (fbt_verbose) {
                cmn_err(CE_NOTE, "fbt is failing for probe %s "
                    "(short FBT table in %s)",
                    fbt->fbtp_name, ctl->mod_modname);
            }

            return (0);
        }
    }

    /*
     * If this module has disappeared since we discovered its probes,
     * refuse to enable it.
     */
    if (!fbt->fbtp_primary && !ctl->mod_loaded) {
        if (fbt_verbose) {
            cmn_err(CE_NOTE, "fbt is failing for probe %s "
                "(module %s unloaded)",
                fbt->fbtp_name, ctl->mod_modname);
        }

        return (0);
    }

    /*
     * Now check that our modctl has the expected load count.  If it
     * doesn't, this module must have been unloaded and reloaded -- and
     * we're not going to touch it.
     */
    if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) {
        if (fbt_verbose) {
            cmn_err(CE_NOTE, "fbt is failing for probe %s "
                "(module %s reloaded)",
                fbt->fbtp_name, ctl->mod_modname);
        }

        return (0);
    }

    for (; fbt != NULL; fbt = fbt->fbtp_next)
        *fbt->fbtp_patchpoint = fbt->fbtp_patchval;

    return (0);
}

/*ARGSUSED*/
static void
fbt_disable(void *arg, dtrace_id_t id, void *parg)
{
    fbt_probe_t *fbt = parg, *f;
    struct modctl *ctl = fbt->fbtp_ctl;

    ASSERT(ctl->mod_nenabled > 0);
    ctl->mod_nenabled--;

    for (f = fbt; f != NULL; f = f->fbtp_next) {
        if (f->fbtp_patchpoint == NULL)
            return;
    }

    if ((!fbt->fbtp_primary && !ctl->mod_loaded) ||
        (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
        return;

    for (; fbt != NULL; fbt = fbt->fbtp_next)
        *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
}

/*ARGSUSED*/
static void
fbt_suspend(void *arg, dtrace_id_t id, void *parg)
{
    fbt_probe_t *fbt = parg;
    struct modctl *ctl = fbt->fbtp_ctl;

    if (!fbt->fbtp_primary && !ctl->mod_loaded)
        return;

    if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
        return;

    ASSERT(ctl->mod_nenabled > 0);

    for (; fbt != NULL; fbt = fbt->fbtp_next)
        *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
}

/*ARGSUSED*/
static void
fbt_resume(void *arg, dtrace_id_t id, void *parg)
{
    fbt_probe_t *fbt = parg;
    struct modctl *ctl = fbt->fbtp_ctl;

    if (!fbt->fbtp_primary && !ctl->mod_loaded)
        return;

    if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
        return;

    ASSERT(ctl->mod_nenabled > 0);

    for (; fbt != NULL; fbt = fbt->fbtp_next)
        *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
}

/*ARGSUSED*/
static void
fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
{
    fbt_probe_t *fbt = parg;
    struct modctl *ctl = fbt->fbtp_ctl;
    struct module *mp = ctl->mod_mp;
    ctf_file_t *fp = NULL, *pfp;
    ctf_funcinfo_t f;
    int error;
    ctf_id_t argv[32], type;
    int argc = sizeof (argv) / sizeof (ctf_id_t);
    const char *parent;

    if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
        goto err;

    if (fbt->fbtp_return && desc->dtargd_ndx == 0) {
        (void) strcpy(desc->dtargd_native, "int");
        return;
    }

    if ((fp = ctf_modopen(mp, &error)) == NULL) {
        /*
         * We have no CTF information for this module -- and therefore
         * no args[] information.
         */
        goto err;
    }

    /*
     * If we have a parent container, we must manually import it.
     */
    if ((parent = ctf_parent_name(fp)) != NULL) {
        struct modctl *mp = &modules;
        struct modctl *mod = NULL;

        /*
         * We must iterate over all modules to find the module that
         * is our parent.
         */
        do {
            if (strcmp(mp->mod_modname, parent) == 0) {
                mod = mp;
                break;
            }
        } while ((mp = mp->mod_next) != &modules);

        if (mod == NULL)
            goto err;

        if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL)
            goto err;

        if (ctf_import(fp, pfp) != 0) {
            ctf_close(pfp);
            goto err;
        }

        ctf_close(pfp);
    }

    if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR)
        goto err;

    if (fbt->fbtp_return) {
        if (desc->dtargd_ndx > 1)
            goto err;

        ASSERT(desc->dtargd_ndx == 1);
        type = f.ctc_return;
    } else {
        if (desc->dtargd_ndx + 1 > f.ctc_argc)
            goto err;

        if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR)
            goto err;

        type = argv[desc->dtargd_ndx];
    }

    if (ctf_type_name(fp, type, desc->dtargd_native,
        DTRACE_ARGTYPELEN) != NULL) {
        ctf_close(fp);
        return;
    }
err:
    if (fp != NULL)
        ctf_close(fp);

    desc->dtargd_ndx = DTRACE_ARGNONE;
}

static dtrace_pattr_t fbt_attr = {
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
};

static dtrace_pops_t fbt_pops = {
    NULL,
    fbt_provide_module,
    fbt_enable,
    fbt_disable,
    fbt_suspend,
    fbt_resume,
    fbt_getargdesc,
    NULL,
    NULL,
    fbt_destroy
};

static int
fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
    switch (cmd) {
    case DDI_ATTACH:
        break;
    case DDI_RESUME:
        return (DDI_SUCCESS);
    default:
        return (DDI_FAILURE);
    }

    if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0,
        DDI_PSEUDO, NULL) == DDI_FAILURE ||
        dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL,
        &fbt_pops, NULL, &fbt_id) != 0) {
        ddi_remove_minor_node(devi, NULL);
        return (DDI_FAILURE);
    }

    ddi_report_dev(devi);
    fbt_devi = devi;
    return (DDI_SUCCESS);
}

static int
fbt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
{
    switch (cmd) {
    case DDI_DETACH:
        break;
    case DDI_SUSPEND:
        return (DDI_SUCCESS);
    default:
        return (DDI_FAILURE);
    }

    if (dtrace_unregister(fbt_id) != 0)
        return (DDI_FAILURE);

    ddi_remove_minor_node(devi, NULL);
    return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
fbt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
{
    int error;

    switch (infocmd) {
    case DDI_INFO_DEVT2DEVINFO:
        *result = (void *)fbt_devi;
        error = DDI_SUCCESS;
        break;
    case DDI_INFO_DEVT2INSTANCE:
        *result = (void *)0;
        error = DDI_SUCCESS;
        break;
    default:
        error = DDI_FAILURE;
    }
    return (error);
}

/*ARGSUSED*/
static int
fbt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
{
    return (0);
}

static struct cb_ops fbt_cb_ops = {
    fbt_open,       /* open */
    nodev,          /* close */
    nulldev,        /* strategy */
    nulldev,        /* print */
    nodev,          /* dump */
    nodev,          /* read */
    nodev,          /* write */
    nodev,          /* ioctl */
    nodev,          /* devmap */
    nodev,          /* mmap */
    nodev,          /* segmap */
    nochpoll,       /* poll */
    ddi_prop_op,        /* cb_prop_op */
    0,          /* streamtab  */
    D_NEW | D_MP        /* Driver compatibility flag */
};

static struct dev_ops fbt_ops = {
    DEVO_REV,       /* devo_rev */
    0,          /* refcnt */
    fbt_info,       /* get_dev_info */
    nulldev,        /* identify */
    nulldev,        /* probe */
    fbt_attach,     /* attach */
    fbt_detach,     /* detach */
    nodev,          /* reset */
    &fbt_cb_ops,        /* driver operations */
    NULL,           /* bus operations */
    nodev,          /* dev power */
    ddi_quiesce_not_needed,     /* quiesce */
};

/*
 * Module linkage information for the kernel.
 */
static struct modldrv modldrv = {
    &mod_driverops,     /* module type (this is a pseudo driver) */
    "Function Boundary Tracing",    /* name of module */
    &fbt_ops,       /* driver ops */
};

static struct modlinkage modlinkage = {
    MODREV_1,
    (void *)&modldrv,
    NULL
};

int
_init(void)
{
    return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
    return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
    return (mod_remove(&modlinkage));
}