dtrace.c revision deef35fd18fdfb1c42002a4793ebb2c181b08680
fa9e4066f08beec538e775443c5be79dd423fcabahrens * CDDL HEADER START
fa9e4066f08beec538e775443c5be79dd423fcabahrens * The contents of this file are subject to the terms of the
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * Common Development and Distribution License (the "License").
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * You may not use this file except in compliance with the License.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
fa9e4066f08beec538e775443c5be79dd423fcabahrens * See the License for the specific language governing permissions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * and limitations under the License.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * When distributing Covered Code, include this CDDL HEADER in each
fa9e4066f08beec538e775443c5be79dd423fcabahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * If applicable, add the following below this CDDL HEADER, with the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * fields enclosed by brackets "[]" replaced with your own identifying
fa9e4066f08beec538e775443c5be79dd423fcabahrens * information: Portions Copyright [yyyy] [name of copyright owner]
fa9e4066f08beec538e775443c5be79dd423fcabahrens * CDDL HEADER END
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Copyright (c) 2011, Joyent, Inc. All rights reserved.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * DTrace - Dynamic Tracing for Solaris
fa9e4066f08beec538e775443c5be79dd423fcabahrens * This is the implementation of the Solaris Dynamic Tracing framework
fa9e4066f08beec538e775443c5be79dd423fcabahrens * (DTrace). The user-visible interface to DTrace is described at length in
fa9e4066f08beec538e775443c5be79dd423fcabahrens * the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace
fa9e4066f08beec538e775443c5be79dd423fcabahrens * library, the in-kernel DTrace framework, and the DTrace providers are
fa9e4066f08beec538e775443c5be79dd423fcabahrens * described in the block comments in the <sys/dtrace.h> header file. The
fa9e4066f08beec538e775443c5be79dd423fcabahrens * internal architecture of DTrace is described in the block comments in the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * <sys/dtrace_impl.h> header file. The comments contained within the DTrace
13506d1eefbbc37e2f12a0528831d9f6d4c361d7maybee * implementation very much assume mastery of all of these sources; if one has
13506d1eefbbc37e2f12a0528831d9f6d4c361d7maybee * an unanswered question about the implementation, one should consult them
fa9e4066f08beec538e775443c5be79dd423fcabahrens * The functions here are ordered roughly as follows:
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Probe context functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Probe hashing functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Non-probe context utility functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Matching functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Provider-to-Framework API functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Probe management functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - DIF object functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Format functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Predicate functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - ECB functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Buffer functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Enabling functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - DOF functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Anonymous enabling functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Consumer state functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Helper functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Hook functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * - Driver cookbook functions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Each group of functions begins with a block comment labelled the "DTrace
fa9e4066f08beec538e775443c5be79dd423fcabahrens * [Group] Functions", allowing one to find each block by searching forward
fa9e4066f08beec538e775443c5be79dd423fcabahrens * on capital-f functions.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * DTrace Tunable Variables
fa9e4066f08beec538e775443c5be79dd423fcabahrens * The following variables may be tuned by adding a line to /etc/system that
fa9e4066f08beec538e775443c5be79dd423fcabahrens * includes both the name of the DTrace module ("dtrace") and the name of the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * variable. For example:
fa9e4066f08beec538e775443c5be79dd423fcabahrens * set dtrace:dtrace_destructive_disallow = 1
fa9e4066f08beec538e775443c5be79dd423fcabahrens * In general, the only variables that one should be tuning this way are those
fa9e4066f08beec538e775443c5be79dd423fcabahrens * that affect system-wide DTrace behavior, and for which the default behavior
fa9e4066f08beec538e775443c5be79dd423fcabahrens * is undesirable. Most of these variables are tunable on a per-consumer
fa9e4066f08beec538e775443c5be79dd423fcabahrens * basis using DTrace options, and need not be tuned on a system-wide basis.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * When tuning these variables, avoid pathological values; while some attempt
fa9e4066f08beec538e775443c5be79dd423fcabahrens * is made to verify the integrity of these variables, they are not considered
fa9e4066f08beec538e775443c5be79dd423fcabahrens * part of the supported interface to DTrace, and they are therefore not
fa9e4066f08beec538e775443c5be79dd423fcabahrens * checked comprehensively. Further, these variables should not be tuned
fa9e4066f08beec538e775443c5be79dd423fcabahrens * dynamically via "mdb -kw" or other means; they should only be tuned via
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_cleanrate_default = 9900990; /* 101 hz */
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_cleanrate_min = 200000; /* 5000 hz */
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC; /* 1/minute */
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_aggrate_default = NANOSEC; /* 1 hz */
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_statusrate_default = NANOSEC; /* 1 hz */
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC; /* 6/minute */
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_optval_t dtrace_switchrate_default = NANOSEC; /* 1 hz */
fa9e4066f08beec538e775443c5be79dd423fcabahrenshrtime_t dtrace_chill_max = 500 * (NANOSEC / MILLISEC); /* 500 ms */
fa9e4066f08beec538e775443c5be79dd423fcabahrenshrtime_t dtrace_chill_interval = NANOSEC; /* 1000 ms */
fa9e4066f08beec538e775443c5be79dd423fcabahrenshrtime_t dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
fa9e4066f08beec538e775443c5be79dd423fcabahrenshrtime_t dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
fa9e4066f08beec538e775443c5be79dd423fcabahrenshrtime_t dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;
fa9e4066f08beec538e775443c5be79dd423fcabahrens * DTrace External Variables
fa9e4066f08beec538e775443c5be79dd423fcabahrens * As dtrace(7D) is a kernel module, any DTrace variables are obviously
fa9e4066f08beec538e775443c5be79dd423fcabahrens * available to DTrace consumers via the backtick (`) syntax. One of these,
fa9e4066f08beec538e775443c5be79dd423fcabahrens * dtrace_zero, is made deliberately so: it is provided as a source of
fa9e4066f08beec538e775443c5be79dd423fcabahrens * well-known, zero-filled memory. While this variable is not documented,
fa9e4066f08beec538e775443c5be79dd423fcabahrens * it is used by some translators as an implementation detail.
fa9e4066f08beec538e775443c5be79dd423fcabahrensconst char dtrace_zero[256] = { 0 }; /* zero-filled memory */
fa9e4066f08beec538e775443c5be79dd423fcabahrens * DTrace Internal Variables
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic dtrace_probe_t **dtrace_probes; /* array of all probes */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_provider_t *dtrace_provider; /* provider list */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_meta_t *dtrace_meta_pid; /* user-land meta provider */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic void *dtrace_softstate; /* softstate pointer */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_hash_t *dtrace_bymod; /* probes hashed by module */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_hash_t *dtrace_byfunc; /* probes hashed by function */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_hash_t *dtrace_byname; /* probes hashed by name */
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic dtrace_toxrange_t *dtrace_toxrange; /* toxic range array */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic int dtrace_toxranges; /* number of toxic ranges */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic int dtrace_toxranges_max; /* size of toxic range array */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_anon_t dtrace_anon; /* anonymous enabling */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic uint64_t dtrace_vtime_references; /* number of vtimestamp refs */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic kthread_t *dtrace_panicked; /* panicking thread */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_ecb_t *dtrace_ecb_create_cache; /* cached created ECB */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_genid_t dtrace_probegen; /* current probe generation */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_helpers_t *dtrace_deferred_pid; /* deferred helper list */
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic dtrace_enabling_t *dtrace_retained; /* list of retained enablings */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_genid_t dtrace_retained_gen; /* current retained enab gen */
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_dynvar_t dtrace_dynhash_sink; /* end of dynamic hash chains */
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic int dtrace_dynvar_failclean; /* dynvars failed to clean */
fa9e4066f08beec538e775443c5be79dd423fcabahrens * DTrace Locking
fa9e4066f08beec538e775443c5be79dd423fcabahrens * DTrace is protected by three (relatively coarse-grained) locks:
fa9e4066f08beec538e775443c5be79dd423fcabahrens * (1) dtrace_lock is required to manipulate essentially any DTrace state,
fa9e4066f08beec538e775443c5be79dd423fcabahrens * including enabling state, probes, ECBs, consumer state, helper state,
fa9e4066f08beec538e775443c5be79dd423fcabahrens * etc. Importantly, dtrace_lock is _not_ required when in probe context;
fa9e4066f08beec538e775443c5be79dd423fcabahrens * probe context is lock-free -- synchronization is handled via the
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * dtrace_sync() cross call mechanism.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * (2) dtrace_provider_lock is required when manipulating provider state, or
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * when provider state must be held constant.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * (3) dtrace_meta_lock is required when manipulating meta provider state, or
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * when meta provider state must be held constant.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * The lock ordering between these three locks is dtrace_meta_lock before
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * dtrace_provider_lock before dtrace_lock. (In particular, there are
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * several places where dtrace_provider_lock is held by the framework as it
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * calls into the providers -- which then call back into the framework,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * grabbing dtrace_lock.)
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * There are two other locks in the mix: mod_lock and cpu_lock. With respect
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * role as a coarse-grained lock; it is acquired before both of these locks.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * mod_lock is similar with respect to dtrace_provider_lock in that it must be
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * acquired _between_ dtrace_provider_lock and dtrace_lock.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic kmutex_t dtrace_provider_lock; /* provider state lock */
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic kmutex_t dtrace_meta_lock; /* meta-provider state lock */
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * DTrace Provider Variables
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * These are the variables relating to DTrace as a provider (that is, the
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * provider of the BEGIN, END, and ERROR probes).
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (0);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (void (*)(void *, dtrace_id_t, void *))dtrace_nullop
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic dtrace_id_t dtrace_probeid_end; /* special END probe */
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickdtrace_id_t dtrace_probeid_error; /* special ERROR probe */
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * DTrace Helper Tracing Variables
fa9e4066f08beec538e775443c5be79dd423fcabahrens * DTrace Error Hashing
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * On DEBUG kernels, DTrace will track the errors that has seen in a hash
fa9e4066f08beec538e775443c5be79dd423fcabahrens * table. This is very useful for checking coverage of tests that are
fa9e4066f08beec538e775443c5be79dd423fcabahrens * expected to induce DIF or DOF processing errors, and may be useful for
fa9e4066f08beec538e775443c5be79dd423fcabahrens * debugging problems in the DIF code generator or in DOF generation . The
fa9e4066f08beec538e775443c5be79dd423fcabahrens * error hash may be examined with the ::dtrace_errhash MDB dcmd.
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic const char *dtrace_errlast;
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * DTrace Macros and Constants
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * These are various macros that are useful in various spots in the
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * implementation, along with a few random constants that have no meaning
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * outside of the implementation. There is no real structure to this cpp
fa9e4066f08beec538e775443c5be79dd423fcabahrens * mishmash -- but is there ever?
fa9e4066f08beec538e775443c5be79dd423fcabahrens dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
fa9e4066f08beec538e775443c5be79dd423fcabahrens (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
fa9e4066f08beec538e775443c5be79dd423fcabahrens *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick#define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3)
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * The key for a thread-local variable consists of the lower 61 bits of the
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
fa9e4066f08beec538e775443c5be79dd423fcabahrens * equal to a variable identifier. This is necessary (but not sufficient) to
fa9e4066f08beec538e775443c5be79dd423fcabahrens * assure that global associative arrays never collide with thread-local
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * variables. To guarantee that they cannot collide, we must also define the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * order for keying dynamic variables. That order is:
fa9e4066f08beec538e775443c5be79dd423fcabahrens * [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Because the variable-key and the tls-key are in orthogonal spaces, there is
fa9e4066f08beec538e775443c5be79dd423fcabahrens * no way for a global variable key signature to match a thread-local key
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * signature.
fa9e4066f08beec538e775443c5be79dd423fcabahrens uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick#define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick#define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
fa9e4066f08beec538e775443c5be79dd423fcabahrens#define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
fa9e4066f08beec538e775443c5be79dd423fcabahrens *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (0); \
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * Test whether a range of memory starting at testaddr of size testsz falls
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * within the range of memory described by addr, sz. We take care to avoid
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * problems with overflow and underflow of the unsigned quantities, and
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * disallow all negative sizes. Ranges of size 0 are allowed.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick#define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * Test whether alloc_sz bytes will fit in the scratch region. We isolate
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * alloc_sz on the righthand side of the comparison in order to avoid overflow
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * or underflow in the comparison with it. This is simpler than the INRANGE
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * check above, because we know that the dtms_scratch_ptr is valid in the
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * range. Allocations of size zero are allowed.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick/*CSTYLED*/ \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick /*CSTYLED*/ \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick for (i = 0; i < dtrace_toxranges; i++) { \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick continue; \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick continue; \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * This address falls within a toxic region; return 0. \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (0); \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick /*CSTYLED*/ \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick#define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0')
fa9e4066f08beec538e775443c5be79dd423fcabahrens (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \
fa9e4066f08beec538e775443c5be79dd423fcabahrens ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick ((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \
fa9e4066f08beec538e775443c5be79dd423fcabahrens ((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \
fa9e4066f08beec538e775443c5be79dd423fcabahrens ((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \
fa9e4066f08beec538e775443c5be79dd423fcabahrens ((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic void dtrace_enabling_provide(dtrace_provider_t *);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic int dtrace_enabling_match(dtrace_enabling_t *, int *);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic void dtrace_enabling_matchall(void);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic void dtrace_enabling_reap(void);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic uint64_t dtrace_helper(int, dtrace_mstate_t *,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic dtrace_helpers_t *dtrace_helpers_create(proc_t *);
fa9e4066f08beec538e775443c5be79dd423fcabahrensstatic int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickstatic void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * DTrace Probe Context Functions
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * These functions are called from probe context. Because probe context is
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * any context in which C may be called, arbitrarily locks may be held,
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * As a result, functions called from probe context may only call other DTrace
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * support functions -- they may not interact at all with the system at large.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * (Note that the ASSERT macro is made probe-context safe by redefining it in
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * loads are to be performed from probe context, they _must_ be in terms of
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * the safe dtrace_load*() variants.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * Some functions in this block are not actually called from probe context;
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * for these functions, there will be a comment above the function reading
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * "Note: not called from probe context."
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickdtrace_assfail(const char *a, const char *f, int l)
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * We just need something here that even the most clever compiler
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * cannot optimize away.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (a[(uintptr_t)f]);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * Atomically increment a specified error counter from probe context.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * Most counters stored to in probe context are per-CPU counters.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * However, there are some error conditions that are sufficiently
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * arcane that they don't merit per-CPU storage. If these counters
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * are incremented concurrently on different CPUs, scalability will be
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * adversely affected -- but we don't expect them to be white-hot in a
fa9e4066f08beec538e775443c5be79dd423fcabahrens * correctly constructed enabling...
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * If the counter would wrap, set it to 1 -- assuring
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * that the counter is never zero when we have seen
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * errors. (The counter must be 32-bits because we
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * aren't guaranteed a 64-bit compare&swap operation.)
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * To save this code both the infamy of being fingered
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * by a priggish news story and the indignity of being
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * the target of a neo-puritan witch trial, we're
fa9e4066f08beec538e775443c5be79dd423fcabahrens * carefully avoiding any colorful description of the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * likelihood of this condition -- but suffice it to
fa9e4066f08beec538e775443c5be79dd423fcabahrens * say that it is only slightly more likely than the
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * overflow of predicate cache IDs, as discussed in
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * dtrace_predicate_create().
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick } while (dtrace_cas32(counter, oval, nval) != oval);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
fa9e4066f08beec538e775443c5be79dd423fcabahrens * uint8_t, a uint16_t, a uint32_t and a uint64_t.
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (0);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (0);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (0);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (1);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick for (i = 0; i < nsvars; i++) {
fa9e4066f08beec538e775443c5be79dd423fcabahrens if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (0);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * Check to see if the address is within a memory region to which a store may
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * be issued. This includes the DTrace scratch areas, and any DTrace variable
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * region. The caller of dtrace_canstore() is responsible for performing any
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * alignment checks that are needed before stores are actually executed.
44cd46cadd9aab751dae6a4023c1cb5bf316d274billmdtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * First, check to see if the address is in scratch space...
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm return (1);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * Now check to see if it's a dynamic variable. This check will pick
fa9e4066f08beec538e775443c5be79dd423fcabahrens * up both thread-local variables and any global dynamically-allocated
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * variables.
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm if (DTRACE_INRANGE(addr, sz, (uintptr_t)vstate->dtvs_dynvars.dtds_base,
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * Before we assume that we can store here, we need to make
fa9e4066f08beec538e775443c5be79dd423fcabahrens * sure that it isn't in our metadata -- storing to our
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * dynamic variable metadata would corrupt our state. For
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * the range to not include any dynamic variable metadata,
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * (1) Start above the hash table that is at the base of
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * the dynamic variable space
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * (2) Have a starting chunk offset that is beyond the
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * dtrace_dynvar_t that is at the base of every chunk
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * (3) Not span a chunk boundary
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm return (0);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm return (0);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm return (0);
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (1);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * Finally, check the static local and global variables. These checks
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * take the longest, so we perform them last.
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (0);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * Convenience routine to check to see if the address is within a memory
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * region in which a load may be issued given the user's privilege level;
fa9e4066f08beec538e775443c5be79dd423fcabahrens * if not, it sets the appropriate error flags and loads 'addr' into the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * illegal value slot.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * DTrace subroutines (DIF_SUBR_*) should use this helper to implement
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * appropriate memory access protection.
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwickdtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
fa9e4066f08beec538e775443c5be79dd423fcabahrens * If we hold the privilege to read from kernel memory, then
fa9e4066f08beec538e775443c5be79dd423fcabahrens * everything is readable.
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * You can obviously read that which you can store.
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * We're allowed to read from our own string table.
fa9e4066f08beec538e775443c5be79dd423fcabahrens if (DTRACE_INRANGE(addr, sz, (uintptr_t)mstate->dtms_difo->dtdo_strtab,
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (0);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * Convenience routine to check to see if a given string is within a memory
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * region in which a load may be issued given the user's privilege level;
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * this exists so that we don't need to issue unnecessary dtrace_strlen()
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * calls in the event that the user has all privileges.
44cd46cadd9aab751dae6a4023c1cb5bf316d274billmdtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * If we hold the privilege to read from kernel memory, then
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * everything is readable.
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (1);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm return (1);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm return (0);
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * Convenience routine to check to see if a given variable is within a memory
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * region in which a load may be issued given the user's privilege level.
fa9e4066f08beec538e775443c5be79dd423fcabahrensdtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate,
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * If we hold the privilege to read from kernel memory, then
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm * everything is readable.
fa9e4066f08beec538e775443c5be79dd423fcabahrens if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1;
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (dtrace_canload((uintptr_t)src, sz, mstate, vstate));
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Compare two strings using safe loads.
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (0);
fa9e4066f08beec538e775443c5be79dd423fcabahrens flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (0);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Compute strlen(s) for a string using safe memory accesses. The additional
fa9e4066f08beec538e775443c5be79dd423fcabahrens * len parameter is used to specify a maximum length to ensure completion.
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * Check if an address falls within a toxic region.
fa9e4066f08beec538e775443c5be79dd423fcabahrens for (i = 0; i < dtrace_toxranges; i++) {
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (1);
fa9e4066f08beec538e775443c5be79dd423fcabahrens return (0);
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * Copy src to dst using safe memory accesses. The src is assumed to be unsafe
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * memory specified by the DIF program. The dst is assumed to be safe memory
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * that we can store to directly because it is managed by DTrace. As with
ecc2d604e885a75cc75e647b5641af99d5a6f4a6bonwick * standard bcopy, overlapping copies are handled properly.
fa9e4066f08beec538e775443c5be79dd423fcabahrens if (len != 0) {
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick } while (--len != 0);
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick } while (--len != 0);
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * Copy src to dst using safe memory accesses, up to either the specified
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * length, or the point that a nul byte is encountered. The src is assumed to
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * be unsafe memory specified by the DIF program. The dst is assumed to be
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * safe memory that we can store to directly because it is managed by DTrace.
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * Unlike dtrace_bcopy(), overlapping regions are not handled.
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwickdtrace_strcpy(const void *src, void *dst, size_t len)
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick if (len != 0) {
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * Copy src to dst, deriving the size and type from the specified (BYREF)
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * variable type. The src is assumed to be unsafe memory specified by the DIF
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * program. The dst is assumed to be DTrace variable memory that is of the
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * specified type; we assume that we can store to directly.
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwickdtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type)
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * Compare s1 to s2 using safe memory accesses. The s1 data is assumed to be
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * unsafe memory specified by the DIF program. The s2 data is assumed to be
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * safe memory that we can access directly because it is managed by DTrace.
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwickdtrace_bcmp(const void *s1, const void *s2, size_t len)
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick return (0);
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick return (1);
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick return (1);
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick return (0);
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * Zero the specified region using a simple byte-by-byte loop. Note that this
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick * is for safe DTrace-managed memory only.
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwickdtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
d80c45e0f58fa434ba37259ea2e2b12e0380c19abonwick (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
goto bad;
goto bad;
goto bad;
dtrace_priv_proc_common_nocd() == 0)
goto bad;
bad:
* cleaning is explained in detail in <sys/dtrace_impl.h>.
int i, j, work = 0;
for (i = 0; i < NCPU; i++) {
for (j = 0; j < NCPU; j++) {
if (j == NCPU) {
if (!work) {
dtrace_sync();
for (i = 0; i < NCPU; i++) {
dtrace_sync();
uint_t i;
for (i = 0; i < nkeys; i++) {
for (j = 0; j < size; j++) {
return (NULL);
top:
goto next;
goto next;
if (dtrace_bcmp(
goto next;
goto next;
return (dvar);
goto top;
return (NULL);
next:
goto top;
goto top;
return (NULL);
return (NULL);
void *rval;
case DTRACE_DSTATE_CLEAN: {
cpu = 0;
goto retry;
goto retry;
case DTRACE_DSTATE_DIRTY:
case DTRACE_DSTATE_RINSING:
case DTRACE_DSTATE_EMPTY:
return (NULL);
goto retry;
goto retry;
for (i = 0; i < nkeys; i++) {
if (kesize != 0) {
return (dvar);
if (val < 0) {
for (i = 0; i < zero; i++) {
ASSERT(0);
return (base);
data[0]++;
data[0]++;
if (snval < 0)
sizeof (dtrace_aggbuffer_t));
for (; i < limit; i++) {
for (; i < limit; i++) {
goto next;
next:
int nul;
if (nul) {
kdata[i] = 0;
* to the state transition diagram outlined in <sys/dtrace_impl.h>
if (which == 0)
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_DISCARDING:
case DTRACESPEC_COMMITTING:
case DTRACESPEC_ACTIVE:
case DTRACESPEC_ACTIVEONE:
case DTRACESPEC_ACTIVEMANY:
ASSERT(0);
goto out;
out:
* according to the state transition diagram outlined in <sys/dtrace_impl.h>
if (which == 0)
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_COMMITTING:
case DTRACESPEC_DISCARDING:
case DTRACESPEC_ACTIVE:
case DTRACESPEC_ACTIVEMANY:
case DTRACESPEC_ACTIVEONE:
ASSERT(0);
work++;
if (!work)
static dtrace_buffer_t *
if (which == 0)
return (NULL);
return (NULL);
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_DISCARDING:
return (NULL);
case DTRACESPEC_COMMITTING:
return (NULL);
case DTRACESPEC_ACTIVEONE:
return (buf);
case DTRACESPEC_ACTIVEMANY:
return (buf);
case DTRACESPEC_ACTIVE:
ASSERT(0);
return (buf);
return (addr);
return (NULL);
strsz);
return (ret);
static uint64_t
v = DIF_VAR_ARGS;
case DIF_VAR_ARGS:
return (val);
ASSERT(0);
case DIF_VAR_UREGS: {
case DIF_VAR_VMREGS: {
return (rval);
case DIF_VAR_CURTHREAD:
case DIF_VAR_TIMESTAMP:
case DIF_VAR_VTIMESTAMP:
case DIF_VAR_WALLTIMESTAMP:
case DIF_VAR_IPL:
case DIF_VAR_EPID:
case DIF_VAR_ID:
case DIF_VAR_STACKDEPTH:
case DIF_VAR_USTACKDEPTH:
case DIF_VAR_CALLER:
case DIF_VAR_UCALLER:
case DIF_VAR_PROBEPROV:
return (dtrace_dif_varstr(
case DIF_VAR_PROBEMOD:
return (dtrace_dif_varstr(
case DIF_VAR_PROBEFUNC:
return (dtrace_dif_varstr(
case DIF_VAR_PROBENAME:
return (dtrace_dif_varstr(
case DIF_VAR_PID:
case DIF_VAR_PPID:
case DIF_VAR_TID:
case DIF_VAR_EXECNAME:
return (dtrace_dif_varstr(
case DIF_VAR_ZONENAME:
return (dtrace_dif_varstr(
case DIF_VAR_UID:
case DIF_VAR_GID:
case DIF_VAR_ERRNO: {
switch (subr) {
case DIF_SUBR_RAND:
case DIF_SUBR_MUTEX_OWNED:
case DIF_SUBR_MUTEX_OWNER:
case DIF_SUBR_MUTEX_TYPE_SPIN:
case DIF_SUBR_RW_READ_HELD: {
case DIF_SUBR_RW_WRITE_HELD:
case DIF_SUBR_RW_ISWRITER:
case DIF_SUBR_BCOPY: {
case DIF_SUBR_ALLOCA:
case DIF_SUBR_COPYIN: {
case DIF_SUBR_COPYINTO: {
case DIF_SUBR_COPYINSTR: {
case DIF_SUBR_MSGSIZE:
case DIF_SUBR_MSGDSIZE: {
int cont = 0;
vstate)) {
case DIF_SUBR_PROGENYOF: {
proc_t *p;
int rval = 0;
case DIF_SUBR_SPECULATION:
case DIF_SUBR_COPYOUT: {
if (!dtrace_destructive_disallow &&
case DIF_SUBR_COPYOUTSTR: {
if (!dtrace_destructive_disallow &&
case DIF_SUBR_STRLEN: {
case DIF_SUBR_STRCHR:
case DIF_SUBR_STRRCHR: {
case DIF_SUBR_STRSTR:
case DIF_SUBR_INDEX:
case DIF_SUBR_RINDEX: {
vstate)) {
if (pos < 0) {
if (sublen == 0)
if (pos < 0)
pos = 0;
if (sublen == 0)
case DIF_SUBR_STRTOK: {
for (i = 0; i < sizeof (tokmap); i++)
tokmap[i] = 0;
dest[i++] = c;
case DIF_SUBR_SUBSTR: {
int64_t i;
if (index < 0) {
index = 0;
remaining = 0;
} else if (remaining < 0) {
for (i = 0; i < remaining; i++) {
case DIF_SUBR_TOUPPER:
case DIF_SUBR_TOLOWER: {
int64_t i;
dest[i] = c;
case DIF_SUBR_GETMAJOR:
#ifdef _LP64
case DIF_SUBR_GETMINOR:
#ifdef _LP64
case DIF_SUBR_DDI_PATHNAME: {
uint64_t m;
#ifdef _LP64
if (m != minor) {
if (len != 0) {
if (len != 0) {
case DIF_SUBR_STRJOIN: {
if (i >= size) {
if (i >= size) {
if (i < size) {
case DIF_SUBR_LLTOSTR: {
case DIF_SUBR_HTONS:
case DIF_SUBR_NTOHS:
#ifdef _BIG_ENDIAN
case DIF_SUBR_HTONL:
case DIF_SUBR_NTOHL:
#ifdef _BIG_ENDIAN
case DIF_SUBR_HTONLL:
case DIF_SUBR_NTOHLL:
#ifdef _BIG_ENDIAN
case DIF_SUBR_DIRNAME:
case DIF_SUBR_BASENAME: {
if (len == 0) {
lastbase = i;
lastdir = i;
lastdir = 0;
firstbase = 0;
lastdir = 0;
start = 0;
case DIF_SUBR_CLEANPATH: {
next:
dest[j++] = c;
goto next;
dest[j++] = c;
goto next;
dest[j++] = c;
dest[j++] = c;
case DIF_SUBR_INET_NTOA:
case DIF_SUBR_INET_NTOA6:
case DIF_SUBR_INET_NTOP: {
argi = 0;
if (val == 0) {
for (i = 0; i < sizeof (struct in6_addr); i++) {
tryzero = i;
i >= DTRACE_V4MAPPED_OFFSET; i--) {
if (val == 0) {
if (i > DTRACE_V4MAPPED_OFFSET)
goto inetout;
if (val == 0) {
static uint64_t
dtrace_difv_t *v;
case DIF_OP_OR:
case DIF_OP_XOR:
case DIF_OP_AND:
case DIF_OP_SLL:
case DIF_OP_SRL:
case DIF_OP_SUB:
case DIF_OP_ADD:
case DIF_OP_MUL:
case DIF_OP_SDIV:
case DIF_OP_UDIV:
case DIF_OP_SREM:
case DIF_OP_UREM:
case DIF_OP_NOT:
case DIF_OP_MOV:
case DIF_OP_CMP:
cc_v = 0;
case DIF_OP_TST:
case DIF_OP_BA:
case DIF_OP_BE:
if (cc_z)
case DIF_OP_BNE:
if (cc_z == 0)
case DIF_OP_BG:
case DIF_OP_BGU:
case DIF_OP_BGE:
case DIF_OP_BGEU:
if (cc_c == 0)
case DIF_OP_BL:
case DIF_OP_BLU:
if (cc_c)
case DIF_OP_BLE:
case DIF_OP_BLEU:
case DIF_OP_RLDSB:
case DIF_OP_LDSB:
case DIF_OP_RLDSH:
case DIF_OP_LDSH:
case DIF_OP_RLDSW:
case DIF_OP_LDSW:
case DIF_OP_RLDUB:
case DIF_OP_LDUB:
case DIF_OP_RLDUH:
case DIF_OP_LDUH:
case DIF_OP_RLDUW:
case DIF_OP_LDUW:
case DIF_OP_RLDX:
case DIF_OP_LDX:
case DIF_OP_ULDSB:
case DIF_OP_ULDSH:
case DIF_OP_ULDSW:
case DIF_OP_ULDUB:
case DIF_OP_ULDUH:
case DIF_OP_ULDUW:
case DIF_OP_ULDX:
case DIF_OP_RET:
case DIF_OP_NOP:
case DIF_OP_SETX:
case DIF_OP_SETS:
case DIF_OP_SCMP: {
case DIF_OP_LDGA:
case DIF_OP_LDGS:
uintptr_t a;
case DIF_OP_STGS:
*(uint8_t *)a = 0;
a += sizeof (uint64_t);
if (!dtrace_vcanload(
(void *)a, &v->dtdv_type);
case DIF_OP_LDTA:
case DIF_OP_LDLS:
case DIF_OP_STLS:
*(uint8_t *)a = 0;
a += sizeof (uint64_t);
if (!dtrace_vcanload(
(void *)a, &v->dtdv_type);
case DIF_OP_LDTS: {
case DIF_OP_STTS: {
if (!dtrace_vcanload(
case DIF_OP_SRA:
case DIF_OP_CALL:
case DIF_OP_PUSHTR:
case DIF_OP_PUSHTV:
case DIF_OP_POPTS:
if (ttop != 0)
ttop--;
case DIF_OP_FLUSHTS:
ttop = 0;
case DIF_OP_LDGAA:
case DIF_OP_LDTAA: {
case DIF_OP_STGAA:
case DIF_OP_STTAA: {
if (!dtrace_vcanload(
case DIF_OP_ALLOCS: {
case DIF_OP_COPYS:
case DIF_OP_STB:
case DIF_OP_STH:
case DIF_OP_STW:
case DIF_OP_STX:
return (rval);
c[i++] = *msg++;
c[i++] = *str;
c[i++] = *str;
c[i++] = *str;
c[i++] = *str;
c[i++] = *ecbmsg++;
while (shift >= 0) {
debug_enter(c);
dtrace_action_stop(void)
char *sym;
goto out;
for (i = 0; i < nframes; i++) {
out:
if (panic_quiesce) {
int committed = 0;
#ifdef lint
int rval;
case DTRACEACT_STOP:
&mstate))
case DTRACEACT_BREAKPOINT:
case DTRACEACT_PANIC:
case DTRACEACT_STACK:
case DTRACEACT_JSTACK:
case DTRACEACT_USTACK:
case DTRACEACT_SPECULATE:
if (offs < 0) {
case DTRACEACT_CHILL:
case DTRACEACT_RAISE:
&mstate))
case DTRACEACT_COMMIT:
case DTRACEACT_DISCARD:
case DTRACEACT_DIFEXPR:
case DTRACEACT_LIBACT:
case DTRACEACT_PRINTF:
case DTRACEACT_PRINTA:
case DTRACEACT_SYSTEM:
case DTRACEACT_FREOPEN:
case DTRACEACT_TRACEMEM:
case DTRACEACT_SYM:
case DTRACEACT_MOD:
case DTRACEACT_USYM:
case DTRACEACT_UMOD:
case DTRACEACT_UADDR: {
case DTRACEACT_EXIT: {
ASSERT(0);
if (tracememsize != 0 &&
tracememsize = 0;
size_t s;
for (s = 0; s < size; s++) {
valoffs++, c);
switch (size) {
case sizeof (uint8_t):
case sizeof (uint16_t):
case sizeof (uint32_t):
case sizeof (uint64_t):
ASSERT(0);
int ndx;
if (vtime) {
if (!committed)
if (vtime)
static uint_t
dtrace_hash_str(char *p)
hval &= ~g;
return (hval);
static dtrace_hash_t *
return (hash);
#ifdef DEBUG
for (i = 0; i < size; i++) {
goto add;
add:
static dtrace_probe_t *
return (NULL);
return (NULL);
if (b == bucket) {
b = b->dthb_next;
return (new);
#define DTRACE_ISALPHA(c) \
dtrace_badname(const char *s)
priv = 0;
#ifdef DTRACE_ERRDEBUG
int occupied = 0;
goto out;
goto out;
out:
DTRACE_PRIV_KERNEL)) == 0)
int rv;
return (rv);
return (rv);
return (rv);
return (rv);
return (rv);
const char *olds;
char s1, c;
int gs;
if (s == NULL)
top:
olds = s;
s1 = *s++;
if (p == NULL)
if (notflag) {
ok++;
ok++;
if (notflag) {
if (s1 != c)
ok++;
} else if (s1 == c)
ok++;
if (ok)
goto top;
if (c != s1)
goto top;
return (gs);
dtrace_id_t i;
return (DTRACE_MATCH_FAIL);
nmatched++;
return (nmatched);
for (i = 0; i < dtrace_nprobes; i++) {
zoneid) <= 0)
nmatched++;
return (DTRACE_MATCH_FAIL);
return (nmatched);
nmatched++;
return (DTRACE_MATCH_FAIL);
return (nmatched);
static dtrace_probekey_f *
dtrace_probekey_func(const char *p)
return (&dtrace_match_nul);
return (&dtrace_match_glob);
return (&dtrace_match_string);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EBUSY);
if (!self) {
return (EBUSY);
for (i = 0; i < dtrace_nprobes; i++) {
if (!self) {
if (noreap)
return (EBUSY);
return (EAGAIN);
for (i = 0; i < dtrace_nprobes; i++) {
dtrace_sync();
if (!self) {
dtrace_attached(void)
for (i = 0; i < dtrace_nprobes; i++) {
if (nsize == 0) {
dtrace_sync();
return (id);
static dtrace_probe_t *
return (NULL);
return (DTRACE_MATCH_DONE);
int match;
return (rval);
int all = 0;
for (i = 0; i < dtrace_nprobes; i++) {
enab));
char *strtab;
void *parg;
for (i = 0; i < nprobes; i++) {
char *strtab;
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EBUSY);
if (dtrace_err_verbose) {
#ifdef DTRACE_ERRDEBUG
int err = 0, i;
int kcheckload;
switch (op) {
case DIF_OP_OR:
case DIF_OP_XOR:
case DIF_OP_AND:
case DIF_OP_SLL:
case DIF_OP_SRL:
case DIF_OP_SRA:
case DIF_OP_SUB:
case DIF_OP_ADD:
case DIF_OP_MUL:
case DIF_OP_SDIV:
case DIF_OP_UDIV:
case DIF_OP_SREM:
case DIF_OP_UREM:
case DIF_OP_COPYS:
if (rd == 0)
case DIF_OP_NOT:
case DIF_OP_MOV:
case DIF_OP_ALLOCS:
if (r2 != 0)
if (rd == 0)
case DIF_OP_LDSB:
case DIF_OP_LDSH:
case DIF_OP_LDSW:
case DIF_OP_LDUB:
case DIF_OP_LDUH:
case DIF_OP_LDUW:
case DIF_OP_LDX:
if (r2 != 0)
if (rd == 0)
if (kcheckload)
case DIF_OP_RLDSB:
case DIF_OP_RLDSH:
case DIF_OP_RLDSW:
case DIF_OP_RLDUB:
case DIF_OP_RLDUH:
case DIF_OP_RLDUW:
case DIF_OP_RLDX:
if (r2 != 0)
if (rd == 0)
case DIF_OP_ULDSB:
case DIF_OP_ULDSH:
case DIF_OP_ULDSW:
case DIF_OP_ULDUB:
case DIF_OP_ULDUH:
case DIF_OP_ULDUW:
case DIF_OP_ULDX:
if (r2 != 0)
if (rd == 0)
case DIF_OP_STB:
case DIF_OP_STH:
case DIF_OP_STW:
case DIF_OP_STX:
if (r2 != 0)
if (rd == 0)
case DIF_OP_CMP:
case DIF_OP_SCMP:
if (rd != 0)
case DIF_OP_TST:
case DIF_OP_BA:
case DIF_OP_BE:
case DIF_OP_BNE:
case DIF_OP_BG:
case DIF_OP_BGU:
case DIF_OP_BGE:
case DIF_OP_BGEU:
case DIF_OP_BL:
case DIF_OP_BLU:
case DIF_OP_BLE:
case DIF_OP_BLEU:
label);
label);
case DIF_OP_RET:
case DIF_OP_NOP:
case DIF_OP_POPTS:
case DIF_OP_FLUSHTS:
case DIF_OP_SETX:
if (rd == 0)
case DIF_OP_SETS:
if (rd == 0)
case DIF_OP_LDGA:
case DIF_OP_LDTA:
if (rd == 0)
case DIF_OP_LDGS:
case DIF_OP_LDTS:
case DIF_OP_LDLS:
case DIF_OP_LDGAA:
case DIF_OP_LDTAA:
if (rd == 0)
case DIF_OP_STGS:
case DIF_OP_STTS:
case DIF_OP_STLS:
case DIF_OP_STGAA:
case DIF_OP_STTAA:
case DIF_OP_CALL:
if (rd == 0)
case DIF_OP_PUSHTR:
case DIF_OP_PUSHTV:
case sizeof (uint8_t):
case sizeof (uint16_t):
case sizeof (uint32_t):
case sizeof (uint64_t):
v->dtdv_scope);
v->dtdv_kind);
switch (v->dtdv_scope) {
case DIFV_SCOPE_GLOBAL:
case DIFV_SCOPE_THREAD:
case DIFV_SCOPE_LOCAL:
return (err);
int err = 0;
switch (op) {
case DIF_OP_OR:
case DIF_OP_XOR:
case DIF_OP_AND:
case DIF_OP_SLL:
case DIF_OP_SRL:
case DIF_OP_SRA:
case DIF_OP_SUB:
case DIF_OP_ADD:
case DIF_OP_MUL:
case DIF_OP_SDIV:
case DIF_OP_UDIV:
case DIF_OP_SREM:
case DIF_OP_UREM:
case DIF_OP_COPYS:
case DIF_OP_NOT:
case DIF_OP_MOV:
case DIF_OP_RLDSB:
case DIF_OP_RLDSH:
case DIF_OP_RLDSW:
case DIF_OP_RLDUB:
case DIF_OP_RLDUH:
case DIF_OP_RLDUW:
case DIF_OP_RLDX:
case DIF_OP_ULDSB:
case DIF_OP_ULDSH:
case DIF_OP_ULDSW:
case DIF_OP_ULDUB:
case DIF_OP_ULDUH:
case DIF_OP_ULDUW:
case DIF_OP_ULDX:
case DIF_OP_STB:
case DIF_OP_STH:
case DIF_OP_STW:
case DIF_OP_STX:
case DIF_OP_ALLOCS:
case DIF_OP_CMP:
case DIF_OP_SCMP:
case DIF_OP_TST:
case DIF_OP_BA:
case DIF_OP_BE:
case DIF_OP_BNE:
case DIF_OP_BG:
case DIF_OP_BGU:
case DIF_OP_BGE:
case DIF_OP_BGEU:
case DIF_OP_BL:
case DIF_OP_BLU:
case DIF_OP_BLE:
case DIF_OP_BLEU:
case DIF_OP_RET:
case DIF_OP_NOP:
case DIF_OP_POPTS:
case DIF_OP_FLUSHTS:
case DIF_OP_SETX:
case DIF_OP_SETS:
case DIF_OP_LDGA:
case DIF_OP_LDLS:
case DIF_OP_STGS:
case DIF_OP_STLS:
case DIF_OP_PUSHTR:
case DIF_OP_PUSHTV:
case DIF_OP_LDGS:
if (v >= DIF_VAR_OTHER_UBASE)
case DIF_OP_LDTA:
case DIF_OP_LDTS:
case DIF_OP_LDGAA:
case DIF_OP_LDTAA:
case DIF_OP_STTS:
case DIF_OP_STGAA:
case DIF_OP_STTAA:
case DIF_OP_CALL:
return (err);
switch (v->dtdv_id) {
case DIF_VAR_CURTHREAD:
case DIF_VAR_PID:
case DIF_VAR_TID:
case DIF_VAR_EXECNAME:
case DIF_VAR_ZONENAME:
if (dtrace_vtime_references++ == 0)
switch (op) {
case DIF_OP_SETX:
case DIF_OP_STTS:
case DIF_OP_STGAA:
case DIF_OP_STTAA:
case DIF_OP_PUSHTR:
if (srd == 0)
case DIF_OP_PUSHTV:
case DIF_OP_FLUSHTS:
ttop = 0;
case DIF_OP_POPTS:
if (ttop != 0)
ttop--;
sval = 0;
srd = 0;
if (nkeys == 0)
int *np;
switch (scope) {
case DIFV_SCOPE_THREAD:
if (osz != 0) {
case DIFV_SCOPE_LOCAL:
sizeof (uint64_t));
case DIFV_SCOPE_GLOBAL:
sizeof (uint64_t);
ASSERT(0);
if (oldsize != 0) {
static dtrace_difo_t *
return (new);
int *np;
switch (scope) {
case DIFV_SCOPE_THREAD:
case DIFV_SCOPE_LOCAL:
case DIFV_SCOPE_GLOBAL:
ASSERT(0);
if (--dtrace_vtime_references == 0)
static uint16_t
char *fmt;
static dtrace_predicate_t *
return (pred);
return (pred);
return (pred);
static dtrace_actdesc_t *
return (act);
static dtrace_ecb_t *
if (necbs == 0) {
dtrace_sync();
return (ecb);
dtrace_sync();
int wastuple = 0;
wastuple = 0;
static dtrace_action_t *
case DTRACEAGG_MIN:
case DTRACEAGG_MAX:
case DTRACEAGG_COUNT:
case DTRACEAGG_QUANTIZE:
sizeof (uint64_t);
case DTRACEAGG_LQUANTIZE: {
goto err;
case DTRACEAGG_LLQUANTIZE: {
int64_t v;
goto err;
goto err;
case DTRACEAGG_AVG:
case DTRACEAGG_STDDEV:
case DTRACEAGG_SUM:
goto err;
if (ntuple == 0)
goto err;
if (--ntuple == 0) {
goto success;
err:
return (NULL);
if (naggs == 0) {
return (EINVAL);
return (EINVAL);
return (EINVAL);
case DTRACEACT_PRINTF:
case DTRACEACT_PRINTA:
case DTRACEACT_SYSTEM:
case DTRACEACT_FREOPEN:
case DTRACEACT_DIFEXPR:
format = 0;
case DTRACEACT_LIBACT:
case DTRACEACT_TRACEMEM:
return (EINVAL);
return (EINVAL);
case DTRACEACT_STACK:
case DTRACEACT_JSTACK:
case DTRACEACT_USTACK:
case DTRACEACT_SYM:
case DTRACEACT_MOD:
sizeof (uint64_t)) ||
return (EINVAL);
case DTRACEACT_USYM:
case DTRACEACT_UMOD:
case DTRACEACT_UADDR:
return (EINVAL);
case DTRACEACT_STOP:
case DTRACEACT_BREAKPOINT:
case DTRACEACT_PANIC:
case DTRACEACT_CHILL:
case DTRACEACT_DISCARD:
case DTRACEACT_RAISE:
return (EINVAL);
case DTRACEACT_EXIT:
return (EINVAL);
case DTRACEACT_SPECULATE:
return (EINVAL);
return (EINVAL);
case DTRACEACT_COMMIT: {
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
dtrace_sync();
dtrace_sync();
if (p != NULL)
static dtrace_ecb_t *
return (ecb);
return (NULL);
return (DTRACE_MATCH_NEXT);
return (DTRACE_MATCH_DONE);
return (DTRACE_MATCH_FAIL);
return (DTRACE_MATCH_NEXT);
static dtrace_ecb_t *
return (NULL);
static dtrace_aggregation_t *
return (NULL);
return (EFBIG);
goto err;
goto err;
err:
allocated++;
allocated++;
return (ENOMEM);
static intptr_t
return (offs);
return (offs);
goto out;
woffs = 0;
woffs = 0;
offs = 0;
goto out;
if (offs == 0) {
woffs = 0;
out:
return (offs);
return (offs);
for (i = 0; i < NCPU; i++) {
for (i = 0; i < NCPU; i++) {
static dtrace_enabling_t *
return (enab);
return (ENOSPC);
return (err);
return (EBUSY);
dtrace_enabling_matchall(void)
int i, all = 0;
goto retry;
dtrace_enabling_reap(void)
for (i = 0; i < dtrace_nprobes; i++) {
if (dtrace_err_verbose)
#ifdef DTRACE_ERRDEBUG
static dof_hdr_t *
for (i = 0; i < DTRACEOPT_MAX; i++) {
return (dof);
static dof_hdr_t *
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (dof);
static dof_hdr_t *
unsigned int len, i;
return (NULL);
for (i = 0; i < len; i++)
return (NULL);
return (NULL);
return (NULL);
return (dof);
static dof_sec_t *
return (NULL);
return (NULL);
return (NULL);
return (sec);
static dtrace_probedesc_t *
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (desc);
static dtrace_difo_t *
int section;
int bufoffs;
int lenoffs;
int entsize;
int align;
const char *msg;
} difo[] = {
return (NULL);
return (NULL);
return (NULL);
void **bufp;
goto err;
goto err;
goto err;
goto err;
goto err;
goto err;
goto err;
goto err;
goto err;
return (dp);
err:
return (NULL);
static dtrace_predicate_t *
return (NULL);
static dtrace_actdesc_t *
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
uint64_t i;
goto err;
goto err;
goto err;
arg = 0;
goto err;
goto err;
return (first);
err:
return (NULL);
static dtrace_ecbdesc_t *
return (NULL);
return (NULL);
return (NULL);
goto err;
goto err;
goto err;
goto err;
goto err;
return (ep);
err:
return (NULL);
dof_relodesc_t *r;
uint_t i, n;
switch (r->dofr_type) {
case DOF_RELO_NONE:
case DOF_RELO_SETX:
uint_t i;
if (noprobes) {
case DOF_SECT_PROVIDER:
case DOF_SECT_PROBES:
case DOF_SECT_PRARGS:
case DOF_SECT_PROFFS:
case DOF_SECT_URELHDR:
int i, rval;
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (rval);
void *base;
return (ENOMEM);
hashsize--;
for (i = 0; i < hashsize; i++)
for (i = 0; i < NCPU; i++) {
if (maxper == 0)
sizeof (dtrace_statvar_t *));
sizeof (dtrace_difv_t));
sizeof (dtrace_statvar_t *));
dtrace_sync();
return (NULL);
for (i = 0; i < DTRACEOPT_MAX; i++)
return (state);
return (E2BIG);
return (rval);
return (rval);
return (ENOMEM);
int rval, i;
DTRACEOPT_BUFSIZE)) != 0)
return (rval);
DTRACEOPT_AGGSIZE)) != 0)
return (rval);
return (rval);
goto out;
goto out;
goto out;
goto out;
for (i = 0; i < nspec; i++) {
goto err;
goto out;
goto out;
goto out;
goto out;
goto err;
if (rval == 0)
goto err;
if (rval != 0)
goto err;
goto out;
err:
goto out;
out:
return (rval);
return (EINVAL);
dtrace_sync();
dtrace_sync();
dtrace_sync();
return (EBUSY);
return (EINVAL);
return (EINVAL);
switch (option) {
case DTRACEOPT_DESTRUCTIVE:
return (EACCES);
case DTRACEOPT_BUFSIZE:
case DTRACEOPT_DYNVARSIZE:
case DTRACEOPT_AGGSIZE:
case DTRACEOPT_SPECSIZE:
case DTRACEOPT_STRSIZE:
if (val < 0)
return (EINVAL);
dtrace_sync();
if (!match)
dtrace_sync();
for (i = 0; i < nspec; i++)
#ifdef DEBUG
for (i = 0; i < nspec; i++)
static dtrace_state_t *
dtrace_anon_grab(void)
return (NULL);
return (state);
dtrace_anon_property(void)
int i, rv;
dtrace_err_verbose = 0;
if (rv == 0)
dtrace_err_verbose = 0;
if (rv != 0) {
int rval;
if (!dtrace_helptrace_enabled)
next = 0;
static uint64_t
if (trace)
goto next;
goto err;
if (trace)
goto err;
next:
if (trace)
if (trace)
return (rval);
err:
if (trace)
return (NULL);
return (EINVAL);
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
last = h;
p->p_pid);
int err = 0, i;
return (err == 0);
return (EINVAL);
count++;
return (ENOSPC);
goto err;
goto err;
nactions++;
goto err;
err:
return (EINVAL);
p->p_pid);
return (ENOSPC);
return (EALREADY);
sizeof (dtrace_helper_provider_t *));
sizeof (dtrace_helper_provider_t *));
sizeof (dof_provider_t))) {
for (j = 0; j < nprobes; j++) {
return (rv);
nprovs++;
ep)) != 0) {
nhelpers++;
destroy = 0;
if (destroy)
return (gen);
static dtrace_helpers_t *
return (help);
dtrace_helpers_destroy(void)
dtrace_sync();
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
h = next;
sizeof (dtrace_helper_provider_t *));
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
KM_SLEEP);
if (hasprovs)
if (dtrace_err_verbose) {
dtrace_sync();
dtrace_suspend(void)
dtrace_resume(void)
switch (what) {
case CPU_CONFIG: {
case CPU_UNCONFIG:
if (osize == 0) {
sizeof (dtrace_state_t), 0) != 0) {
return (DDI_FAILURE);
return (DDI_FAILURE);
if (dtrace_helptrace_enabled) {
return (DDI_SUCCESS);
return (ENXIO);
return (EACCES);
dtrace_opens++;
dtrace_opens--;
return (EBUSY);
return (EAGAIN);
int rval;
switch (cmd) {
case DTRACEHIOC_ADDDOF:
return (EFAULT);
case DTRACEHIOC_ADD: {
return (rval);
rval = 0;
return (rval);
case DTRACEHIOC_REMOVE: {
return (rval);
return (ENOTTY);
int rval;
switch (cmd) {
case DTRACEIOC_PROVIDER: {
return (EFAULT);
return (ESRCH);
return (EFAULT);
case DTRACEIOC_EPROBE: {
void *buf;
int nrecs;
return (EFAULT);
return (EINVAL);
return (EINVAL);
if (nrecs-- == 0)
sizeof (dtrace_recdesc_t));
return (EFAULT);
case DTRACEIOC_AGGDESC: {
int nrecs;
void *buf;
return (EFAULT);
return (EINVAL);
if (nrecs-- == 0)
return (EFAULT);
case DTRACEIOC_ENABLE: {
int err = 0;
*rv = 0;
return (rval);
return (EBUSY);
return (EINVAL);
return (rval);
return (err);
case DTRACEIOC_REPLICATE: {
int err;
return (EFAULT);
return (err);
case DTRACEIOC_PROBEMATCH:
case DTRACEIOC_PROBES: {
dtrace_id_t i;
return (EFAULT);
return (EINVAL);
return (ESRCH);
return (EFAULT);
case DTRACEIOC_PROBEARG: {
return (EFAULT);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EFAULT);
case DTRACEIOC_GO: {
if (rval != 0)
return (rval);
return (EFAULT);
case DTRACEIOC_STOP: {
if (rval != 0)
return (rval);
return (EFAULT);
case DTRACEIOC_DOFGET: {
return (EFAULT);
case DTRACEIOC_AGGSNAP:
case DTRACEIOC_BUFSNAP: {
return (EFAULT);
return (EINVAL);
return (EBUSY);
return (EFAULT);
return (EFAULT);
return (EFAULT);
return (ENOENT);
return (ENOENT);
return (EFAULT);
return (EFAULT);
case DTRACEIOC_CONF: {
return (EFAULT);
case DTRACEIOC_STATUS: {
return (ENOENT);
for (i = 0; i < NCPU; i++) {
return (EFAULT);
case DTRACEIOC_FORMAT: {
char *str;
int len;
return (EFAULT);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (ENOTTY);
switch (cmd) {
case DDI_DETACH:
case DDI_SUSPEND:
return (DDI_SUCCESS);
return (DDI_FAILURE);
if (dtrace_helpers > 0) {
return (DDI_FAILURE);
return (DDI_FAILURE);
if (dtrace_helptrace_enabled) {
dtrace_nprobes = 0;
dtrace_toxranges = 0;
dtrace_toxranges_max = 0;
return (DDI_SUCCESS);
int error;
switch (infocmd) {
case DDI_INFO_DEVT2DEVINFO:
case DDI_INFO_DEVT2INSTANCE:
*result = (void *)0;
return (error);
(void *)&modldrv,
_init(void)
_fini(void)