dtrace.c revision f497f9fe231e0e400f339c84a7d80c4aae2ac4d5
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * CDDL HEADER START
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * The contents of this file are subject to the terms of the
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * Common Development and Distribution License (the "License").
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * You may not use this file except in compliance with the License.
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * or http://www.opensolaris.org/os/licensing.
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * See the License for the specific language governing permissions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * and limitations under the License.
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * When distributing Covered Code, include this CDDL HEADER in each
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * If applicable, add the following below this CDDL HEADER, with the
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * fields enclosed by brackets "[]" replaced with your own identifying
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * information: Portions Copyright [yyyy] [name of copyright owner]
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * CDDL HEADER END
ad867662936a4c7ab2c7116d804c272338801231Lennart Poettering * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * Copyright (c) 2013, Joyent, Inc. All rights reserved.
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * Copyright (c) 2012 by Delphix. All rights reserved.
958b66ea16deddd794b3a52643bd44633e165eadLennart Poettering * DTrace - Dynamic Tracing for Solaris
5f402ae84bbc08fe8de5682e371b3f66c387da52Daniel Mack * This is the implementation of the Solaris Dynamic Tracing framework
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * (DTrace). The user-visible interface to DTrace is described at length in
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poettering * library, the in-kernel DTrace framework, and the DTrace providers are
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poettering * described in the block comments in the <sys/dtrace.h> header file. The
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poettering * internal architecture of DTrace is described in the block comments in the
0dd25fb9f005d8ab7ac4bc10a609d00569f8c56aLennart Poettering * <sys/dtrace_impl.h> header file. The comments contained within the DTrace
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * implementation very much assume mastery of all of these sources; if one has
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * an unanswered question about the implementation, one should consult them
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * The functions here are ordered roughly as follows:
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * - Probe context functions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * - Probe hashing functions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * - Non-probe context utility functions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * - Matching functions
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * - Provider-to-Framework API functions
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * - Probe management functions
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * - DIF object functions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * - Format functions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * - Predicate functions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * - ECB functions
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * - Buffer functions
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * - Enabling functions
46f08bea4b09e2cce4b50e3c082df4a92a22598cLennart Poettering * - DOF functions
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * - Anonymous enabling functions
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poettering * - Consumer state functions
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poettering * - Helper functions
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poettering * - Hook functions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * - Driver cookbook functions
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * Each group of functions begins with a block comment labelled the "DTrace
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * [Group] Functions", allowing one to find each block by searching forward
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * on capital-f functions.
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * DTrace Tunable Variables
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * The following variables may be tuned by adding a line to /etc/system that
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * includes both the name of the DTrace module ("dtrace") and the name of the
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * variable. For example:
2c27fbca2d88214bd305272308a370a962818f1eLennart Poettering * set dtrace:dtrace_destructive_disallow = 1
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * In general, the only variables that one should be tuning this way are those
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * that affect system-wide DTrace behavior, and for which the default behavior
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * is undesirable. Most of these variables are tunable on a per-consumer
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * basis using DTrace options, and need not be tuned on a system-wide basis.
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * When tuning these variables, avoid pathological values; while some attempt
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * is made to verify the integrity of these variables, they are not considered
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * part of the supported interface to DTrace, and they are therefore not
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * checked comprehensively. Further, these variables should not be tuned
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * dynamically via "mdb -kw" or other means; they should only be tuned via
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringdtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringsize_t dtrace_difo_maxsize = (256 * 1024);
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringdtrace_optval_t dtrace_dof_maxsize = (256 * 1024);
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringsize_t dtrace_global_maxsize = (16 * 1024);
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringdtrace_optval_t dtrace_helper_actions_max = 1024;
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringdtrace_optval_t dtrace_helper_providers_max = 32;
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringdtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);
471d40d92fc8e7b452dff99a156f9e0b520ded20Tom Gundersendtrace_optval_t dtrace_cleanrate_default = 9900990; /* 101 hz */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringdtrace_optval_t dtrace_cleanrate_min = 200000; /* 5000 hz */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringdtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC; /* 1/minute */
0dd25fb9f005d8ab7ac4bc10a609d00569f8c56aLennart Poetteringdtrace_optval_t dtrace_aggrate_default = NANOSEC; /* 1 hz */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringdtrace_optval_t dtrace_statusrate_default = NANOSEC; /* 1 hz */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringdtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC; /* 6/minute */
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringdtrace_optval_t dtrace_switchrate_default = NANOSEC; /* 1 hz */
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringdtrace_optval_t dtrace_specsize_default = 32 * 1024;
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringdtrace_optval_t dtrace_stackframes_default = 20;
471d40d92fc8e7b452dff99a156f9e0b520ded20Tom Gundersendtrace_optval_t dtrace_ustackframes_default = 20;
ad867662936a4c7ab2c7116d804c272338801231Lennart Poetteringdtrace_optval_t dtrace_jstackframes_default = 50;
ad867662936a4c7ab2c7116d804c272338801231Lennart Poetteringdtrace_optval_t dtrace_jstackstrsize_default = 512;
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringhrtime_t dtrace_chill_max = 500 * (NANOSEC / MILLISEC); /* 500 ms */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringhrtime_t dtrace_chill_interval = NANOSEC; /* 1000 ms */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringhrtime_t dtrace_deadman_interval = NANOSEC;
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringhrtime_t dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
15411c0cb1192799b37ec8f25d6f30e8d7292fc6David Herrmannhrtime_t dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringhrtime_t dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * DTrace External Variables
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * As dtrace(7D) is a kernel module, any DTrace variables are obviously
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * available to DTrace consumers via the backtick (`) syntax. One of these,
72290734be81e83e6ef9520c07692f68095eb5b2Tom Gundersen * dtrace_zero, is made deliberately so: it is provided as a source of
72290734be81e83e6ef9520c07692f68095eb5b2Tom Gundersen * well-known, zero-filled memory. While this variable is not documented,
72290734be81e83e6ef9520c07692f68095eb5b2Tom Gundersen * it is used by some translators as an implementation detail.
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringconst char dtrace_zero[256] = { 0 }; /* zero-filled memory */
15411c0cb1192799b37ec8f25d6f30e8d7292fc6David Herrmann * DTrace Internal Variables
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poetteringstatic dev_info_t *dtrace_devi; /* device info */
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poetteringstatic vmem_t *dtrace_arena; /* probe ID arena */
aea2429d6ec32261dbf6b9caa125fcc6ea9ea76aLennart Poetteringstatic vmem_t *dtrace_minor; /* minor number arena */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic taskq_t *dtrace_taskq; /* task queue */
22a37591ede1e9d5f325d6f10495cc91b40b775fDaniel Mackstatic dtrace_probe_t **dtrace_probes; /* array of all probes */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic int dtrace_nprobes; /* number of probes */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic dtrace_provider_t *dtrace_provider; /* provider list */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic dtrace_meta_t *dtrace_meta_pid; /* user-land meta provider */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic int dtrace_opens; /* number of opens */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic int dtrace_helpers; /* number of helpers */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic int dtrace_getf; /* number of unpriv getf()s */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic void *dtrace_softstate; /* softstate pointer */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic dtrace_hash_t *dtrace_bymod; /* probes hashed by module */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic dtrace_hash_t *dtrace_byfunc; /* probes hashed by function */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic dtrace_hash_t *dtrace_byname; /* probes hashed by name */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic dtrace_toxrange_t *dtrace_toxrange; /* toxic range array */
72290734be81e83e6ef9520c07692f68095eb5b2Tom Gundersenstatic int dtrace_toxranges; /* number of toxic ranges */
72290734be81e83e6ef9520c07692f68095eb5b2Tom Gundersenstatic int dtrace_toxranges_max; /* size of toxic range array */
72290734be81e83e6ef9520c07692f68095eb5b2Tom Gundersenstatic dtrace_anon_t dtrace_anon; /* anonymous enabling */
72290734be81e83e6ef9520c07692f68095eb5b2Tom Gundersenstatic kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poetteringstatic uint64_t dtrace_vtime_references; /* number of vtimestamp refs */
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringstatic kthread_t *dtrace_panicked; /* panicking thread */
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringstatic dtrace_ecb_t *dtrace_ecb_create_cache; /* cached created ECB */
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringstatic dtrace_genid_t dtrace_probegen; /* current probe generation */
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringstatic dtrace_helpers_t *dtrace_deferred_pid; /* deferred helper list */
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringstatic dtrace_enabling_t *dtrace_retained; /* list of retained enablings */
0db643664cf37111be163c0c64ccd66b519daf34Tom Gundersenstatic dtrace_genid_t dtrace_retained_gen; /* current retained enab gen */
8300ba218e3cf5049496937be8bce10f22d09bbcTom Gundersenstatic dtrace_dynvar_t dtrace_dynhash_sink; /* end of dynamic hash chains */
ad867662936a4c7ab2c7116d804c272338801231Lennart Poetteringstatic int dtrace_dynvar_failclean; /* dynvars failed to clean */
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * DTrace Locking
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * DTrace is protected by three (relatively coarse-grained) locks:
ad867662936a4c7ab2c7116d804c272338801231Lennart Poettering * (1) dtrace_lock is required to manipulate essentially any DTrace state,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * including enabling state, probes, ECBs, consumer state, helper state,
ad867662936a4c7ab2c7116d804c272338801231Lennart Poettering * etc. Importantly, dtrace_lock is _not_ required when in probe context;
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * probe context is lock-free -- synchronization is handled via the
2c27fbca2d88214bd305272308a370a962818f1eLennart Poettering * dtrace_sync() cross call mechanism.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * (2) dtrace_provider_lock is required when manipulating provider state, or
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * when provider state must be held constant.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * (3) dtrace_meta_lock is required when manipulating meta provider state, or
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * when meta provider state must be held constant.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * The lock ordering between these three locks is dtrace_meta_lock before
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * dtrace_provider_lock before dtrace_lock. (In particular, there are
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * several places where dtrace_provider_lock is held by the framework as it
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * calls into the providers -- which then call back into the framework,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * grabbing dtrace_lock.)
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * There are two other locks in the mix: mod_lock and cpu_lock. With respect
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * role as a coarse-grained lock; it is acquired before both of these locks.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * mod_lock is similar with respect to dtrace_provider_lock in that it must be
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * acquired _between_ dtrace_provider_lock and dtrace_lock.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic kmutex_t dtrace_lock; /* probe state lock */
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic kmutex_t dtrace_provider_lock; /* provider state lock */
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic kmutex_t dtrace_meta_lock; /* meta-provider state lock */
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * DTrace Provider Variables
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * These are the variables relating to DTrace as a provider (that is, the
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * provider of the BEGIN, END, and ERROR probes).
ad867662936a4c7ab2c7116d804c272338801231Lennart Poettering{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
ad867662936a4c7ab2c7116d804c272338801231Lennart Poettering{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
ad867662936a4c7ab2c7116d804c272338801231Lennart Poettering{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
0db643664cf37111be163c0c64ccd66b519daf34Tom Gundersen{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
0db643664cf37111be163c0c64ccd66b519daf34Tom Gundersen{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic dtrace_pops_t dtrace_provider_ops = {
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (void (*)(void *, struct modctl *))dtrace_nullop,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (void (*)(void *, dtrace_id_t, void *))dtrace_nullop
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic dtrace_id_t dtrace_probeid_end; /* special END probe */
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringdtrace_id_t dtrace_probeid_error; /* special ERROR probe */
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * DTrace Helper Tracing Variables
ad867662936a4c7ab2c7116d804c272338801231Lennart Poetteringint dtrace_helptrace_bufsize = 512 * 1024;
ad867662936a4c7ab2c7116d804c272338801231Lennart Poettering * DTrace Error Hashing
0db643664cf37111be163c0c64ccd66b519daf34Tom Gundersen * On DEBUG kernels, DTrace will track the errors that has seen in a hash
0db643664cf37111be163c0c64ccd66b519daf34Tom Gundersen * table. This is very useful for checking coverage of tests that are
0db643664cf37111be163c0c64ccd66b519daf34Tom Gundersen * expected to induce DIF or DOF processing errors, and may be useful for
0db643664cf37111be163c0c64ccd66b519daf34Tom Gundersen * debugging problems in the DIF code generator or in DOF generation . The
0db643664cf37111be163c0c64ccd66b519daf34Tom Gundersen * error hash may be examined with the ::dtrace_errhash MDB dcmd.
51323288fc628a5cac50914df915545d685b793eLennart Poetteringstatic dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poetteringstatic const char *dtrace_errlast;
51323288fc628a5cac50914df915545d685b793eLennart Poettering * DTrace Macros and Constants
51323288fc628a5cac50914df915545d685b793eLennart Poettering * These are various macros that are useful in various spots in the
51323288fc628a5cac50914df915545d685b793eLennart Poettering * implementation, along with a few random constants that have no meaning
51323288fc628a5cac50914df915545d685b793eLennart Poettering * outside of the implementation. There is no real structure to this cpp
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * mishmash -- but is there ever?
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
faec72d5dedae93f43c2dd624de1392ed9bacd80Lennart Poettering (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
9436e8cae4709b50ed57f2f5858a3ffad03d5d32Lennart Poettering (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
9436e8cae4709b50ed57f2f5858a3ffad03d5d32Lennart Poettering *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
faa133f3aa7a18f26563dc5d6b95898cb315c37aLennart Poettering#define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3)
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * The key for a thread-local variable consists of the lower 61 bits of the
faa133f3aa7a18f26563dc5d6b95898cb315c37aLennart Poettering * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * equal to a variable identifier. This is necessary (but not sufficient) to
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * assure that global associative arrays never collide with thread-local
faa133f3aa7a18f26563dc5d6b95898cb315c37aLennart Poettering * variables. To guarantee that they cannot collide, we must also define the
faa133f3aa7a18f26563dc5d6b95898cb315c37aLennart Poettering * order for keying dynamic variables. That order is:
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * Because the variable-key and the tls-key are in orthogonal spaces, there is
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering * no way for a global variable key signature to match a thread-local key
77209c3505fa856dae23ae566b729c862a9b71f4Lennart Poettering uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
74b2466e14a1961bf3ac0e8a60cfaceec705bd59Lennart Poettering (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering#define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering#define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering#define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering#define DTRACE_STORE(type, tomax, offset, what) \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering#define DTRACE_ALIGNCHECK(addr, size, flags) \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr; \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering#define DTRACE_ALIGNCHECK(addr, size, flags)
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * Test whether a range of memory starting at testaddr of size testsz falls
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * within the range of memory described by addr, sz. We take care to avoid
5ba73e9b646af4d8109a5a633aa235665858144dLennart Poettering * problems with overflow and underflow of the unsigned quantities, and
4de120ee442797bc30ecfd25b4d539cac9cb56ceLennart Poettering * disallow all negative sizes. Ranges of size 0 are allowed.
4de120ee442797bc30ecfd25b4d539cac9cb56ceLennart Poettering#define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
4de120ee442797bc30ecfd25b4d539cac9cb56ceLennart Poettering ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \
5ba73e9b646af4d8109a5a633aa235665858144dLennart Poettering (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * Test whether alloc_sz bytes will fit in the scratch region. We isolate
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * alloc_sz on the righthand side of the comparison in order to avoid overflow
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * or underflow in the comparison with it. This is simpler than the INRANGE
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * check above, because we know that the dtms_scratch_ptr is valid in the
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * range. Allocations of size zero are allowed.
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering#define DTRACE_INSCRATCH(mstate, alloc_sz) \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
7b4c2ee75faf08b7a415337b46efc670f986128aLennart Poettering (mstate)->dtms_scratch_ptr >= (alloc_sz))
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering volatile uint16_t *flags = (volatile uint16_t *) \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering &cpu_core[CPU->cpu_id].cpuc_dtrace_flags; \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering for (i = 0; i < dtrace_toxranges; i++) { \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering if (addr >= dtrace_toxrange[i].dtt_limit) \
dc75168823540076b354135f6e2de7a9a978fbcaZbigniew Jędrzejewski-Szmek if (addr + size <= dtrace_toxrange[i].dtt_base) \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering * This address falls within a toxic region; return 0. \
1716f6dcf54d4c181c2e2558e3d5414f54c8d9caLennart Poettering cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr; \
ec2c5e4398f9d65e5dfe61530f2556224733d1e6Lennart Poettering rval = *((volatile uint##bits##_t *)addr); \
ec2c5e4398f9d65e5dfe61530f2556224733d1e6Lennart Poettering return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \
75cd513ef830d8e00d0d2d6a64917fec533315dbThomas Hindoe Paaboel Andersen#define DTRACE_DYNHASH_VALID 2
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering#define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0')
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP : \
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV : \
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV : \
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering ((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \
ec2c5e4398f9d65e5dfe61530f2556224733d1e6Lennart Poettering ((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering ((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering ((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering ((act)->dta_kind == DTRACEACT_DIFEXPR && \
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic size_t dtrace_strlen(const char *, size_t);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic void dtrace_enabling_provide(dtrace_provider_t *);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic int dtrace_enabling_match(dtrace_enabling_t *, int *);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic void dtrace_enabling_matchall(void);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic dtrace_state_t *dtrace_anon_grab(void);
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poetteringstatic uint64_t dtrace_helper(int, dtrace_mstate_t *,
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poetteringstatic dtrace_helpers_t *dtrace_helpers_create(proc_t *);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic void dtrace_buffer_drop(dtrace_buffer_t *);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringstatic intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poetteringstatic int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poetteringstatic int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poetteringstatic void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poetteringstatic int dtrace_priv_proc(dtrace_state_t *, dtrace_mstate_t *);
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poettering * DTrace Probe Context Functions
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poettering * These functions are called from probe context. Because probe context is
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * any context in which C may be called, arbitrarily locks may be held,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * As a result, functions called from probe context may only call other DTrace
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * support functions -- they may not interact at all with the system at large.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * (Note that the ASSERT macro is made probe-context safe by redefining it in
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * loads are to be performed from probe context, they _must_ be in terms of
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * the safe dtrace_load*() variants.
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * Some functions in this block are not actually called from probe context;
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * for these functions, there will be a comment above the function reading
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * "Note: not called from probe context."
8bf52d3d17d364438191077d0750b8b80b5dc53aLennart Poetteringdtrace_assfail(const char *a, const char *f, int l)
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * We just need something here that even the most clever compiler
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * cannot optimize away.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering return (a[(uintptr_t)f]);
2442b93d15f5523aba0c5dc56a42757af889c483Lennart Poettering * Atomically increment a specified error counter from probe context.
2442b93d15f5523aba0c5dc56a42757af889c483Lennart Poettering * Most counters stored to in probe context are per-CPU counters.
2442b93d15f5523aba0c5dc56a42757af889c483Lennart Poettering * However, there are some error conditions that are sufficiently
2442b93d15f5523aba0c5dc56a42757af889c483Lennart Poettering * arcane that they don't merit per-CPU storage. If these counters
2442b93d15f5523aba0c5dc56a42757af889c483Lennart Poettering * are incremented concurrently on different CPUs, scalability will be
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * adversely affected -- but we don't expect them to be white-hot in a
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * correctly constructed enabling...
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering * If the counter would wrap, set it to 1 -- assuring
ea917db9e662ae6e6d0ae07e0118b323688c8616Lennart Poettering * that the counter is never zero when we have seen
ec2c5e4398f9d65e5dfe61530f2556224733d1e6Lennart Poettering * errors. (The counter must be 32-bits because we
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * aren't guaranteed a 64-bit compare&swap operation.)
da927ba997d68401563b927f92e6e40e021a8e5cMichal Schmidt * To save this code both the infamy of being fingered
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * by a priggish news story and the indignity of being
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * the target of a neo-puritan witch trial, we're
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * carefully avoiding any colorful description of the
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * likelihood of this condition -- but suffice it to
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * say that it is only slightly more likely than the
fcf57f9cf706ff5be2b5e6d41b2ac48e3e98467bLennart Poettering * overflow of predicate cache IDs, as discussed in
fcf57f9cf706ff5be2b5e6d41b2ac48e3e98467bLennart Poettering * dtrace_predicate_create().
da927ba997d68401563b927f92e6e40e021a8e5cMichal Schmidt } while (dtrace_cas32(counter, oval, nval) != oval);
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering * uint8_t, a uint16_t, a uint32_t and a uint64_t.
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringdtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering if (dest + size > mstate->dtms_scratch_ptr)
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poetteringdtrace_canstore_statvar(uint64_t addr, size_t sz,
623a4c97b9175f95c4b1c6fc34e36c56f1e4ddbfLennart Poettering for (i = 0; i < nsvars; i++) {
ec2c5e4398f9d65e5dfe61530f2556224733d1e6Lennart Poettering if (svar == NULL || svar->dtsv_size == 0)
ec2c5e4398f9d65e5dfe61530f2556224733d1e6Lennart Poettering if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
dc4d47e2c79aafa3ef646e32ff3422c4ce935c1bLennart Poettering * Check to see if the address is within a memory region to which a store may
dc4d47e2c79aafa3ef646e32ff3422c4ce935c1bLennart Poettering * be issued. This includes the DTrace scratch areas, and any DTrace variable
dc4d47e2c79aafa3ef646e32ff3422c4ce935c1bLennart Poettering * region. The caller of dtrace_canstore() is responsible for performing any
dc4d47e2c79aafa3ef646e32ff3422c4ce935c1bLennart Poettering * alignment checks that are needed before stores are actually executed.
dc4d47e2c79aafa3ef646e32ff3422c4ce935c1bLennart Poetteringdtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
ec2c5e4398f9d65e5dfe61530f2556224733d1e6Lennart Poettering * First, check to see if the address is in scratch space...
ec2c5e4398f9d65e5dfe61530f2556224733d1e6Lennart Poettering if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * Now check to see if it's a dynamic variable. This check will pick
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * up both thread-local variables and any global dynamically-allocated
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering uintptr_t base = (uintptr_t)dstate->dtds_base +
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t));
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * Before we assume that we can store here, we need to make
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * sure that it isn't in our metadata -- storing to our
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * dynamic variable metadata would corrupt our state. For
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * the range to not include any dynamic variable metadata,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * (1) Start above the hash table that is at the base of
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * the dynamic variable space
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * (2) Have a starting chunk offset that is beyond the
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * dtrace_dynvar_t that is at the base of every chunk
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * (3) Not span a chunk boundary
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering chunkoffs = (addr - base) % dstate->dtds_chunksize;
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (chunkoffs < sizeof (dtrace_dynvar_t))
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (chunkoffs + sz > dstate->dtds_chunksize)
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * Finally, check the static local and global variables. These checks
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * take the longest, so we perform them last.
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering vstate->dtvs_locals, vstate->dtvs_nlocals))
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering vstate->dtvs_globals, vstate->dtvs_nglobals))
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * Convenience routine to check to see if the address is within a memory
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * region in which a load may be issued given the user's privilege level;
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * if not, it sets the appropriate error flags and loads 'addr' into the
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * illegal value slot.
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * DTrace subroutines (DIF_SUBR_*) should use this helper to implement
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * appropriate memory access protection.
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poetteringdtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * If we hold the privilege to read from kernel memory, then
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * everything is readable.
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * You can obviously read that which you can store.
1e43061b67336052b5b231840a38508a5397a363Michal Schmidt if (dtrace_canstore(addr, sz, mstate, vstate))
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * We're allowed to read from our own string table.
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering dtrace_priv_proc(vstate->dtvs_state, mstate)) {
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * When we have privileges to the current process, there are
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * several context-related kernel structures that are safe to
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * read, even absent the privilege to read from kernel memory.
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * These reads are safe because these structures contain only
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * state that (1) we're permitted to read, (2) is harmless or
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * (3) contains pointers to additional kernel state that we're
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * not permitted to read (and as such, do not present an
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * opportunity for privilege escalation). Finally (and
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * critically), because of the nature of their relation with
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * the current thread context, the memory associated with these
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * structures cannot change over the duration of probe context,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * and it is therefore impossible for this memory to be
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * deallocated and reallocated as something else while it's
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * being operated upon.
f647962d64e844689f3e2acfce6102fc47e76df2Michal Schmidt if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t)))
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering sz, curthread->t_procp, sizeof (proc_t))) {
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz,
8b757a38611006a751c90933d1810cccaa47e1afDaniel Mack if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz,
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) {
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * When getf() returns a file_t, the enabling is implicitly
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * granted the (transient) right to read the returned file_t
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * as well as the v_path and v_op->vnop_name of the underlying
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * vnode. These accesses are allowed after a successful
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * getf() because the members that they refer to cannot change
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * once set -- and the barrier logic in the kernel's closef()
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * path assures that the file_t and its referenced vode_t
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * cannot themselves be stale (that is, it impossible for
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * either dtms_getf itself or its f_vnode member to reference
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering * freed memory).
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t)))
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz))
a407657425a3e47fd2b559cd3bc800f791303f63Lennart Poettering if (vp->v_path != NULL && DTRACE_INRANGE(addr, sz,
* NOTE: This function uses various macros from strtolctype.h to manipulate
static int64_t
pos++;
c = ccc;
static size_t
return (len);
for (i = 0; i < dtrace_toxranges; i++) {
if (len != 0) {
} while (--len != 0);
} while (--len != 0);
if (len != 0) {
*cp++ = 0;
goto bad;
goto bad;
goto bad;
dtrace_priv_proc_common_nocd() == 0)
goto bad;
bad:
* cleaning is explained in detail in <sys/dtrace_impl.h>.
int i, j, work = 0;
for (i = 0; i < NCPU; i++) {
for (j = 0; j < NCPU; j++) {
if (j == NCPU) {
if (!work) {
dtrace_sync();
for (i = 0; i < NCPU; i++) {
dtrace_sync();
uint_t i;
for (i = 0; i < nkeys; i++) {
for (j = 0; j < size; j++) {
return (NULL);
top:
goto next;
goto next;
if (dtrace_bcmp(
goto next;
goto next;
return (dvar);
goto top;
return (NULL);
next:
goto top;
goto top;
return (NULL);
return (NULL);
void *rval;
case DTRACE_DSTATE_CLEAN: {
cpu = 0;
goto retry;
goto retry;
case DTRACE_DSTATE_DIRTY:
case DTRACE_DSTATE_RINSING:
case DTRACE_DSTATE_EMPTY:
return (NULL);
goto retry;
goto retry;
for (i = 0; i < nkeys; i++) {
if (kesize != 0) {
return (dvar);
if (val < 0) {
for (i = 0; i < zero; i++) {
ASSERT(0);
return (base);
data[0]++;
data[0]++;
if (snval < 0)
sizeof (dtrace_aggbuffer_t));
for (; i < limit; i++) {
for (; i < limit; i++) {
goto next;
next:
int nul;
if (nul) {
kdata[i] = 0;
* to the state transition diagram outlined in <sys/dtrace_impl.h>
if (which == 0)
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_DISCARDING:
case DTRACESPEC_COMMITTING:
case DTRACESPEC_ACTIVE:
case DTRACESPEC_ACTIVEONE:
case DTRACESPEC_ACTIVEMANY:
ASSERT(0);
goto out;
out:
* according to the state transition diagram outlined in <sys/dtrace_impl.h>
if (which == 0)
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_COMMITTING:
case DTRACESPEC_DISCARDING:
case DTRACESPEC_ACTIVE:
case DTRACESPEC_ACTIVEMANY:
case DTRACESPEC_ACTIVEONE:
ASSERT(0);
work++;
if (!work)
static dtrace_buffer_t *
if (which == 0)
return (NULL);
return (NULL);
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_DISCARDING:
return (NULL);
case DTRACESPEC_COMMITTING:
return (NULL);
case DTRACESPEC_ACTIVEONE:
return (buf);
case DTRACESPEC_ACTIVEMANY:
return (buf);
case DTRACESPEC_ACTIVE:
ASSERT(0);
return (buf);
return (addr);
return (NULL);
strsz);
return (ret);
static uint64_t
v = DIF_VAR_ARGS;
case DIF_VAR_ARGS:
return (val);
ASSERT(0);
case DIF_VAR_UREGS: {
case DIF_VAR_VMREGS: {
return (rval);
case DIF_VAR_CURTHREAD:
case DIF_VAR_TIMESTAMP:
case DIF_VAR_VTIMESTAMP:
case DIF_VAR_WALLTIMESTAMP:
case DIF_VAR_IPL:
case DIF_VAR_EPID:
case DIF_VAR_ID:
case DIF_VAR_STACKDEPTH:
case DIF_VAR_USTACKDEPTH:
case DIF_VAR_CALLER:
case DIF_VAR_UCALLER:
case DIF_VAR_PROBEPROV:
return (dtrace_dif_varstr(
case DIF_VAR_PROBEMOD:
return (dtrace_dif_varstr(
case DIF_VAR_PROBEFUNC:
return (dtrace_dif_varstr(
case DIF_VAR_PROBENAME:
return (dtrace_dif_varstr(
case DIF_VAR_PID:
case DIF_VAR_PPID:
case DIF_VAR_TID:
case DIF_VAR_EXECNAME:
return (dtrace_dif_varstr(
case DIF_VAR_ZONENAME:
return (dtrace_dif_varstr(
case DIF_VAR_UID:
case DIF_VAR_GID:
case DIF_VAR_ERRNO: {
typedef enum dtrace_json_state {
* specification (http://json.org/). Briefly:
* NOTE: This function uses various macros from strtolctype.h to manipulate
char *dest)
return (NULL);
switch (state) {
case DTRACE_JSON_REST:
array_pos = 0;
return (NULL);
case DTRACE_JSON_OBJECT:
return (NULL);
case DTRACE_JSON_STRING:
if (collect_object) {
if (string_is_key) {
size) == 0)
} else if (found_key) {
return (NULL);
return (dest);
escape_unicount = 0;
return (NULL);
case DTRACE_JSON_COLON:
return (NULL);
case DTRACE_JSON_COMMA:
if (in_array) {
return (NULL);
case DTRACE_JSON_IDENTIFIER:
if (found_key) {
return (NULL);
return (dest);
cur--;
return (NULL);
case DTRACE_JSON_NUMBER:
return (NULL);
case DTRACE_JSON_NUMBER_FRAC:
return (NULL);
case DTRACE_JSON_NUMBER_EXP:
if (found_key) {
return (NULL);
return (dest);
cur--;
case DTRACE_JSON_VALUE:
elem++;
nelems--;
if (in_array) {
array_pos = 0;
return (NULL);
return (NULL);
if (brackets-- == 0) {
return (NULL);
if (braces-- == 0) {
return (NULL);
braces++;
brackets++;
if (found_key) {
return (dest);
return (NULL);
switch (subr) {
case DIF_SUBR_RAND:
case DIF_SUBR_MUTEX_OWNED:
case DIF_SUBR_MUTEX_OWNER:
case DIF_SUBR_MUTEX_TYPE_SPIN:
case DIF_SUBR_RW_READ_HELD: {
case DIF_SUBR_RW_WRITE_HELD:
case DIF_SUBR_RW_ISWRITER:
case DIF_SUBR_BCOPY: {
case DIF_SUBR_ALLOCA:
case DIF_SUBR_COPYIN: {
case DIF_SUBR_COPYINTO: {
case DIF_SUBR_COPYINSTR: {
case DIF_SUBR_MSGSIZE:
case DIF_SUBR_MSGDSIZE: {
int cont = 0;
vstate)) {
case DIF_SUBR_PROGENYOF: {
proc_t *p;
int rval = 0;
case DIF_SUBR_SPECULATION:
case DIF_SUBR_COPYOUT: {
if (!dtrace_destructive_disallow &&
case DIF_SUBR_COPYOUTSTR: {
if (!dtrace_destructive_disallow &&
case DIF_SUBR_STRLEN: {
case DIF_SUBR_STRCHR:
case DIF_SUBR_STRRCHR: {
case DIF_SUBR_STRSTR:
case DIF_SUBR_INDEX:
case DIF_SUBR_RINDEX: {
vstate)) {
if (pos < 0) {
if (sublen == 0)
if (pos < 0)
pos = 0;
if (sublen == 0)
case DIF_SUBR_STRTOK: {
for (i = 0; i < sizeof (tokmap); i++)
tokmap[i] = 0;
dest[i++] = c;
case DIF_SUBR_SUBSTR: {
int64_t i;
if (index < 0) {
index = 0;
remaining = 0;
} else if (remaining < 0) {
for (i = 0; i < remaining; i++) {
case DIF_SUBR_JSON: {
nelems++;
case DIF_SUBR_TOUPPER:
case DIF_SUBR_TOLOWER: {
int64_t i;
dest[i] = c;
case DIF_SUBR_GETMAJOR:
#ifdef _LP64
case DIF_SUBR_GETMINOR:
#ifdef _LP64
case DIF_SUBR_DDI_PATHNAME: {
uint64_t m;
#ifdef _LP64
if (m != minor) {
if (len != 0) {
if (len != 0) {
case DIF_SUBR_STRJOIN: {
if (i >= size) {
if (i >= size) {
if (i < size) {
case DIF_SUBR_STRTOLL: {
case DIF_SUBR_LLTOSTR: {
case DIF_SUBR_HTONS:
case DIF_SUBR_NTOHS:
#ifdef _BIG_ENDIAN
case DIF_SUBR_HTONL:
case DIF_SUBR_NTOHL:
#ifdef _BIG_ENDIAN
case DIF_SUBR_HTONLL:
case DIF_SUBR_NTOHLL:
#ifdef _BIG_ENDIAN
case DIF_SUBR_DIRNAME:
case DIF_SUBR_BASENAME: {
if (len == 0) {
lastbase = i;
lastdir = i;
lastdir = 0;
firstbase = 0;
lastdir = 0;
start = 0;
case DIF_SUBR_GETF: {
case DIF_SUBR_CLEANPATH: {
zone_t *z;
next:
dest[j++] = c;
goto next;
dest[j++] = c;
goto next;
dest[j++] = c;
dest[j++] = c;
case DIF_SUBR_INET_NTOA:
case DIF_SUBR_INET_NTOA6:
case DIF_SUBR_INET_NTOP: {
argi = 0;
if (val == 0) {
for (i = 0; i < sizeof (struct in6_addr); i++) {
tryzero = i;
i >= DTRACE_V4MAPPED_OFFSET; i--) {
if (val == 0) {
if (i > DTRACE_V4MAPPED_OFFSET)
goto inetout;
if (val == 0) {
static uint64_t
dtrace_difv_t *v;
case DIF_OP_OR:
case DIF_OP_XOR:
case DIF_OP_AND:
case DIF_OP_SLL:
case DIF_OP_SRL:
case DIF_OP_SUB:
case DIF_OP_ADD:
case DIF_OP_MUL:
case DIF_OP_SDIV:
case DIF_OP_UDIV:
case DIF_OP_SREM:
case DIF_OP_UREM:
case DIF_OP_NOT:
case DIF_OP_MOV:
case DIF_OP_CMP:
cc_v = 0;
case DIF_OP_TST:
case DIF_OP_BA:
case DIF_OP_BE:
if (cc_z)
case DIF_OP_BNE:
if (cc_z == 0)
case DIF_OP_BG:
case DIF_OP_BGU:
case DIF_OP_BGE:
case DIF_OP_BGEU:
if (cc_c == 0)
case DIF_OP_BL:
case DIF_OP_BLU:
if (cc_c)
case DIF_OP_BLE:
case DIF_OP_BLEU:
case DIF_OP_RLDSB:
case DIF_OP_LDSB:
case DIF_OP_RLDSH:
case DIF_OP_LDSH:
case DIF_OP_RLDSW:
case DIF_OP_LDSW:
case DIF_OP_RLDUB:
case DIF_OP_LDUB:
case DIF_OP_RLDUH:
case DIF_OP_LDUH:
case DIF_OP_RLDUW:
case DIF_OP_LDUW:
case DIF_OP_RLDX:
case DIF_OP_LDX:
case DIF_OP_ULDSB:
case DIF_OP_ULDSH:
case DIF_OP_ULDSW:
case DIF_OP_ULDUB:
case DIF_OP_ULDUH:
case DIF_OP_ULDUW:
case DIF_OP_ULDX:
case DIF_OP_RET:
case DIF_OP_NOP:
case DIF_OP_SETX:
case DIF_OP_SETS:
case DIF_OP_SCMP: {
case DIF_OP_LDGA:
case DIF_OP_LDGS:
uintptr_t a;
case DIF_OP_STGS:
*(uint8_t *)a = 0;
a += sizeof (uint64_t);
if (!dtrace_vcanload(
(void *)a, &v->dtdv_type);
case DIF_OP_LDTA:
case DIF_OP_LDLS:
case DIF_OP_STLS:
*(uint8_t *)a = 0;
a += sizeof (uint64_t);
if (!dtrace_vcanload(
(void *)a, &v->dtdv_type);
case DIF_OP_LDTS: {
case DIF_OP_STTS: {
if (!dtrace_vcanload(
case DIF_OP_SRA:
case DIF_OP_CALL:
case DIF_OP_PUSHTR:
case DIF_OP_PUSHTV:
case DIF_OP_POPTS:
if (ttop != 0)
ttop--;
case DIF_OP_FLUSHTS:
ttop = 0;
case DIF_OP_LDGAA:
case DIF_OP_LDTAA: {
case DIF_OP_STGAA:
case DIF_OP_STTAA: {
if (!dtrace_vcanload(
case DIF_OP_ALLOCS: {
case DIF_OP_COPYS:
case DIF_OP_STB:
case DIF_OP_STH:
case DIF_OP_STW:
case DIF_OP_STX:
return (rval);
c[i++] = *msg++;
c[i++] = *str;
c[i++] = *str;
c[i++] = *str;
c[i++] = *str;
c[i++] = *ecbmsg++;
while (shift >= 0) {
debug_enter(c);
dtrace_action_stop(void)
char *sym;
goto out;
for (i = 0; i < nframes; i++) {
out:
if (panic_quiesce) {
int committed = 0;
#ifdef lint
int rval;
case DTRACEACT_STOP:
&mstate))
case DTRACEACT_BREAKPOINT:
case DTRACEACT_PANIC:
case DTRACEACT_STACK:
case DTRACEACT_JSTACK:
case DTRACEACT_USTACK:
valoffs);
case DTRACEACT_SPECULATE: {
if (offs < 0) {
sizeof (dtrace_rechdr_t));
case DTRACEACT_CHILL:
case DTRACEACT_RAISE:
&mstate))
case DTRACEACT_COMMIT:
case DTRACEACT_DISCARD:
case DTRACEACT_DIFEXPR:
case DTRACEACT_LIBACT:
case DTRACEACT_PRINTF:
case DTRACEACT_PRINTA:
case DTRACEACT_SYSTEM:
case DTRACEACT_FREOPEN:
case DTRACEACT_TRACEMEM:
case DTRACEACT_SYM:
case DTRACEACT_MOD:
case DTRACEACT_USYM:
case DTRACEACT_UMOD:
case DTRACEACT_UADDR: {
case DTRACEACT_EXIT: {
ASSERT(0);
if (tracememsize != 0 &&
tracememsize = 0;
size_t s;
for (s = 0; s < size; s++) {
valoffs++, c);
switch (size) {
case sizeof (uint8_t):
case sizeof (uint16_t):
case sizeof (uint32_t):
case sizeof (uint64_t):
ASSERT(0);
int ndx;
if (vtime) {
if (!committed)
if (vtime)
static uint_t
dtrace_hash_str(char *p)
hval &= ~g;
return (hval);
static dtrace_hash_t *
return (hash);
#ifdef DEBUG
for (i = 0; i < size; i++) {
goto add;
add:
static dtrace_probe_t *
return (NULL);
return (NULL);
if (b == bucket) {
b = b->dthb_next;
return (new);
#define DTRACE_ISALPHA(c) \
dtrace_badname(const char *s)
priv = 0;
#ifdef DTRACE_ERRDEBUG
int occupied = 0;
goto out;
goto out;
out:
DTRACE_PRIV_KERNEL)) == 0)
int rv;
return (rv);
return (rv);
return (rv);
return (rv);
return (rv);
const char *olds;
char s1, c;
int gs;
if (s == NULL)
top:
olds = s;
s1 = *s++;
if (p == NULL)
if (notflag) {
ok++;
ok++;
if (notflag) {
if (s1 != c)
ok++;
} else if (s1 == c)
ok++;
if (ok)
goto top;
if (c != s1)
goto top;
return (gs);
dtrace_id_t i;
return (DTRACE_MATCH_FAIL);
nmatched++;
return (nmatched);
for (i = 0; i < dtrace_nprobes; i++) {
zoneid) <= 0)
nmatched++;
return (DTRACE_MATCH_FAIL);
return (nmatched);
nmatched++;
return (DTRACE_MATCH_FAIL);
return (nmatched);
static dtrace_probekey_f *
dtrace_probekey_func(const char *p)
return (&dtrace_match_nul);
return (&dtrace_match_glob);
return (&dtrace_match_string);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EBUSY);
if (!self) {
return (EBUSY);
for (i = 0; i < dtrace_nprobes; i++) {
if (!self) {
if (noreap)
return (EBUSY);
return (EAGAIN);
for (i = 0; i < dtrace_nprobes; i++) {
dtrace_sync();
if (!self) {
dtrace_attached(void)
for (i = 0; i < dtrace_nprobes; i++) {
if (nsize == 0) {
dtrace_sync();
return (id);
static dtrace_probe_t *
return (NULL);
return (DTRACE_MATCH_DONE);
int match;
return (rval);
int all = 0;
for (i = 0; i < dtrace_nprobes; i++) {
enab));
char *strtab;
void *parg;
for (i = 0; i < nprobes; i++) {
char *strtab;
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EBUSY);
if (dtrace_err_verbose) {
#ifdef DTRACE_ERRDEBUG
int err = 0, i;
int kcheckload;
switch (op) {
case DIF_OP_OR:
case DIF_OP_XOR:
case DIF_OP_AND:
case DIF_OP_SLL:
case DIF_OP_SRL:
case DIF_OP_SRA:
case DIF_OP_SUB:
case DIF_OP_ADD:
case DIF_OP_MUL:
case DIF_OP_SDIV:
case DIF_OP_UDIV:
case DIF_OP_SREM:
case DIF_OP_UREM:
case DIF_OP_COPYS:
if (rd == 0)
case DIF_OP_NOT:
case DIF_OP_MOV:
case DIF_OP_ALLOCS:
if (r2 != 0)
if (rd == 0)
case DIF_OP_LDSB:
case DIF_OP_LDSH:
case DIF_OP_LDSW:
case DIF_OP_LDUB:
case DIF_OP_LDUH:
case DIF_OP_LDUW:
case DIF_OP_LDX:
if (r2 != 0)
if (rd == 0)
if (kcheckload)
case DIF_OP_RLDSB:
case DIF_OP_RLDSH:
case DIF_OP_RLDSW:
case DIF_OP_RLDUB:
case DIF_OP_RLDUH:
case DIF_OP_RLDUW:
case DIF_OP_RLDX:
if (r2 != 0)
if (rd == 0)
case DIF_OP_ULDSB:
case DIF_OP_ULDSH:
case DIF_OP_ULDSW:
case DIF_OP_ULDUB:
case DIF_OP_ULDUH:
case DIF_OP_ULDUW:
case DIF_OP_ULDX:
if (r2 != 0)
if (rd == 0)
case DIF_OP_STB:
case DIF_OP_STH:
case DIF_OP_STW:
case DIF_OP_STX:
if (r2 != 0)
if (rd == 0)
case DIF_OP_CMP:
case DIF_OP_SCMP:
if (rd != 0)
case DIF_OP_TST:
case DIF_OP_BA:
case DIF_OP_BE:
case DIF_OP_BNE:
case DIF_OP_BG:
case DIF_OP_BGU:
case DIF_OP_BGE:
case DIF_OP_BGEU:
case DIF_OP_BL:
case DIF_OP_BLU:
case DIF_OP_BLE:
case DIF_OP_BLEU:
label);
label);
case DIF_OP_RET:
case DIF_OP_NOP:
case DIF_OP_POPTS:
case DIF_OP_FLUSHTS:
case DIF_OP_SETX:
if (rd == 0)
case DIF_OP_SETS:
if (rd == 0)
case DIF_OP_LDGA:
case DIF_OP_LDTA:
if (rd == 0)
case DIF_OP_LDGS:
case DIF_OP_LDTS:
case DIF_OP_LDLS:
case DIF_OP_LDGAA:
case DIF_OP_LDTAA:
if (rd == 0)
case DIF_OP_STGS:
case DIF_OP_STTS:
case DIF_OP_STLS:
case DIF_OP_STGAA:
case DIF_OP_STTAA:
case DIF_OP_CALL:
if (rd == 0)
case DIF_OP_PUSHTR:
case DIF_OP_PUSHTV:
case sizeof (uint8_t):
case sizeof (uint16_t):
case sizeof (uint32_t):
case sizeof (uint64_t):
v->dtdv_scope);
v->dtdv_kind);
switch (v->dtdv_scope) {
case DIFV_SCOPE_GLOBAL:
case DIFV_SCOPE_THREAD:
case DIFV_SCOPE_LOCAL:
return (err);
int err = 0;
switch (op) {
case DIF_OP_OR:
case DIF_OP_XOR:
case DIF_OP_AND:
case DIF_OP_SLL:
case DIF_OP_SRL:
case DIF_OP_SRA:
case DIF_OP_SUB:
case DIF_OP_ADD:
case DIF_OP_MUL:
case DIF_OP_SDIV:
case DIF_OP_UDIV:
case DIF_OP_SREM:
case DIF_OP_UREM:
case DIF_OP_COPYS:
case DIF_OP_NOT:
case DIF_OP_MOV:
case DIF_OP_RLDSB:
case DIF_OP_RLDSH:
case DIF_OP_RLDSW:
case DIF_OP_RLDUB:
case DIF_OP_RLDUH:
case DIF_OP_RLDUW:
case DIF_OP_RLDX:
case DIF_OP_ULDSB:
case DIF_OP_ULDSH:
case DIF_OP_ULDSW:
case DIF_OP_ULDUB:
case DIF_OP_ULDUH:
case DIF_OP_ULDUW:
case DIF_OP_ULDX:
case DIF_OP_STB:
case DIF_OP_STH:
case DIF_OP_STW:
case DIF_OP_STX:
case DIF_OP_ALLOCS:
case DIF_OP_CMP:
case DIF_OP_SCMP:
case DIF_OP_TST:
case DIF_OP_BA:
case DIF_OP_BE:
case DIF_OP_BNE:
case DIF_OP_BG:
case DIF_OP_BGU:
case DIF_OP_BGE:
case DIF_OP_BGEU:
case DIF_OP_BL:
case DIF_OP_BLU:
case DIF_OP_BLE:
case DIF_OP_BLEU:
case DIF_OP_RET:
case DIF_OP_NOP:
case DIF_OP_POPTS:
case DIF_OP_FLUSHTS:
case DIF_OP_SETX:
case DIF_OP_SETS:
case DIF_OP_LDGA:
case DIF_OP_LDLS:
case DIF_OP_STGS:
case DIF_OP_STLS:
case DIF_OP_PUSHTR:
case DIF_OP_PUSHTV:
case DIF_OP_LDGS:
if (v >= DIF_VAR_OTHER_UBASE)
case DIF_OP_LDTA:
case DIF_OP_LDTS:
case DIF_OP_LDGAA:
case DIF_OP_LDTAA:
case DIF_OP_STTS:
case DIF_OP_STGAA:
case DIF_OP_STTAA:
case DIF_OP_CALL:
return (err);
switch (v->dtdv_id) {
case DIF_VAR_CURTHREAD:
case DIF_VAR_PID:
case DIF_VAR_TID:
case DIF_VAR_EXECNAME:
case DIF_VAR_ZONENAME:
if (dtrace_vtime_references++ == 0)
switch (op) {
case DIF_OP_SETX:
case DIF_OP_STTS:
case DIF_OP_STGAA:
case DIF_OP_STTAA:
case DIF_OP_PUSHTR:
if (srd == 0)
case DIF_OP_PUSHTV:
case DIF_OP_FLUSHTS:
ttop = 0;
case DIF_OP_POPTS:
if (ttop != 0)
ttop--;
sval = 0;
srd = 0;
if (nkeys == 0)
int *np;
switch (scope) {
case DIFV_SCOPE_THREAD:
if (osz != 0) {
case DIFV_SCOPE_LOCAL:
sizeof (uint64_t));
case DIFV_SCOPE_GLOBAL:
sizeof (uint64_t);
ASSERT(0);
if (oldsize != 0) {
static dtrace_difo_t *
return (new);
int *np;
switch (scope) {
case DIFV_SCOPE_THREAD:
case DIFV_SCOPE_LOCAL:
case DIFV_SCOPE_GLOBAL:
ASSERT(0);
if (--dtrace_vtime_references == 0)
static uint16_t
char *fmt;
static dtrace_predicate_t *
return (pred);
return (pred);
return (pred);
static dtrace_actdesc_t *
return (act);
static dtrace_ecb_t *
if (necbs == 0) {
dtrace_sync();
return (ecb);
dtrace_sync();
sizeof (uint64_t)));
static dtrace_action_t *
case DTRACEAGG_MIN:
case DTRACEAGG_MAX:
case DTRACEAGG_COUNT:
case DTRACEAGG_QUANTIZE:
sizeof (uint64_t);
case DTRACEAGG_LQUANTIZE: {
goto err;
case DTRACEAGG_LLQUANTIZE: {
int64_t v;
goto err;
goto err;
case DTRACEAGG_AVG:
case DTRACEAGG_STDDEV:
case DTRACEAGG_SUM:
goto err;
if (ntuple == 0)
goto err;
if (--ntuple == 0) {
goto success;
err:
return (NULL);
if (naggs == 0) {
return (EINVAL);
return (EINVAL);
return (EINVAL);
case DTRACEACT_PRINTF:
case DTRACEACT_PRINTA:
case DTRACEACT_SYSTEM:
case DTRACEACT_FREOPEN:
case DTRACEACT_DIFEXPR:
format = 0;
case DTRACEACT_LIBACT:
case DTRACEACT_TRACEMEM:
return (EINVAL);
return (EINVAL);
case DTRACEACT_STACK:
case DTRACEACT_JSTACK:
case DTRACEACT_USTACK:
case DTRACEACT_SYM:
case DTRACEACT_MOD:
sizeof (uint64_t)) ||
return (EINVAL);
case DTRACEACT_USYM:
case DTRACEACT_UMOD:
case DTRACEACT_UADDR:
return (EINVAL);
case DTRACEACT_STOP:
case DTRACEACT_BREAKPOINT:
case DTRACEACT_PANIC:
case DTRACEACT_CHILL:
case DTRACEACT_DISCARD:
case DTRACEACT_RAISE:
return (EINVAL);
case DTRACEACT_EXIT:
return (EINVAL);
case DTRACEACT_SPECULATE:
return (EINVAL);
return (EINVAL);
case DTRACEACT_COMMIT: {
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
dtrace_sync();
dtrace_sync();
if (p != NULL)
static dtrace_ecb_t *
return (ecb);
return (NULL);
return (DTRACE_MATCH_NEXT);
return (DTRACE_MATCH_DONE);
return (DTRACE_MATCH_FAIL);
return (DTRACE_MATCH_NEXT);
static dtrace_ecb_t *
return (NULL);
static dtrace_aggregation_t *
return (NULL);
return (EFBIG);
goto err;
goto err;
err:
allocated++;
allocated++;
return (ENOMEM);
static intptr_t
return (offs);
return (offs);
goto out;
woffs = 0;
woffs = 0;
offs = 0;
goto out;
if (offs == 0) {
woffs = 0;
out:
return (offs);
return (offs);
for (i = 0; i < NCPU; i++) {
for (i = 0; i < NCPU; i++) {
static dtrace_enabling_t *
return (enab);
return (ENOSPC);
return (err);
return (EBUSY);
dtrace_enabling_matchall(void)
int i, all = 0;
goto retry;
dtrace_enabling_reap(void)
for (i = 0; i < dtrace_nprobes; i++) {
if (dtrace_err_verbose)
#ifdef DTRACE_ERRDEBUG
static dof_hdr_t *
for (i = 0; i < DTRACEOPT_MAX; i++) {
return (dof);
static dof_hdr_t *
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (dof);
static dof_hdr_t *
unsigned int len, i;
return (NULL);
for (i = 0; i < len; i++)
return (NULL);
return (NULL);
return (NULL);
return (dof);
static dof_sec_t *
return (NULL);
return (NULL);
return (NULL);
return (sec);
static dtrace_probedesc_t *
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (desc);
static dtrace_difo_t *
int section;
int bufoffs;
int lenoffs;
int entsize;
int align;
const char *msg;
} difo[] = {
return (NULL);
return (NULL);
return (NULL);
void **bufp;
goto err;
goto err;
goto err;
goto err;
goto err;
goto err;
goto err;
goto err;
goto err;
return (dp);
err:
return (NULL);
static dtrace_predicate_t *
return (NULL);
static dtrace_actdesc_t *
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
uint64_t i;
goto err;
goto err;
goto err;
arg = 0;
goto err;
goto err;
return (first);
err:
return (NULL);
static dtrace_ecbdesc_t *
return (NULL);
return (NULL);
return (NULL);
goto err;
goto err;
goto err;
goto err;
goto err;
return (ep);
err:
return (NULL);
dof_relodesc_t *r;
uint_t i, n;
switch (r->dofr_type) {
case DOF_RELO_NONE:
case DOF_RELO_SETX:
uint_t i;
if (noprobes) {
case DOF_SECT_PROVIDER:
case DOF_SECT_PROBES:
case DOF_SECT_PRARGS:
case DOF_SECT_PROFFS:
case DOF_SECT_URELHDR:
int i, rval;
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (rval);
void *base;
return (ENOMEM);
hashsize--;
for (i = 0; i < hashsize; i++)
for (i = 0; i < NCPU; i++) {
if (maxper == 0)
sizeof (dtrace_statvar_t *));
sizeof (dtrace_difv_t));
sizeof (dtrace_statvar_t *));
dtrace_sync();
return (NULL);
for (i = 0; i < DTRACEOPT_MAX; i++)
return (state);
return (E2BIG);
return (rval);
return (rval);
return (ENOMEM);
int rval, i;
DTRACEOPT_BUFSIZE)) != 0)
return (rval);
DTRACEOPT_AGGSIZE)) != 0)
return (rval);
return (rval);
goto out;
goto out;
goto out;
goto out;
for (i = 0; i < nspec; i++) {
goto err;
goto out;
goto out;
goto out;
goto out;
goto err;
if (rval == 0)
goto err;
if (rval != 0)
goto err;
if (dtrace_getf++ == 0) {
goto out;
err:
goto out;
out:
return (rval);
return (EINVAL);
dtrace_sync();
dtrace_sync();
dtrace_sync();
if (--dtrace_getf == 0)
return (EBUSY);
return (EINVAL);
return (EINVAL);
switch (option) {
case DTRACEOPT_DESTRUCTIVE:
return (EACCES);
case DTRACEOPT_BUFSIZE:
case DTRACEOPT_DYNVARSIZE:
case DTRACEOPT_AGGSIZE:
case DTRACEOPT_SPECSIZE:
case DTRACEOPT_STRSIZE:
if (val < 0)
return (EINVAL);
dtrace_sync();
if (!match)
dtrace_sync();
for (i = 0; i < nspec; i++)
#ifdef DEBUG
for (i = 0; i < nspec; i++)
static dtrace_state_t *
dtrace_anon_grab(void)
return (NULL);
return (state);
dtrace_anon_property(void)
int i, rv;
dtrace_err_verbose = 0;
if (rv == 0)
dtrace_err_verbose = 0;
if (rv != 0) {
int rval;
if (!dtrace_helptrace_enabled)
next = 0;
static uint64_t
if (trace)
goto next;
goto err;
if (trace)
goto err;
next:
if (trace)
if (trace)
return (rval);
err:
if (trace)
return (NULL);
return (EINVAL);
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
last = h;
p->p_pid);
int err = 0, i;
return (err == 0);
return (EINVAL);
count++;
return (ENOSPC);
goto err;
goto err;
nactions++;
goto err;
err:
return (EINVAL);
p->p_pid);
return (ENOSPC);
return (EALREADY);
sizeof (dtrace_helper_provider_t *));
sizeof (dtrace_helper_provider_t *));
sizeof (dof_provider_t))) {
for (j = 0; j < nprobes; j++) {
return (rv);
nprovs++;
ep)) != 0) {
nhelpers++;
destroy = 0;
if (destroy)
return (gen);
static dtrace_helpers_t *
return (help);
dtrace_helpers_destroy(void)
dtrace_sync();
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
h = next;
sizeof (dtrace_helper_provider_t *));
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
KM_SLEEP);
if (hasprovs)
if (dtrace_err_verbose) {
dtrace_sync();
dtrace_suspend(void)
dtrace_resume(void)
switch (what) {
case CPU_CONFIG: {
case CPU_UNCONFIG:
if (osize == 0) {
dtrace_sync();
sizeof (dtrace_state_t), 0) != 0) {
return (DDI_FAILURE);
return (DDI_FAILURE);
if (dtrace_helptrace_enabled) {
return (DDI_SUCCESS);
return (ENXIO);
return (EACCES);
dtrace_opens++;
dtrace_opens--;
return (EBUSY);
return (EAGAIN);
int rval;
switch (cmd) {
case DTRACEHIOC_ADDDOF:
return (EFAULT);
case DTRACEHIOC_ADD: {
return (rval);
rval = 0;
return (rval);
case DTRACEHIOC_REMOVE: {
return (rval);
return (ENOTTY);
int rval;
switch (cmd) {
case DTRACEIOC_PROVIDER: {
return (EFAULT);
return (ESRCH);
return (EFAULT);
case DTRACEIOC_EPROBE: {
void *buf;
int nrecs;
return (EFAULT);
return (EINVAL);
return (EINVAL);
if (nrecs-- == 0)
sizeof (dtrace_recdesc_t));
return (EFAULT);
case DTRACEIOC_AGGDESC: {
int nrecs;
void *buf;
return (EFAULT);
return (EINVAL);
if (nrecs-- == 0)
return (EFAULT);
case DTRACEIOC_ENABLE: {
int err = 0;
*rv = 0;
return (rval);
return (EBUSY);
return (EINVAL);
return (rval);
return (err);
case DTRACEIOC_REPLICATE: {
int err;
return (EFAULT);
return (err);
case DTRACEIOC_PROBEMATCH:
case DTRACEIOC_PROBES: {
dtrace_id_t i;
return (EFAULT);
return (EINVAL);
return (ESRCH);
return (EFAULT);
case DTRACEIOC_PROBEARG: {
return (EFAULT);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (EFAULT);
case DTRACEIOC_GO: {
if (rval != 0)
return (rval);
return (EFAULT);
case DTRACEIOC_STOP: {
if (rval != 0)
return (rval);
return (EFAULT);
case DTRACEIOC_DOFGET: {
return (EFAULT);
case DTRACEIOC_AGGSNAP:
case DTRACEIOC_BUFSNAP: {
return (EFAULT);
return (EINVAL);
return (EBUSY);
return (EFAULT);
return (EFAULT);
return (EFAULT);
return (ENOENT);
return (ENOENT);
return (EFAULT);
return (EFAULT);
case DTRACEIOC_CONF: {
return (EFAULT);
case DTRACEIOC_STATUS: {
return (ENOENT);
for (i = 0; i < NCPU; i++) {
return (EFAULT);
case DTRACEIOC_FORMAT: {
char *str;
int len;
return (EFAULT);
return (EINVAL);
return (EINVAL);
return (EINVAL);
return (ENOTTY);
switch (cmd) {
case DDI_DETACH:
case DDI_SUSPEND:
return (DDI_SUCCESS);
return (DDI_FAILURE);
if (dtrace_helpers > 0) {
return (DDI_FAILURE);
return (DDI_FAILURE);
if (dtrace_helptrace_enabled) {
dtrace_nprobes = 0;
dtrace_toxranges = 0;
dtrace_toxranges_max = 0;
return (DDI_SUCCESS);
int error;
switch (infocmd) {
case DDI_INFO_DEVT2DEVINFO:
case DDI_INFO_DEVT2INSTANCE:
*result = (void *)0;
return (error);
(void *)&modldrv,
_init(void)
_fini(void)