dtrace_impl.h revision 0b38a8bdfd75ac6144f9d462bb38d0c1b3f0ca50
6185db853e024a486ff8837e6784dd290d866112dougm * CDDL HEADER START
6185db853e024a486ff8837e6784dd290d866112dougm * The contents of this file are subject to the terms of the
6185db853e024a486ff8837e6784dd290d866112dougm * Common Development and Distribution License, Version 1.0 only
6185db853e024a486ff8837e6784dd290d866112dougm * (the "License"). You may not use this file except in compliance
6185db853e024a486ff8837e6784dd290d866112dougm * with the License.
6185db853e024a486ff8837e6784dd290d866112dougm * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
6185db853e024a486ff8837e6784dd290d866112dougm * See the License for the specific language governing permissions
6185db853e024a486ff8837e6784dd290d866112dougm * and limitations under the License.
6185db853e024a486ff8837e6784dd290d866112dougm * When distributing Covered Code, include this CDDL HEADER in each
6185db853e024a486ff8837e6784dd290d866112dougm * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
6185db853e024a486ff8837e6784dd290d866112dougm * If applicable, add the following below this CDDL HEADER, with the
6185db853e024a486ff8837e6784dd290d866112dougm * fields enclosed by brackets "[]" replaced with your own identifying
6185db853e024a486ff8837e6784dd290d866112dougm * information: Portions Copyright [yyyy] [name of copyright owner]
6185db853e024a486ff8837e6784dd290d866112dougm * CDDL HEADER END
dc20a3024900c47dd2ee44b9707e6df38f7d62a5as * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
6185db853e024a486ff8837e6784dd290d866112dougm * Use is subject to license terms.
6185db853e024a486ff8837e6784dd290d866112dougm#pragma ident "%Z%%M% %I% %E% SMI"
6185db853e024a486ff8837e6784dd290d866112dougmextern "C" {
6185db853e024a486ff8837e6784dd290d866112dougm * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces
6185db853e024a486ff8837e6784dd290d866112dougm * Note: The contents of this file are private to the implementation of the
6185db853e024a486ff8837e6784dd290d866112dougm * Solaris system and DTrace subsystem and are subject to change at any time
6185db853e024a486ff8837e6784dd290d866112dougm * without notice. Applications and drivers using these interfaces will fail
6185db853e024a486ff8837e6784dd290d866112dougm * to run on future releases. These interfaces should not be used for any
6185db853e024a486ff8837e6784dd290d866112dougm * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB).
6185db853e024a486ff8837e6784dd290d866112dougm * Please refer to the "Solaris Dynamic Tracing Guide" for more information.
6185db853e024a486ff8837e6784dd290d866112dougm * DTrace Implementation Constants and Typedefs
6185db853e024a486ff8837e6784dd290d866112dougm * DTrace Probes
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * The probe is the fundamental unit of the DTrace architecture. Probes are
6185db853e024a486ff8837e6784dd290d866112dougm * created by DTrace providers, and managed by the DTrace framework. A probe
6185db853e024a486ff8837e6784dd290d866112dougm * is identified by a unique <provider, module, function, name> tuple, and has
6185db853e024a486ff8837e6784dd290d866112dougm * a unique probe identifier assigned to it. (Some probes are not associated
6185db853e024a486ff8837e6784dd290d866112dougm * with a specific point in text; these are called _unanchored probes_ and have
6185db853e024a486ff8837e6784dd290d866112dougm * no module or function associated with them.) Probes are represented as a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_probe structure. To allow quick lookups based on each element of the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * probe tuple, probes are hashed by each of provider, module, function and
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * name. (If a lookup is performed based on a regular expression, a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_probekey is prepared, and a linear search is performed.) Each probe
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * is additionally pointed to by a linear array indexed by its identifier. The
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * identifier is the provider's mechanism for indicating to the DTrace
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * framework that a probe has fired: the identifier is passed as the first
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * argument to dtrace_probe(), where it is then mapped into the corresponding
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_probe structure. From the dtrace_probe structure, dtrace_probe() can
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * iterate over the probe's list of enabling control blocks; see "DTrace
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Enabling Control Blocks", below.)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_cacheid_t dtpr_predcache; /* predicate cache ID */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_provider_t *dtpr_provider; /* pointer to provider */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_probe_t *dtpr_prevmod; /* previous in module hash */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_probe_t *dtpr_nextfunc; /* next in function hash */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_probe_t *dtpr_prevfunc; /* previous in function hash */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_probe_t *dtpr_prevname; /* previous in name hash */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef int dtrace_probekey_f(const char *, const char *, int);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_probekey {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_probekey_f *dtpk_pmatch; /* provider matching function */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_probekey_f *dtpk_mmatch; /* module matching function */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_probekey_f *dtpk_fmatch; /* func matching function */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_probekey_f *dtpk_nmatch; /* name matching function */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_hashbucket {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw struct dtrace_hashbucket *dthb_next; /* next on hash chain */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_hash {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTrace Enabling Control Blocks
55bf511df53aad0fdb7eb3fa349f0308cc05234cas * When a provider wishes to fire a probe, it calls into dtrace_probe(),
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * passing the probe identifier as the first argument. As described above,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * structure. This structure contains information about the probe, and a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * pointer to the list of Enabling Control Blocks (ECBs). Each ECB points to
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTrace consumer state, and contains an optional predicate, and a list of
55bf511df53aad0fdb7eb3fa349f0308cc05234cas * actions. (Shown schematically below.) The ECB abstraction allows a single
55bf511df53aad0fdb7eb3fa349f0308cc05234cas * probe to be multiplexed across disjoint consumers, or across disjoint
55bf511df53aad0fdb7eb3fa349f0308cc05234cas * enablings of a single probe within one consumer.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Enabling Control Block
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_ecb_t
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +------------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_state_t * ------+--------------> State associated with this ECB
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_predicate_t * --+---------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_action_t * -----+----+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_ecb_t * ---+ | | | Predicate (if any)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +-------------------+----+ | | dtrace_predicate_t
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | | +---> +--------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | | | dtrace_difo_t * ---+----> DIFO
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | | +--------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Next ECB | | Action
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * (if any) | | dtrace_action_t
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * : +--> +-------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * : | dtrace_actkind_t -+------> kind
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * v | dtrace_difo_t * --+------> DIFO (if any)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_recdesc_t -+------> record descr.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_action_t * +------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +-------------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | Next action
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +-------------------------------+ (if any)
55bf511df53aad0fdb7eb3fa349f0308cc05234cas * | Action
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_action_t
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +--> +-------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_actkind_t -+------> kind
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_difo_t * --+------> DIFO (if any)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | dtrace_action_t * +------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +-------------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | Next action
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +-------------------------------+ (if any)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_probe() iterates over the ECB list. If the ECB needs less space
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * than is available in the principal buffer, the ECB is processed: if the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * predicate is non-NULL, the DIF object is executed. If the result is
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * non-zero, the action list is processed, with each action being executed
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * accordingly. When the action list has been completely executed, processing
55bf511df53aad0fdb7eb3fa349f0308cc05234cas * advances to the next ECB. processing advances to the next ECB. If the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * result is non-zero; For each ECB, it first determines the The ECB
55bf511df53aad0fdb7eb3fa349f0308cc05234cas * abstraction allows disjoint consumers to multiplex on single probes.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_predicate_t *dte_predicate; /* predicate, if any */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_action_t *dte_action_last; /* last action on ECB */
573b0c00a1ee520c3f6938dda8d693236f37ae60dougmtypedef struct dtrace_aggregation {
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm dtrace_action_t dtag_action; /* action; must be first */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_action_t *dtag_first; /* first action in tuple */
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * DTrace Buffers
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Principal buffers, aggregation buffers, and speculative buffers are all
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * managed with the dtrace_buffer structure. By default, this structure
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * active and passive buffers, respectively. For speculative buffers,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * to a scratch buffer. For all buffer types, the dtrace_buffer structure is
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * always allocated on a per-CPU basis; a single dtrace_buffer structure is
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * never shared among CPUs. (That is, there is never true sharing of the
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * dtrace_buffer structure; to prevent false sharing of the structure, it must
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * always be aligned to the coherence granularity -- generally 64 bytes.)
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * One of the critical design decisions of DTrace is that a given ECB always
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * stores the same quantity and type of data. This is done to assure that the
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * only metadata required for an ECB's traced data is the EPID. That is, from
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * the EPID, the consumer can determine the data layout. (The data buffer
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * layout is shown schematically below.) By assuring that one can determine
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * data layout from the EPID, the metadata stream can be separated from the
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * data stream -- simplifying the data stream enormously.
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * base of data buffer ---> +------+--------------------+------+
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * | EPID | data | EPID |
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * +------+--------+------+----+------+
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * | data | EPID | data |
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * +---------------+------+-----------+
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * | data, cont. |
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * +------+--------------------+------+
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * | EPID | data | |
573b0c00a1ee520c3f6938dda8d693236f37ae60dougm * +------+--------------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * limit of data buffer ---> +----------------------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * principal buffer (both scratch and payload) exceed the available space. If
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * the ECB's needs exceed available space (and if the principal buffer policy
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * is the default "switch" policy), the ECB is dropped, the buffer's drop count
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * is incremented, and processing advances to the next ECB. If the ECB's needs
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * can be met with the available space, the ECB is processed, but the offset in
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * the principal buffer is only advanced if the ECB completes processing
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * without error.
6185db853e024a486ff8837e6784dd290d866112dougm * When a buffer is to be switched (either because the buffer is the principal
6185db853e024a486ff8837e6784dd290d866112dougm * buffer with a "switch" policy or because it is an aggregation buffer), a
6185db853e024a486ff8837e6784dd290d866112dougm * cross call is issued to the CPU associated with the buffer. In the cross
6185db853e024a486ff8837e6784dd290d866112dougm * call context, interrupts are disabled, and the active and the inactive
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * buffers are atomically switched. This involves switching the data pointers,
6185db853e024a486ff8837e6784dd290d866112dougm * copying the various state fields (offset, drops, errors, etc.) into their
6185db853e024a486ff8837e6784dd290d866112dougm * inactive equivalents, and clearing the state fields. Because interrupts are
6185db853e024a486ff8837e6784dd290d866112dougm * disabled during this procedure, the switch is guaranteed to appear atomic to
6185db853e024a486ff8837e6784dd290d866112dougm * dtrace_probe().
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Ring Buffering
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * To process a ring buffer correctly, one must know the oldest valid record.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Processing starts at the oldest record in the buffer and continues until
6185db853e024a486ff8837e6784dd290d866112dougm * the end of the buffer is reached. Processing then resumes starting with
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * the record stored at offset 0 in the buffer, and continues until the
6185db853e024a486ff8837e6784dd290d866112dougm * youngest record is processed. If trace records are of a fixed-length,
6185db853e024a486ff8837e6784dd290d866112dougm * determining the oldest record is trivial:
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * - If the ring buffer has not wrapped, the oldest record is the record
6185db853e024a486ff8837e6784dd290d866112dougm * stored at offset 0.
6185db853e024a486ff8837e6784dd290d866112dougm * - If the ring buffer has wrapped, the oldest record is the record stored
6185db853e024a486ff8837e6784dd290d866112dougm * at the current offset.
6185db853e024a486ff8837e6784dd290d866112dougm * With variable length records, however, just knowing the current offset
6185db853e024a486ff8837e6784dd290d866112dougm * doesn't suffice for determining the oldest valid record: assuming that one
6185db853e024a486ff8837e6784dd290d866112dougm * allows for arbitrary data, one has no way of searching forward from the
6185db853e024a486ff8837e6784dd290d866112dougm * current offset to find the oldest valid record. (That is, one has no way
6185db853e024a486ff8837e6784dd290d866112dougm * of separating data from metadata.) It would be possible to simply refuse to
6185db853e024a486ff8837e6784dd290d866112dougm * process any data in the ring buffer between the current offset and the
6185db853e024a486ff8837e6784dd290d866112dougm * limit, but this leaves (potentially) an enormous amount of otherwise valid
6185db853e024a486ff8837e6784dd290d866112dougm * data unprocessed.
6185db853e024a486ff8837e6784dd290d866112dougm * To effect ring buffering, we track two offsets in the buffer: the current
6185db853e024a486ff8837e6784dd290d866112dougm * offset and the _wrapped_ offset. If a request is made to reserve some
6185db853e024a486ff8837e6784dd290d866112dougm * amount of data, and the buffer has wrapped, the wrapped offset is
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * incremented until the wrapped offset minus the current offset is greater
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * than or equal to the reserve request. This is done by repeatedly looking
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * up the ECB corresponding to the EPID at the current wrapped offset, and
6185db853e024a486ff8837e6784dd290d866112dougm * incrementing the wrapped offset by the size of the data payload
6185db853e024a486ff8837e6784dd290d866112dougm * corresponding to that ECB. If this offset is greater than or equal to the
6185db853e024a486ff8837e6784dd290d866112dougm * limit of the data buffer, the wrapped offset is set to 0. Thus, the
6185db853e024a486ff8837e6784dd290d866112dougm * current offset effectively "chases" the wrapped offset around the buffer.
6185db853e024a486ff8837e6784dd290d866112dougm * Schematically:
6185db853e024a486ff8837e6784dd290d866112dougm * base of data buffer ---> +------+--------------------+------+
6185db853e024a486ff8837e6784dd290d866112dougm * | EPID | data | EPID |
6185db853e024a486ff8837e6784dd290d866112dougm * +------+--------+------+----+------+
6185db853e024a486ff8837e6784dd290d866112dougm * | data | EPID | data |
6185db853e024a486ff8837e6784dd290d866112dougm * +---------------+------+-----------+
6185db853e024a486ff8837e6784dd290d866112dougm * | data, cont. |
6185db853e024a486ff8837e6784dd290d866112dougm * +------+---------------------------+
6185db853e024a486ff8837e6784dd290d866112dougm * | EPID | data |
6185db853e024a486ff8837e6784dd290d866112dougm * current offset ---> +------+---------------------------+
6185db853e024a486ff8837e6784dd290d866112dougm * | invalid data |
6185db853e024a486ff8837e6784dd290d866112dougm * wrapped offset ---> +------+--------------------+------+
6185db853e024a486ff8837e6784dd290d866112dougm * | EPID | data | EPID |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * +------+--------+------+----+------+
6185db853e024a486ff8837e6784dd290d866112dougm * | data | EPID | data |
6185db853e024a486ff8837e6784dd290d866112dougm * +---------------+------+-----------+
6185db853e024a486ff8837e6784dd290d866112dougm * . ... valid data ... .
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * +------+-------------+------+------+
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | EPID | data | EPID | data |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * +------+------------++------+------+
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | data, cont. | leftover |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * limit of data buffer ---> +-------------------+--------------+
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * If the amount of requested buffer space exceeds the amount of space
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * available between the current offset and the end of the buffer:
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * (1) all words in the data buffer between the current offset and the limit
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * of the data buffer (marked "leftover", above) are set to
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTRACE_EPIDNONE
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * (2) the wrapped offset is set to zero
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * (3) the iteration process described above occurs until the wrapped offset
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * is greater than the amount of desired space.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * The wrapped offset is implemented by (re-)using the inactive offset.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * In a "switch" buffer policy, the inactive offset stores the offset in
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * the inactive buffer; in a "ring" buffer policy, it stores the wrapped
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Scratch Buffering
6185db853e024a486ff8837e6784dd290d866112dougm * Some ECBs may wish to allocate dynamically-sized temporary scratch memory.
6185db853e024a486ff8837e6784dd290d866112dougm * To accommodate such requests easily, scratch memory may be allocated in
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * the buffer beyond the current offset plus the needed memory of the current
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * ECB. If there isn't sufficient room in the buffer for the requested amount
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * of scratch space, the allocation fails and an error is generated. Scratch
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * memory is tracked in the dtrace_mstate_t and is automatically freed when
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * the ECB ceases processing. Note that ring buffers cannot allocate their
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * scratch from the principal buffer -- lest they needlessly overwrite older,
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * valid data. Ring buffers therefore have their own dedicated scratch buffer
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * from which scratch is allocated.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#define DTRACEBUF_RING 0x0001 /* bufpolicy set to "ring" */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#define DTRACEBUF_FILL 0x0002 /* bufpolicy set to "fill" */
6185db853e024a486ff8837e6784dd290d866112dougm#define DTRACEBUF_NOSWITCH 0x0004 /* do not switch buffer */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#define DTRACEBUF_WRAPPED 0x0008 /* ring buffer has wrapped */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#define DTRACEBUF_FULL 0x0040 /* "fill" buffer is full */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#define DTRACEBUF_CONSUMED 0x0080 /* buffer has been consumed */
6185db853e024a486ff8837e6784dd290d866112dougm#define DTRACEBUF_INACTIVE 0x0100 /* buffer is not yet active */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmtypedef struct dtrace_buffer {
6185db853e024a486ff8837e6784dd290d866112dougm uint32_t dtb_xamot_drops; /* drops in inactive buffer */
6185db853e024a486ff8837e6784dd290d866112dougm uint64_t dtb_xamot_offset; /* offset in inactive buffer */
6185db853e024a486ff8837e6784dd290d866112dougm uint32_t dtb_xamot_errors; /* errors in inactive buffer */
6185db853e024a486ff8837e6784dd290d866112dougm * DTrace Aggregation Buffers
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * Aggregation buffers use much of the same mechanism as described above
6185db853e024a486ff8837e6784dd290d866112dougm * ("DTrace Buffers"). However, because an aggregation is fundamentally a
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * hash, there exists dynamic metadata associated with an aggregation buffer
6185db853e024a486ff8837e6784dd290d866112dougm * that is not associated with other kinds of buffers. This aggregation
6185db853e024a486ff8837e6784dd290d866112dougm * metadata is _only_ relevant for the in-kernel implementation of
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * aggregations; it is not actually relevant to user-level consumers. To do
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * this, we allocate dynamic aggregation data (hash keys and hash buckets)
6185db853e024a486ff8837e6784dd290d866112dougm * starting below the _limit_ of the buffer, and we allocate data from the
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * _base_ of the buffer. When the aggregation buffer is copied out, _only_ the
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * data is copied out; the metadata is simply discarded. Schematically,
6185db853e024a486ff8837e6784dd290d866112dougm * aggregation buffers look like:
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * base of data buffer ---> +-------+------+-----------+-------+
6185db853e024a486ff8837e6784dd290d866112dougm * | aggid | key | value | aggid |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * +-------+------+-----------+-------+
6185db853e024a486ff8837e6784dd290d866112dougm * +-------+-------+-----+------------+
6185db853e024a486ff8837e6784dd290d866112dougm * | value | aggid | key | value |
6185db853e024a486ff8837e6784dd290d866112dougm * +-------+------++-----+------+-----+
6185db853e024a486ff8837e6784dd290d866112dougm * | aggid | key | value | |
6185db853e024a486ff8837e6784dd290d866112dougm * +-------+------+-------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | || +------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | || | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +---------------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | hash keys |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | (dtrace_aggkey structures) |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------------------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | hash buckets |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | (dtrace_aggbuffer structure) |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * limit of data buffer ---> +----------------------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * As implied above, just as we assure that ECBs always store a constant
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * amount of data, we assure that a given aggregation -- identified by its
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * aggregation ID -- always stores data of a constant quantity and type.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * As with EPIDs, this allows the aggregation ID to serve as the metadata for a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * given record.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * aligned. (If this the structure changes such that this becomes false, an
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * assertion will fail in dtrace_aggregate().)
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_aggkey {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw struct dtrace_aggkey *dtak_next; /* next in hash chain */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_aggbuffer {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTrace Speculations
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Speculations have a per-CPU buffer and a global state. Once a speculation
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * buffer has been comitted or discarded, it cannot be reused until all CPUs
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * have taken the same action (commit or discard) on their respective
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * speculative buffer. However, because DTrace probes may execute in arbitrary
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * context, other CPUs cannot simply be cross-called at probe firing time to
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * perform the necessary commit or discard. The speculation states thus
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * optimize for the case that a speculative buffer is only active on one CPU at
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * the time of a commit() or discard() -- for if this is the case, other CPUs
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * need not take action, and the speculation is immediately available for
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * reuse. If the speculation is active on multiple CPUs, it must be
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * asynchronously cleaned -- potentially leading to a higher rate of dirty
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * speculative drops. The speculation states are as follows:
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTRACESPEC_INACTIVE <= Initial state; inactive speculation
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTRACESPEC_ACTIVE <= Allocated, but not yet speculatively traced to
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTRACESPEC_ACTIVEONE <= Speculatively traced to on one CPU
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTRACESPEC_ACTIVEMANY <= Speculatively traced to on more than one CPU
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTRACESPEC_COMMITTING <= Currently being commited on one CPU
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTRACESPEC_DISCARDING <= Currently being discarded on many CPUs
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * The state transition diagram is as follows:
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------------------------------------------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | +------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | +-------------------| COMMITTING |<-----------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | | +------------+ | |
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * | | copied spec. ^ commit() on | | discard() on
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * | | into principal | active CPU | | active CPU
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * | | | commit() | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * V V | | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------+ +--------+ +-----------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE |
6185db853e024a486ff8837e6784dd290d866112dougm * +----------+ speculation() +--------+ speculate() +-----------+
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * ^ ^ | | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | | | discard() | |
6185db853e024a486ff8837e6784dd290d866112dougm * | | asynchronously | discard() on | | speculate()
6185db853e024a486ff8837e6784dd290d866112dougm * | | cleaned V inactive CPU | | on inactive
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * | | +------------+ | | CPU
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * | +-------------------| DISCARDING |<-----------------+ |
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * | +------------+ |
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * | asynchronously ^ |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | copied spec. | discard() |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | into principal +------------------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------------+ commit() +------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | COMMITTINGMANY |<----------------------------------| ACTIVEMANY |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------------+ +------------+
330ef417fbd2286149a25e8033587edf7ae52ae5dougmtypedef struct dtrace_speculation {
330ef417fbd2286149a25e8033587edf7ae52ae5dougm dtrace_speculation_state_t dtsp_state; /* current speculation state */
330ef417fbd2286149a25e8033587edf7ae52ae5dougm dtrace_buffer_t *dtsp_buffer; /* speculative buffer */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Dynamic Variables
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * The dynamic variable problem is obviously decomposed into two subproblems:
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * allocating new dynamic storage, and freeing old dynamic storage. The
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * presence of the second problem makes the first much more complicated -- or
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * rather, the absence of the second renders the first trivial. This is the
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * case with aggregations, for which there is effectively no deallocation of
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * dynamic storage. (Or more accurately, all dynamic storage is deallocated
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * when a snapshot is taken of the aggregation.) As DTrace dynamic variables
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * allow for both dynamic allocation and dynamic deallocation, the
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * implementation of dynamic variables is quite a bit more complicated than
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * that of their aggregation kin.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * We observe that allocating new dynamic storage is tricky only because the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * size can vary -- the allocation problem is much easier if allocation sizes
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * are uniform. We further observe that in D, the size of dynamic variables is
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * actually _not_ dynamic -- dynamic variable sizes may be determined by static
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * analysis of DIF text. (This is true even of putatively dynamically-sized
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * objects like strings and stacks, the sizes of which are dictated by the
549ec3fff108310966327d1dc9004551b63210b7dougm * "stringsize" and "stackframes" variables, respectively.) We exploit this by
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * performing this analysis on all DIF before enabling any probes. For each
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * dynamic load or store, we calculate the dynamically-allocated size plus the
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * size of the dtrace_dynvar structure plus the storage required to key the
6185db853e024a486ff8837e6784dd290d866112dougm * data. For all DIF, we take the largest value and dub it the _chunksize_.
6185db853e024a486ff8837e6784dd290d866112dougm * We then divide dynamic memory into two parts: a hash table that is wide
6185db853e024a486ff8837e6784dd290d866112dougm * enough to have every chunk in its own bucket, and a larger region of equal
6185db853e024a486ff8837e6784dd290d866112dougm * chunksize units. Whenever we wish to dynamically allocate a variable, we
6185db853e024a486ff8837e6784dd290d866112dougm * always allocate a single chunk of memory. Depending on the uniformity of
7d968cb8b4b6274092771b93e94bf88d1ee31c6cdougm * allocation, this will waste some amount of memory -- but it eliminates the
6185db853e024a486ff8837e6784dd290d866112dougm * non-determinism inherent in traditional heap fragmentation.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * Dynamic objects are allocated by storing a non-zero value to them; they are
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * deallocated by storing a zero value to them. Dynamic variables are
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * complicated enormously by being shared between CPUs. In particular,
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * consider the following scenario:
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * CPU A CPU B
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * +---------------------------------+ +---------------------------------+
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | allocates dynamic object a[123] | | |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | by storing the value 345 to it | | |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | ---------> |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | | | wishing to load from object |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | | | a[123], performs lookup in |
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * | | | dynamic variable space |
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * | <--------- |
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * | deallocates object a[123] by | | |
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * | storing 0 to it | | |
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * | allocates dynamic object b[567] | | performs load from a[123] |
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * | by storing the value 789 to it | | |
330ef417fbd2286149a25e8033587edf7ae52ae5dougm * This is obviously a race in the D program, but there are nonetheless only
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * two valid values for CPU B's load from a[123]: 345 or 0. Most importantly,
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * CPU B may _not_ see the value 789 for a[123].
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * There are essentially two ways to deal with this:
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * (1) Explicitly spin-lock variables. That is, if CPU B wishes to load
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * from a[123], it needs to lock a[123] and hold the lock for the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * duration that it wishes to manipulate it.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * (2) Avoid reusing freed chunks until it is known that no CPU is referring
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * to them.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * The implementation of (1) is rife with complexity, because it requires the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * user of a dynamic variable to explicitly decree when they are done using it.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Were all variables by value, this perhaps wouldn't be debilitating -- but
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dynamic variables of non-scalar types are tracked by reference. That is, if
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * a dynamic variable is, say, a string, and that variable is to be traced to,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * say, the principal buffer, the DIF emulation code returns to the main
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_probe() loop a pointer to the underlying storage, not the contents of
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * the storage. Further, code calling on DIF emulation would have to be aware
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * that the DIF emulation has returned a reference to a dynamic variable that
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * has been potentially locked. The variable would have to be unlocked after
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * the main dtrace_probe() loop is finished with the variable, and the main
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * dtrace_probe() loop would have to be careful to not call any further DIF
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * emulation while the variable is locked to avoid deadlock. More generally,
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * if one were to implement (1), DIF emulation code dealing with dynamic
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * variables could only deal with one dynamic variable at a time (lest deadlock
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * result). To sum, (1) exports too much subtlety to the users of dynamic
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * variables -- increasing maintenance burden and imposing serious constraints
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * on future DTrace development.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * The implementation of (2) is also complex, but the complexity is more
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * manageable. We need to be sure that when a variable is deallocated, it is
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * not placed on a traditional free list, but rather on a _dirty_ list. Once a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * variable is on a dirty list, it cannot be found by CPUs performing a
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * subsequent lookup of the variable -- but it may still be in use by other
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * CPUs. To assure that all CPUs that may be seeing the old variable have
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * cleared out of probe context, a dtrace_sync() can be issued. Once the
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * dtrace_sync() has completed, it can be known that all CPUs are done
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * manipulating the dynamic variable -- the dirty list can be atomically
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * appended to the free list. Unfortunately, there's a slight hiccup in this
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * mechanism: dtrace_sync() may not be issued from probe context. The
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * dtrace_sync() must be therefore issued asynchronously from non-probe
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * context. For this we rely on the DTrace cleaner, a cyclic that runs at the
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * "cleanrate" frequency. To ease this implementation, we define several chunk
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * - Dirty. Deallocated chunks, not yet cleaned. Not available.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * - Rinsing. Formerly dirty chunks that are currently being asynchronously
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * cleaned. Not available, but will be shortly. Dynamic variable
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * allocation may not spin or block for availability, however.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * - Clean. Clean chunks, ready for allocation -- but not on the free list.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * - Free. Available for allocation.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * Moreover, to avoid absurd contention, _each_ of these lists is implemented
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * on a per-CPU basis. This is only for performance, not correctness; chunks
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * may be allocated from another CPU's free list. The algorithm for allocation
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * then is this:
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * (1) Attempt to atomically allocate from current CPU's free list. If list
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * is non-empty and allocation is successful, allocation is complete.
6185db853e024a486ff8837e6784dd290d866112dougm * (2) If the clean list is non-empty, atomically move it to the free list,
6185db853e024a486ff8837e6784dd290d866112dougm * and reattempt (1).
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * (3) If the dynamic variable space is in the CLEAN state, look for free
6185db853e024a486ff8837e6784dd290d866112dougm * and clean lists on other CPUs by setting the current CPU to the next
6185db853e024a486ff8837e6784dd290d866112dougm * CPU, and reattempting (1). If the next CPU is the current CPU (that
6185db853e024a486ff8837e6784dd290d866112dougm * is, if all CPUs have been checked), atomically switch the state of
6185db853e024a486ff8837e6784dd290d866112dougm * the dynamic variable space based on the following:
6185db853e024a486ff8837e6784dd290d866112dougm * - If no free chunks were found and no dirty chunks were found,
6185db853e024a486ff8837e6784dd290d866112dougm * atomically set the state to EMPTY.
6185db853e024a486ff8837e6784dd290d866112dougm * - If dirty chunks were found, atomically set the state to DIRTY.
6185db853e024a486ff8837e6784dd290d866112dougm * - If rinsing chunks were found, atomically set the state to RINSING.
6185db853e024a486ff8837e6784dd290d866112dougm * (4) Based on state of dynamic variable space state, increment appropriate
6185db853e024a486ff8837e6784dd290d866112dougm * counter to indicate dynamic drops (if in EMPTY state) vs. dynamic
6185db853e024a486ff8837e6784dd290d866112dougm * dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in
6185db853e024a486ff8837e6784dd290d866112dougm * RINSING state). Fail the allocation.
6185db853e024a486ff8837e6784dd290d866112dougm * The cleaning cyclic operates with the following algorithm: for all CPUs
6185db853e024a486ff8837e6784dd290d866112dougm * with a non-empty dirty list, atomically move the dirty list to the rinsing
6185db853e024a486ff8837e6784dd290d866112dougm * list. Perform a dtrace_sync(). For all CPUs with a non-empty rinsing list,
6185db853e024a486ff8837e6784dd290d866112dougm * atomically move the rinsing list to the clean list. Perform another
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * dtrace_sync(). By this point, all CPUs have seen the new clean list; the
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * state of the dynamic variable space can be restored to CLEAN.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * There exist two final races that merit explanation. The first is a simple
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * allocation race:
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * CPU A CPU B
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * +---------------------------------+ +---------------------------------+
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | allocates dynamic object a[123] | | allocates dynamic object a[123] |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | by storing the value 345 to it | | by storing the value 567 to it |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * Again, this is a race in the D program. It can be resolved by having a[123]
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * hold the value 345 or a[123] hold the value 567 -- but it must be true that
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * a[123] have only _one_ of these values. (That is, the racing CPUs may not
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * put the same element twice on the same hash chain.) This is resolved
6185db853e024a486ff8837e6784dd290d866112dougm * simply: before the allocation is undertaken, the start of the new chunk's
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * hash chain is noted. Later, after the allocation is complete, the hash
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * chain is atomically switched to point to the new element. If this fails
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * (because of either concurrent allocations or an allocation concurrent with a
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * deletion), the newly allocated chunk is deallocated to the dirty list, and
6185db853e024a486ff8837e6784dd290d866112dougm * the whole process of looking up (and potentially allocating) the dynamic
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * variable is reattempted.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * The final race is a simple deallocation race:
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * CPU A CPU B
6185db853e024a486ff8837e6784dd290d866112dougm * +---------------------------------+ +---------------------------------+
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | deallocates dynamic object | | deallocates dynamic object |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | a[123] by storing the value 0 | | a[123] by storing the value 0 |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * | to it | | to it |
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * Once again, this is a race in the D program, but it is one that we must
6185db853e024a486ff8837e6784dd290d866112dougm * handle without corrupting the underlying data structures. Because
6185db853e024a486ff8837e6784dd290d866112dougm * deallocations require the deletion of a chunk from the middle of a hash
6185db853e024a486ff8837e6784dd290d866112dougm * chain, we cannot use a single-word atomic operation to remove it. For this,
6185db853e024a486ff8837e6784dd290d866112dougm * we add a spin lock to the hash buckets that is _only_ used for deallocations
6185db853e024a486ff8837e6784dd290d866112dougm * (allocation races are handled as above). Further, this spin lock is _only_
6185db853e024a486ff8837e6784dd290d866112dougm * held for the duration of the delete; before control is returned to the DIF
6185db853e024a486ff8837e6784dd290d866112dougm * emulation code, the hash bucket is unlocked.
6185db853e024a486ff8837e6784dd290d866112dougmtypedef struct dtrace_key {
6185db853e024a486ff8837e6784dd290d866112dougmtypedef struct dtrace_tuple {
6185db853e024a486ff8837e6784dd290d866112dougmtypedef struct dtrace_dynvar {
6185db853e024a486ff8837e6784dd290d866112dougm struct dtrace_dynvar *dtdv_next; /* next on list or hash chain */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmtypedef struct dtrace_dynhash {
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_dynvar_t *dtdh_chain; /* hash chain for this bucket */
6185db853e024a486ff8837e6784dd290d866112dougm uintptr_t dtdh_pad[6]; /* pad to avoid false sharing */
6185db853e024a486ff8837e6784dd290d866112dougm uintptr_t dtdh_pad[14]; /* pad to avoid false sharing */
6185db853e024a486ff8837e6784dd290d866112dougmtypedef struct dtrace_dstate_percpu {
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_dynvar_t *dtdsc_free; /* free list for this CPU */
6185db853e024a486ff8837e6784dd290d866112dougm dtrace_dynvar_t *dtdsc_dirty; /* dirty list for this CPU */
6185db853e024a486ff8837e6784dd290d866112dougm dtrace_dynvar_t *dtdsc_rinsing; /* rinsing list for this CPU */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_dynvar_t *dtdsc_clean; /* clean list for this CPU */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm uint64_t dtdsc_dirty_drops; /* number of dirty drops */
6185db853e024a486ff8837e6784dd290d866112dougm uint64_t dtdsc_rinsing_drops; /* number of rinsing drops */
6185db853e024a486ff8837e6784dd290d866112dougm uint64_t dtdsc_pad[2]; /* pad to avoid false sharing */
6185db853e024a486ff8837e6784dd290d866112dougmtypedef struct dtrace_dstate {
6185db853e024a486ff8837e6784dd290d866112dougm dtrace_dynhash_t *dtds_hash; /* pointer to hash table */
6185db853e024a486ff8837e6784dd290d866112dougm dtrace_dstate_state_t dtds_state; /* current dynamic var. state */
6185db853e024a486ff8837e6784dd290d866112dougm dtrace_dstate_percpu_t *dtds_percpu; /* per-CPU dyn. var. state */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTrace Variable State
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * The DTrace variable state tracks user-defined variables in its dtrace_vstate
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * structure. Each DTrace consumer has exactly one dtrace_vstate structure,
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * but some dtrace_vstate structures may exist without a corresponding DTrace
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * consumer (see "DTrace Helpers", below). As described in <sys/dtrace.h>,
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * user-defined variables can have one of three scopes:
6185db853e024a486ff8837e6784dd290d866112dougm * DIFV_SCOPE_GLOBAL => global scope
6185db853e024a486ff8837e6784dd290d866112dougm * DIFV_SCOPE_THREAD => thread-local scope (i.e. "self->" variables)
6185db853e024a486ff8837e6784dd290d866112dougm * DIFV_SCOPE_LOCAL => clause-local scope (i.e. "this->" variables)
6185db853e024a486ff8837e6784dd290d866112dougm * The variable state tracks variables by both their scope and their allocation
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * - The dtvs_globals and dtvs_locals members each point to an array of
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * dtrace_statvar structures. These structures contain both the variable
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * metadata (dtrace_difv structures) and the underlying storage for all
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * statically allocated variables, including statically allocated
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * - The dtvs_tlocals member points to an array of dtrace_difv structures for
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DIFV_SCOPE_THREAD variables. As such, this array tracks _only_ the
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * is allocated out of the dynamic variable space.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * - The dtvs_dynvars member is the dynamic variable state associated with the
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * variable state. The dynamic variable state (described in "DTrace Dynamic
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * Variables", above) tracks all DIFV_SCOPE_THREAD variables and all
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * dynamically-allocated DIFV_SCOPE_GLOBAL variables.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmtypedef struct dtrace_statvar {
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmtypedef struct dtrace_vstate {
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_statvar_t **dtvs_globals; /* statically-allocated glbls */
6185db853e024a486ff8837e6784dd290d866112dougm dtrace_difv_t *dtvs_tlocals; /* thread-local metadata */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_statvar_t **dtvs_locals; /* clause-local data */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_dstate_t dtvs_dynvars; /* dynamic variable state */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Machine State
6185db853e024a486ff8837e6784dd290d866112dougm * In the process of processing a fired probe, DTrace needs to track and/or
6185db853e024a486ff8837e6784dd290d866112dougm * cache some per-CPU state associated with that particular firing. This is
6185db853e024a486ff8837e6784dd290d866112dougm * state that is always discarded after the probe firing has completed, and
6185db853e024a486ff8837e6784dd290d866112dougm * much of it is not specific to any DTrace consumer, remaining valid across
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * all ECBs. This state is tracked in the dtrace_mstate structure.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_mstate {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw uintptr_t dtms_scratch_base; /* base of scratch space */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw uintptr_t dtms_scratch_ptr; /* current scratch pointer */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw hrtime_t dtms_walltimestamp; /* cached wall timestamp */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw#define DTRACE_PROBEKEY_MAXDEPTH 8 /* max glob recursion depth */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTrace Activity
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Each DTrace consumer is in one of several states, which (for purposes of
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * avoiding yet-another overloading of the noun "state") we call the current
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * _activity_. The activity transitions on dtrace_go() (from DTRACIOCGO), on
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action. Activities may
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * only transition in one direction; the activity transition diagram is a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * directed acyclic graph. The activity transition diagram is as follows:
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------+ +--------+ +--------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------+ dtrace_go(), +--------+ dtrace_go(), +--------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * before BEGIN | after BEGIN | | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * exit() action | | | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * from BEGIN ECB | | | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------+ exit() action | | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | DRAINING |<-------------------+ | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +----------+ | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_stop(), | | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * before END | | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +---------+ +----------+ | |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | STOPPED |<------------------| COOLDOWN |<----------------------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +---------+ dtrace_stop(), +----------+ dtrace_stop(), |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * after END before END |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +--------+ |
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * | KILLED |<--------------------------+
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * +--------+ deadman timeout
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Note that once a DTrace consumer has stopped tracing, there is no way to
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * restart it; if a DTrace consumer wishes to restart tracing, it must reopen
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * the DTrace pseudodevice.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef enum dtrace_activity {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTrace Helper Implementation
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * A description of the helper architecture may be found in <sys/dtrace.h>.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Each process contains a pointer to its helpers in its p_dtrace_helpers
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * member. This is a pointer to a dtrace_helpers structure, which contains an
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * array of pointers to dtrace_helper structures, helper variable state (shared
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * among a process's helpers) and a generation count. (The generation count is
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * used to provide an identifier when a helper is added so that it may be
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * subsequently removed.) The dtrace_helper structure is self-explanatory,
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * containing pointers to the objects needed to execute the helper. Note that
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * helpers are _duplicated_ across fork(2), and destroyed on exec(2). No more
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * than dtrace_helpers_max are allowed per-process.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_helper_action {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_difo_t *dthp_predicate; /* helper action predicate */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw struct dtrace_helper_action *dthp_next; /* next helper action */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_helper_provider {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dof_helper_t dthp_prov; /* DOF w/ provider and probes */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_helpers {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_helper_action_t **dthps_actions; /* array of helper actions */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_vstate_t dthps_vstate; /* helper action var. state */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_helper_provider_t **dthps_provs; /* array of providers */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTrace Helper Action Tracing
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Debugging helper actions can be arduous. To ease the development and
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing-
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * framework: helper tracing. If dtrace_helptrace_enabled is non-zero (which
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * it is by default on DEBUG kernels), all helper activity will be traced to a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * global, in-kernel ring buffer. Each entry includes a pointer to the specific
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * helper, the location within the helper, and a trace of all local variables.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * The ring buffer may be displayed in a human-readable format with the
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * ::dtrace_helptrace mdb(1) dcmd.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_helptrace {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw dtrace_helper_action_t *dtht_helper; /* helper action */
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * DTrace Credentials
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * In probe context, we don't have the flexibility to examine the credentials
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * of the DTrace consumer that created a particular enabling. Instead, we use
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * the Least Privilege interfaces to cache the consumer's credentials in a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * dtrace_cred_t structure. That structure contains two important sets of
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * credentials that limit the consumer's breadth of visibility and what
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * actions the consumer may take.
6185db853e024a486ff8837e6784dd290d866112dougm#define DTRACE_CRV_ALL (DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL)
6185db853e024a486ff8837e6784dd290d866112dougm DTRACE_CRA_PROC_DESTRUCTIVE | DTRACE_CRA_PROC_CONTROL | \
6185db853e024a486ff8837e6784dd290d866112dougmtypedef struct dtrace_cred {
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Consumer State
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * Each DTrace consumer has an associated dtrace_state structure that contains
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * its in-kernel DTrace state -- including options, credentials, statistics and
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * pointers to ECBs, buffers, speculations and formats. A dtrace_state
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * structure is also allocated for anonymous enablings. When anonymous state
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * dtrace_state structure.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw struct dtrace_state *dts_anon; /* anon. state, if grabbed */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_buffer_t *dts_aggbuffer; /* aggregation buffer */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_speculation_t *dts_speculations; /* speculation array */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_aggregation_t **dts_aggregations; /* aggregation array */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm vmem_t *dts_aggid_arena; /* arena for aggregation IDs */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm uint32_t dts_speculations_busy; /* number of spec. busy */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm uint32_t dts_speculations_unavail; /* number of spec unavail */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm char dts_destructive; /* boolean: has dest. actions */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */
6185db853e024a486ff8837e6784dd290d866112dougm struct dtrace_provider *dtpv_next; /* next provider */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_mops_t dtm_mops; /* meta provider operations */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Enablings
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * A dtrace_enabling structure is used to track a collection of ECB
6185db853e024a486ff8837e6784dd290d866112dougm * descriptions -- before they have been turned into actual ECBs. This is
6185db853e024a486ff8837e6784dd290d866112dougm * created as a result of DOF processing, and is generally used to generate
6185db853e024a486ff8837e6784dd290d866112dougm * ECBs immediately thereafter. However, enablings are also generally
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * retained should the probes they describe be created at a later time; as
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * each new module or provider registers with the framework, the retained
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * enablings are reevaluated, with any new match resulting in new ECBs. To
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * prevent probes from being matched more than once, the enabling tracks the
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * last probe generation matched, and only matches probes from subsequent
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * generations.
6185db853e024a486ff8837e6784dd290d866112dougmtypedef struct dtrace_enabling {
6185db853e024a486ff8837e6784dd290d866112dougm dtrace_ecbdesc_t **dten_desc; /* all ECB descriptions */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_vstate_t *dten_vstate; /* associated variable state */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_genid_t dten_probegen; /* matched probe generation */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_ecbdesc_t *dten_current; /* current ECB description */
6185db853e024a486ff8837e6784dd290d866112dougm struct dtrace_enabling *dten_prev; /* previous enabling */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm struct dtrace_enabling *dten_next; /* next enabling */
6185db853e024a486ff8837e6784dd290d866112dougm * DTrace Anonymous Enablings
6185db853e024a486ff8837e6784dd290d866112dougm * Anonymous enablings are DTrace enablings that are not associated with a
6185db853e024a486ff8837e6784dd290d866112dougm * controlling process, but rather derive their enabling from DOF stored as
6185db853e024a486ff8837e6784dd290d866112dougm * properties in the dtrace.conf file. If there is an anonymous enabling, a
549ec3fff108310966327d1dc9004551b63210b7dougm * DTrace consumer state and enabling are created on attach. The state may be
6185db853e024a486ff8837e6784dd290d866112dougm * subsequently grabbed by the first consumer specifying the "grabanon"
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * option. As long as an anonymous DTrace enabling exists, dtrace(7D) will
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * refuse to unload.
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmtypedef struct dtrace_anon {
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_state_t *dta_state; /* DTrace consumer state */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm dtrace_enabling_t *dta_enabling; /* pointer to enabling */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm processorid_t dta_beganon; /* which CPU BEGIN ran on */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Error Debugging
6185db853e024a486ff8837e6784dd290d866112dougmtypedef struct dtrace_errhash {
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#define DTRACE_ERRHASHSZ 256 /* must be > number of err msgs */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#endif /* DTRACE_ERRDEBUG */
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Toxic Ranges
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace supports safe loads from probe context; if the address turns out to
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * be invalid, a bit will be set by the kernel indicating that DTrace
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * encountered a memory error, and DTrace will propagate the error to the user
6185db853e024a486ff8837e6784dd290d866112dougm * accordingly. However, there may exist some regions of memory in which an
6185db853e024a486ff8837e6784dd290d866112dougm * arbitrary load can change system state, and from which it is impossible to
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * recover from such a load after it has been attempted. Examples of this may
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * include memory in which programmable I/O registers are mapped (for which a
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * read may have some implications for the device) or (in the specific case of
6185db853e024a486ff8837e6784dd290d866112dougm * UltraSPARC-I and -II) the virtual address hole. The platform is required
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * to make DTrace aware of these toxic ranges; DTrace will then check that
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * target addresses are not in a toxic range before attempting to issue a
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amw * safe load.
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwtypedef struct dtrace_toxrange {
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern int dtrace_getipl(void);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern void *dtrace_casptr(void *, void *, void *);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern void dtrace_copyinstr(uintptr_t, uintptr_t, size_t);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern void dtrace_copyout(uintptr_t, uintptr_t, size_t);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern void dtrace_copyoutstr(uintptr_t, uintptr_t, size_t);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern void dtrace_getpcstack(pc_t *, int, int, uint32_t *);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern int dtrace_getstackdepth(int);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern void dtrace_getufpstack(uint64_t *, uint64_t *, int);
da6c28aaf62fa55f0fdb8004aa40f88f23bf53f0amwextern int dtrace_getustackdepth(void);
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmextern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int,
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmextern int dtrace_assfail(const char *, const char *, int);
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmextern int dtrace_attached(void);
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmextern void dtrace_flush_windows(void);
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmextern void dtrace_flush_user_windows(void);
25a68471b9ababbc21cfdbbb2866014f34f419ecdougmextern void dtrace_copystr(uintptr_t, uintptr_t, size_t);
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace Assertions
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * DTrace calls ASSERT from probe context. To assure that a failed ASSERT
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * does not induce a markedly more catastrophic failure (e.g., one from which
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * may safely be called from probe context. This header file must thus be
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * included by any DTrace component that calls ASSERT from probe context, and
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * _only_ by those components. (The only exception to this is kernel
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm * debugging infrastructure at user-level that doesn't depend on calling
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#define ASSERT(X) ((void)0)
25a68471b9ababbc21cfdbbb2866014f34f419ecdougm#endif /* _SYS_DTRACE_IMPL_H */