dcpc.c revision b885580b43755ee4ea1e280b85428893d2ba9291
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/cap_util.h>
#include <sys/cpc_pcbe.h>
#include <sys/cpc_impl.h>
#include <sys/dtrace_impl.h>
/*
* DTrace CPU Performance Counter Provider
* ---------------------------------------
*
* The DTrace cpc provider allows DTrace consumers to access the CPU
* performance counter overflow mechanism of a CPU. The configuration
* presented in a probe specification is programmed into the performance
* counter hardware of all available CPUs on a system. Programming the
* hardware causes a counter on each CPU to begin counting events of the
* given type. When the specified number of events have occurred, an overflow
* interrupt will be generated and the probe is fired.
*
* The required configuration for the performance counter is encoded into
* the probe specification and this includes the performance counter event
* name, processor mode, overflow rate and an optional unit mask.
*
* Most processors provide several counters (PICs) which can count all or a
* subset of the events available for a given CPU. However, when overflow
* profiling is being used, not all CPUs can detect which counter generated the
* overflow interrupt. In this case we cannot reliably determine which counter
* overflowed and we therefore only allow such CPUs to configure one event at
* a time. Processors that can determine the counter which overflowed are
* allowed to program as many events at one time as possible (in theory up to
* the number of instrumentation counters supported by that platform).
* Therefore, multiple consumers can enable multiple probes at the same time
* on such platforms. Platforms which cannot determine the source of an
* overflow interrupt are only allowed to program a single event at one time.
*
* The performance counter hardware is made available to consumers on a
* first-come, first-served basis. Only a finite amount of hardware resource
* is available and, while we make every attempt to accomodate requests from
* consumers, we must deny requests when hardware resources have been exhausted.
* A consumer will fail to enable probes when resources are currently in use.
*
* The cpc provider contends for shared hardware resources along with other
* consumers of the kernel CPU performance counter subsystem (e.g. cpustat(1M)).
* Only one such consumer can use the performance counters at any one time and
* counters are made available on a first-come, first-served basis. As with
* cpustat, the cpc provider has priority over per-LWP libcpc usage (e.g.
* cputrack(1)). Invoking the cpc provider will cause all existing per-LWP
* counter contexts to be invalidated.
*/
typedef struct dcpc_probe {
char dcpc_event_name[CPC_MAX_EVENT_LEN];
int dcpc_picno; /* pic this event is programmed in */
int dcpc_enabled; /* probe is actually enabled? */
int dcpc_disabling; /* probe is currently being disabled */
int dcpc_actv_req_idx; /* idx into dcpc_actv_reqs[] */
} dcpc_probe_t;
static dev_info_t *dcpc_devi;
static dtrace_provider_id_t dcpc_pid;
static dcpc_probe_t **dcpc_actv_reqs;
static uint32_t dcpc_enablings = 0;
static int dcpc_ovf_mask = 0;
static int dcpc_mult_ovf_cap = 0;
static int dcpc_mask_type = 0;
/*
* When the dcpc provider is loaded, dcpc_min_overflow is set to either
* DCPC_MIN_OVF_DEFAULT or the value that dcpc-min-overflow is set to in
* the dcpc.conf file. Decrease this value to set probes with smaller
* overflow values. Remember that very small values could render a system
* unusable with frequently occurring events.
*/
#define DCPC_MIN_OVF_DEFAULT 5000
static uint32_t dcpc_min_overflow;
static int dcpc_aframes = 0; /* override for artificial frame setting */
#if defined(__x86)
#define DCPC_ARTIFICIAL_FRAMES 8
#define DCPC_ARTIFICIAL_FRAMES 2
#endif
/*
* Called from the platform overflow interrupt handler. 'bitmap' is a mask
* which contains the pic(s) that have overflowed.
*/
static void
{
int i;
/*
* No counter was marked as overflowing. Shout about it and get out.
*/
if ((bitmap & dcpc_ovf_mask) == 0) {
return;
}
/*
* This is the common case of a processor that doesn't support
* multiple overflow events. Such systems are only allowed a single
* enabling and therefore we just look for the first entry in
* the active request array.
*/
if (!dcpc_mult_ovf_cap) {
for (i = 0; i < cpc_ncounters; i++) {
if (dcpc_actv_reqs[i] != NULL) {
CPU->cpu_cpcprofile_upc, 0, 0, 0);
return;
}
}
return;
}
/*
* This is a processor capable of handling multiple overflow events.
* Iterate over the array of active requests and locate the counters
* that overflowed (note: it is possible for more than one counter to
* have overflowed at the same time).
*/
for (i = 0; i < cpc_ncounters; i++) {
if (dcpc_actv_reqs[i] != NULL &&
CPU->cpu_cpcprofile_upc, 0, 0, 0);
}
}
}
static void
{
if (dcpc_aframes)
return;
}
/*ARGSUSED*/
static void
{
/*
* The format of a probe is:
*
* event_name-mode-{optional_umask}-overflow_rate
* e.g.
* DC_refill_from_system-user-0x1e-50000, or,
* DC_refill_from_system-all-10000
*
*/
int i, flag = 0;
char event[CPC_MAX_EVENT_LEN];
/*
* The 'cpc' provider offers no probes by default.
*/
return;
/*
* We have a poor man's strtok() going on here. Replace any hyphens
* in the the probe name with NULL characters in order to make it
* easy to parse the string with regular string functions.
*/
for (i = 0; i < len; i++) {
if (str[i] == '-')
str[i] = '\0';
}
/*
* The first part of the string must be either a platform event
* name or a generic event name.
*/
/*
* The next part of the name is the mode specification. Valid
* settings are "user", "kernel" or "all".
*/
p += evlen + 1;
if (strcmp(p, "user") == 0)
flag |= CPC_COUNT_USER;
else if (strcmp(p, "kernel") == 0)
flag |= CPC_COUNT_SYSTEM;
else if (strcmp(p, "all") == 0)
else
goto err;
/*
* Next we either have a mask specification followed by an overflow
* rate or just an overflow rate on its own.
*/
p += strlen(p) + 1;
if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) {
/*
* A unit mask can only be specified if:
* 1) this performance counter back end supports masks.
* 2) the specified event is platform specific.
* 3) a valid hex number is converted.
* 4) no extraneous characters follow the mask specification.
*/
p += strlen(p) + 1;
} else {
goto err;
}
}
/*
* This final part must be an overflow value which has to be greater
* than the minimum permissible overflow rate.
*/
goto err;
/*
* Validate the event and create the probe.
*/
for (i = 0; i < cpc_ncounters; i++) {
int found = 0, j;
goto err;
for (j = 0; j < llen; j++) {
if (cp[j] == ',')
cp[j] = '\0';
}
found = 1;
}
p += strlen(p) + 1;
}
if (found)
break;
}
err:
}
/*ARGSUSED*/
static void
{
}
/*ARGSUSED*/
static int
{
return (CPU->cpu_cpcprofile_pc == 0);
}
static void
{
int i;
/*
* If a unit mask has been specified then detect which attribute
* the platform needs. For now, it's either "umask" or "emask".
*/
if (pp->dcpc_umask >= 0) {
if (dcpc_mask_type & DCPC_UMASK)
"umask", 5);
else
"emask", 5);
} else {
}
/*
* If this probe is enabled, obtain its current countdown value
* and use that. The CPUs cpc context might not exist yet if we
* are dealing with a CPU that is just coming online.
*/
}
}
} else {
}
}
/*
* Create a fresh request set for the enablings represented in the
* 'dcpc_actv_reqs' array which contains the probes we want to be
* in the set. This can be called for several reasons:
*
* 1) We are on a single or multi overflow platform and we have no
* current events so we can just create the set and initialize it.
* 2) We are on a multi-overflow platform and we already have one or
* more existing events and we are adding a new enabling. Create a
* new set and copy old requests in and then add the new request.
* 3) We are on a multi-overflow platform and we have just removed an
* enabling but we still have enablings whch are valid. Create a new
* set and copy in still valid requests.
*/
static kcpc_set_t *
dcpc_create_set(cpu_t *c)
{
int i, reqno = 0;
int active_requests = 0;
/*
* First get a count of the number of currently active requests.
* Note that dcpc_actv_reqs[] should always reflect which requests
* we want to be in the set that is to be created. It is the
* responsibility of the caller of dcpc_create_set() to adjust that
* array accordingly beforehand.
*/
for (i = 0; i < cpc_ncounters; i++) {
if (dcpc_actv_reqs[i] != NULL)
}
/*
* Look for valid entries in the active requests array and populate
* the request set for any entries found.
*/
for (i = 0; i < cpc_ncounters; i++) {
if (dcpc_actv_reqs[i] != NULL) {
reqno++;
}
}
return (set);
}
static int
{
int i, j, subcode;
set = dcpc_create_set(c);
goto err;
goto err;
for (j = 0; j < cpc_ncounters; j++) {
if (dcpc_actv_reqs[j] != NULL &&
dcpc_actv_reqs[j]->dcpc_event_name) == 0) {
dcpc_actv_reqs[j]->dcpc_picno =
}
}
}
/*
* If we already have an active enabling then save the current cpc
* context away.
*/
octx = c->cpu_cpc_ctx;
kcpc_cpu_program(c, ctx);
}
return (0);
err:
/*
* We failed to configure this request up so free things up and
* get out.
*/
return (-1);
}
static void
{
/*
* Leave this CPU alone if it's already offline.
*/
if (c->cpu_flags & CPU_OFFLINE)
return;
/*
* Grab CPUs CPC context before kcpc_cpu_stop() stops counters and
* changes it.
*/
ctx = c->cpu_cpc_ctx;
kcpc_cpu_stop(c, B_FALSE);
}
/*
* Stop overflow interrupts being actively processed so that per-CPU
* configuration state can be changed safely and correctly. Each CPU has a
* dcpc interrupt state byte which is transitioned from DCPC_INTR_FREE (the
* "free" state) to DCPC_INTR_CONFIG (the "configuration in process" state)
* before any configuration state is changed on any CPUs. The hardware overflow
* handler, kcpc_hw_overflow_intr(), will only process an interrupt when a
* configuration is not in process (i.e. the state is marked as free). During
* interrupt processing the state is set to DCPC_INTR_PROCESSING by the
* overflow handler.
*/
static void
dcpc_block_interrupts(void)
{
cpu_t *c;
c = cpu_list;
do {
continue;
}
/*
* Set all CPUs dcpc interrupt state to DCPC_INTR_FREE to indicate that
* overflow interrupts can be processed safely.
*/
static void
dcpc_release_interrupts(void)
{
do {
}
/*
* dcpc_program_event() can be called owing to a new enabling or if a multi
* overflow platform has disabled a request but needs to program the requests
* that are still valid.
*
* Every invocation of dcpc_program_event() will create a new kcpc_ctx_t
* and a new request set which contains the new enabling and any old enablings
* which are still valid (possible with multi-overflow platforms).
*/
static int
{
cpu_t *c;
int ret = 0;
c = cpu_list;
do {
/*
* Skip CPUs that are currently offline.
*/
if (c->cpu_flags & CPU_OFFLINE)
continue;
/*
* Stop counters but preserve existing DTrace CPC context
* if there is one.
*
* If we come here when the first event is programmed for a CPU,
* there should be no DTrace CPC context installed. In this
* case, kcpc_cpu_stop() will ensure that there is no other
* context on the CPU.
*
* If we add new enabling to the original one, the CPU should
* have the old DTrace CPC context which we need to keep around
* since dcpc_program_event() will add to it.
*/
if (c->cpu_cpc_ctx != NULL)
kcpc_cpu_stop(c, B_TRUE);
/*
* If this enabling is being removed (in the case of a multi event
* capable system with more than one active enabling), we can now
* update the active request array to reflect the enablings that need
* to be reprogrammed.
*/
do {
/*
* Skip CPUs that are currently offline.
*/
if (c->cpu_flags & CPU_OFFLINE)
continue;
ret = dcpc_program_cpu_event(c);
/*
* If dcpc_program_cpu_event() fails then it is because we couldn't
* configure the requests in the set for the CPU and not because of
* an error programming the hardware. If we have a failure here then
* we assume no CPUs have been programmed in the above step as they
* are all configured identically.
*/
if (ret != 0) {
pp->dcpc_enabled = 0;
return (-1);
}
return (0);
}
/*ARGSUSED*/
static int
{
int i, found = 0;
cpu_t *c;
/*
* Bail out if the counters are being used by a libcpc consumer.
*/
if (kcpc_cpuctx > 0) {
return (-1);
}
/*
* Locate this enabling in the first free entry of the active
* request array.
*/
for (i = 0; i < cpc_ncounters; i++) {
if (dcpc_actv_reqs[i] == NULL) {
dcpc_actv_reqs[i] = pp;
pp->dcpc_actv_req_idx = i;
found = 1;
break;
}
}
/*
* If we couldn't find a slot for this probe then there is no
* room at the inn.
*/
if (!found) {
return (-1);
}
/*
* DTrace is taking over CPC contexts, so stop collecting
* capacity/utilization data for all CPUs.
*/
if (dtrace_cpc_in_use == 1)
cu_disable();
/*
* The following must hold true if we are to (attempt to) enable
* this request:
*
* 1) No enablings currently exist. We allow all platforms to
* proceed if this is true.
*
* OR
*
* 2) If the platform is multi overflow capable and there are
* less valid enablings than there are counters. There is no
* guarantee that a platform can accommodate as many events as
* it has counters for but we will at least try to program
* up to that many requests.
*
* The 'dcpc_enablings' variable is implictly protected by locking
* provided by the DTrace framework and the cpu management framework.
*/
if (dcpc_enablings == 0 || (dcpc_mult_ovf_cap &&
dcpc_enablings < cpc_ncounters)) {
/*
* Before attempting to program the first enabling we need to
* invalidate any lwp-based contexts.
*/
if (dcpc_enablings == 0)
if (dcpc_program_event(pp) == 0) {
return (0);
}
}
/*
* If active enablings existed before we failed to enable this probe
* on a multi event capable platform then we need to restart counters
* as they will have been stopped in the attempted configuration. The
* context should now just contain the request prior to this failed
* enabling.
*/
if (dcpc_enablings > 0 && dcpc_mult_ovf_cap) {
c = cpu_list;
do {
/*
* Skip CPUs that are currently offline.
*/
if (c->cpu_flags & CPU_OFFLINE)
continue;
kcpc_cpu_program(c, c->cpu_cpc_ctx);
}
/*
* If all probes are removed, enable capacity/utilization data
* collection for every CPU.
*/
if (dtrace_cpc_in_use == 0)
cu_enable();
return (-1);
}
/*
* If only one enabling is active then remove the context and free
* everything up. If there are multiple enablings active then remove this
* one, its associated meta-data and re-program the hardware.
*/
/*ARGSUSED*/
static void
{
cpu_t *c;
/*
* This probe didn't actually make it as far as being fully enabled
* so we needn't do anything with it.
*/
if (pp->dcpc_enabled == 0) {
/*
* If we actually allocated this request a slot in the
* request array but failed to enabled it then remove the
* entry in the array.
*/
if (pp->dcpc_actv_req_idx >= 0) {
}
return;
}
/*
* If this is the only enabling then stop all the counters and
* free up the meta-data.
*/
if (dcpc_enablings == 1) {
c = cpu_list;
do {
dcpc_disable_cpu(c);
} else {
/*
* This platform can support multiple overflow events and
* the enabling being disabled is not the last one. Remove this
* enabling and re-program the hardware with the new config.
*/
(void) dcpc_program_event(pp);
}
pp->dcpc_enabled = 0;
/*
* If all probes are removed, enable capacity/utilization data
* collection for every CPU
*/
if (dtrace_cpc_in_use == 0)
cu_enable();
}
/*ARGSUSED*/
static int
{
cpu_t *c;
switch (what) {
case CPU_OFF:
/*
* Offline CPUs are not allowed to take part so remove this
* CPU if we are actively tracing.
*/
if (dtrace_cpc_in_use) {
/*
* Indicate that a configuration is in process in
* order to stop overflow interrupts being processed
* on this CPU while we disable it.
*/
continue;
dcpc_disable_cpu(c);
/*
* Reset this CPUs interrupt state as the configuration
* has ended.
*/
}
break;
case CPU_ON:
case CPU_SETUP:
/*
* This CPU is being initialized or brought online so program
* it with the current request set if we are actively tracing.
*/
if (dtrace_cpc_in_use) {
(void) dcpc_program_cpu_event(c);
}
break;
default:
break;
}
return (0);
}
static dtrace_pattr_t dcpc_attr = {
};
static dtrace_pops_t dcpc_pops = {
NULL,
NULL,
NULL,
NULL,
NULL,
};
/*ARGSUSED*/
static int
{
return (0);
}
/*ARGSUSED*/
static int
{
int error;
switch (infocmd) {
case DDI_INFO_DEVT2DEVINFO:
error = DDI_SUCCESS;
break;
case DDI_INFO_DEVT2INSTANCE:
*result = (void *)0;
error = DDI_SUCCESS;
break;
default:
error = DDI_FAILURE;
}
return (error);
}
static int
{
switch (cmd) {
case DDI_DETACH:
break;
case DDI_SUSPEND:
return (DDI_SUCCESS);
default:
return (DDI_FAILURE);
}
if (dtrace_unregister(dcpc_pid) != 0)
return (DDI_FAILURE);
return (DDI_SUCCESS);
}
static int
{
char *attrs;
switch (cmd) {
case DDI_ATTACH:
break;
case DDI_RESUME:
return (DDI_SUCCESS);
default:
return (DDI_FAILURE);
}
if (kcpc_pcbe_loaded() == -1)
return (DDI_FAILURE);
if (!(caps & CPC_CAP_OVERFLOW_INTERRUPT)) {
" on this processor");
return (DDI_FAILURE);
}
return (DDI_FAILURE);
}
ASSERT(dcpc_ovf_mask != 0);
if (caps & CPC_CAP_OVERFLOW_PRECISE)
dcpc_mult_ovf_cap = 1;
/*
* Determine which, if any, mask attribute the back-end can use.
*/
attrs = kcpc_list_attrs();
/*
* The dcpc_actv_reqs array is used to store the requests that
* we currently have programmed. The order of requests in this
* array is not necessarily the order that the event appears in
* the kcpc_request_t array. Once entered into a slot in the array
* the entry is not moved until it's removed.
*/
return (DDI_SUCCESS);
}
static struct cb_ops dcpc_cb_ops = {
dcpc_open, /* open */
nodev, /* close */
nulldev, /* strategy */
nulldev, /* print */
nodev, /* dump */
nodev, /* read */
nodev, /* write */
nodev, /* ioctl */
nodev, /* devmap */
nodev, /* mmap */
nodev, /* segmap */
nochpoll, /* poll */
ddi_prop_op, /* cb_prop_op */
0, /* streamtab */
};
DEVO_REV, /* devo_rev, */
0, /* refcnt */
dcpc_info, /* get_dev_info */
nulldev, /* identify */
nulldev, /* probe */
dcpc_attach, /* attach */
dcpc_detach, /* detach */
nodev, /* reset */
&dcpc_cb_ops, /* driver operations */
NULL, /* bus operations */
nodev, /* dev power */
ddi_quiesce_not_needed /* quiesce */
};
/*
* Module linkage information for the kernel.
*/
&mod_driverops, /* module type */
"DTrace CPC Module", /* name of module */
&dcpc_ops, /* driver ops */
};
static struct modlinkage modlinkage = {
(void *)&modldrv,
};
int
_init(void)
{
return (mod_install(&modlinkage));
}
int
{
}
int
_fini(void)
{
return (mod_remove(&modlinkage));
}