/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Hermon IB Performance Statistics routines
*
* Implements all the routines necessary for setting up, querying, and
* (later) tearing down all the kstats necessary for implementing to
* the interfaces necessary to provide busstat(1M) access.
*/
int reset);
/*
* Hermon IB Performance Events structure
* This structure is read-only and is used to setup the individual kstats
* and to initialize the tki_ib_perfcnt[] array for each Hermon instance.
*/
{"port_xmit_data", 0, 0},
{"port_recv_data", 0, 0},
{"port_xmit_pkts", 0, 0},
{"port_recv_pkts", 0, 0},
{"port_recv_err", 0, 0},
{"port_xmit_discards", 0, 0},
{"vl15_dropped", 0, 0},
{"port_xmit_wait", 0, 0},
{"port_recv_remote_phys_err", 0, 0},
{"port_xmit_constraint_err", 0, 0},
{"port_recv_constraint_err", 0, 0},
{"symbol_err_counter", 0, 0},
{"link_err_recovery_cnt", 0, 0},
{"link_downed_cnt", 0, 0},
{"excessive_buffer_overruns", 0, 0},
{"local_link_integrity_err", 0, 0},
{"clear_pic", 0, 0}
};
/*
* Return the maximum of (x) and (y)
*/
#define MAX(x, y) (((x) > (y)) ? (x) : (y))
/*
* Set (x) to the maximum of (x) and (y)
*/
#define SET_TO_MAX(x, y) \
{ \
if ((x) < (y)) \
(x) = (y); \
}
/*
* hermon_kstat_init()
* Context: Only called from attach() path context
*/
int
{
int i;
/* Allocate a kstat info structure */
KM_SLEEP);
return (DDI_FAILURE);
}
/*
* Create as many "pic" and perfcntr64 kstats as we have IB ports.
* Enable all of the events specified in the "hermon_ib_perfcnt_list"
* structure.
*/
for (i = 0; i < numports; i++) {
goto kstat_init_fail;
}
goto kstat_init_fail;
}
}
/* Create the "counters" kstat too */
goto kstat_init_fail;
}
/* Initialize the control register and initial counter values */
/*
* Initialize the Hermon hki_ib_perfcnt[] array values using the
* default values in hermon_ib_perfcnt_list[]
*/
for (i = 0; i < HERMON_CNTR_NUMENTRIES; i++) {
}
return (DDI_SUCCESS);
/* Delete all the previously created kstats */
}
for (i = 0; i < numports; i++) {
}
}
}
/* Free the kstat info structure */
return (DDI_FAILURE);
}
/*
* hermon_kstat_init()
*/
void
{
int i;
/* Get pointer to kstat info */
/*
* Signal the perfcntr64_update_thread to exit and wait until the
* thread exits.
*/
/* Delete all the "pic" and perfcntr64 kstats (one per port) */
for (i = 0; i < numports; i++) {
}
}
}
/* Delete the "counter" kstats (one per port) */
/* Free the kstat info structure */
}
/*
* hermon_kstat_picN_create()
* Context: Only called from attach() path context
*/
static kstat_t *
{
int drv_instance, i;
char *drv_name;
/*
* Create the "picN" kstat. In the steps, below we will attach
* all of our named event types to it.
*/
return (NULL);
}
/*
* Write event names and their associated pcr masks. The last entry
* in the array (clear_pic) is added separately below (as its pic
* value must be inverted).
*/
for (i = 0; i < num_evt - 1; i++) {
}
/* Add the "clear_pic" entry */
/* Install the kstat */
return (picN_ksp);
}
/*
* hermon_kstat_cntr_create()
* Context: Only called from attach() path context
*/
static kstat_t *
{
int drv_instance, i;
char *drv_name;
/*
* Create the "counters" kstat. In the steps, below we will attach
* all of our "pic" to it. Note: The size of this kstat is
* num_pic + 1 because it also contains the "%pcr".
*/
return (NULL);
}
/*
* Initialize the named kstats (for the "pcr" and for the
* individual "pic" kstats)
*/
for (i = 0; i < num_pic; i++) {
}
/*
* Store the Hermon softstate pointer in the kstat's private field so
* that it is available to the update function.
*/
/* Install the kstat */
return (cntr_ksp);
}
/*
* hermon_kstat_cntr_update()
* Context: Called from the kstat context
*/
static int
{
int status;
/*
* Extract the Hermon softstate pointer, kstat data, pointer to the
* kstat info structure, and pointer to the hki_ib_perfcnt[] array
* from the input parameters. Note: For warlock purposes, these
* parameters are all accessed only in this routine and are,
* therefore, protected by the lock used by the kstat framework.
*/
/*
* Depending on whether we are reading the "pic" counters or
* writing the "pcr" control register, we need to handle and
* fill in the kstat data appropriately.
*
* If this is a write to the "pcr", then extract the value from
* the kstat data and store it in the kstat info structure.
*
* Otherwise, if this is a read of the "pic" counter(s), then
* extract the register offset, size, and mask values from the
* ib_perf[] array. Then read the corresponding register and store
* than one port is configured.
*/
if (rw == KSTAT_WRITE) {
/* Update the stored "pcr" value */
return (0);
} else {
/*
* Get the current "pcr" value and extract the lower
* portion (corresponding to the counters for "pic0")
*/
/*
* Fill in the "pic0" counter, corresponding to port 1.
* This involves reading in the current value in the register
* and calculating how many events have happened since this
* register was last polled. Then we save away the current
* value for the counter and increment the "pic0" total by
* the number of new events.
*/
if (status != HERMON_CMD_SUCCESS) {
return (-1);
}
switch (indx) {
case 0: /* port_xmit_data */
break;
case 1: /* port_recv_data */
break;
case 2: /* port_xmit_pkts */
break;
case 3: /* port_recv_pkts */
break;
case 4: /* port_recv_err */
break;
case 5: /* port_xmit_discards */
break;
case 6: /* vl15_dropped */
break;
case 7: /* port_xmit_wait */
break;
case 8: /* port_recv_remote_phys_err */
break;
case 9: /* port_xmit_constraint_err */
break;
case 10: /* port_recv_constraint_err */
break;
case 11: /* symbol_err_counter */
break;
case 12: /* link_err_recovery_cnt */
break;
case 13: /* link_downed_cnt */
break;
case 14: /* excessive_buffer_overruns */
break;
case 15: /* local_link_integrity_err */
break;
case 16: /* clear_pic */
tmp = 0; /* XXX */
break;
default:
}
/*
* If necessary, fill in the "pic1" counter for port 2.
* This works the same as above except that we extract the
* upper bits (corresponding to the counters for "pic1")
*/
if (numports == HERMON_MAX_PORTS) {
if (status != HERMON_CMD_SUCCESS) {
return (-1);
}
switch (indx) {
case 0: /* port_xmit_data */
break;
case 1: /* port_recv_data */
break;
case 2: /* port_xmit_pkts */
break;
case 3: /* port_recv_pkts */
break;
case 4: /* port_recv_err */
break;
case 5: /* port_xmit_discards */
break;
case 6: /* vl15_dropped */
break;
case 7: /* port_xmit_wait */
break;
case 8: /* port_recv_remote_phys_err */
break;
case 9: /* port_xmit_constraint_err */
break;
case 10: /* port_recv_constraint_err */
break;
case 11: /* symbol_err_counter */
break;
case 12: /* link_err_recovery_cnt */
break;
case 13: /* link_downed_cnt */
break;
case 14: /* excessive_buffer_overruns */
break;
case 15: /* local_link_integrity_err */
break;
case 16: /* clear_pic */
tmp = 0; /* XXX */
break;
default:
}
}
return (0);
}
}
/*
* 64 bit kstats for performance counters:
*
* Export 64 bit performance counters in kstats.
*
* If the HCA hardware supports 64 bit extended port counters, we use the
* hardware based counters. If the HCA hardware does not support extended port
* counters, we maintain 64 bit performance counters in software using the
* 32 bit hardware port counters.
*
* The software based counters are maintained as follows:
*
* We create a thread that, every one second, reads the values of 32 bit
* hardware counters and adds them to the 64 bit software counters. Immediately
* after reading, it resets the 32 bit hardware counters to zero (so that they
* start counting from zero again). At any time the current value of a counter
* is going to be the sum of the 64 bit software counter and the 32 bit
* hardware counter.
*
* Since this work need not be done if there is no consumer, by default
* we do not maintain 64 bit software counters. To enable this the consumer
* needs to write a non-zero value to the "enable" component of the of
* perf_counters kstat. Writing zero to this component will disable this work.
* NOTE: The enabling or disabling applies to software based counters only.
* Hardware based counters counters are always enabled.
*
* If performance monitor is enabled in subnet manager, the SM could
* periodically reset the hardware counters by sending perf-MADs. So only
* one of either our software 64 bit counters or the SM performance monitor
* could be enabled at the same time. However, if both of them are enabled at
* the same time we still do our best by keeping track of the values of the
* last read 32 bit hardware counters. If the current read of a 32 bit hardware
* counter is less than the last read of the counter, we ignore the current
* value and go with the last read value.
*/
/*
* hermon_kstat_perfcntr64_create()
* Context: Only called from attach() path context
*
* Create "port#/perf_counters" kstat for the specified port number.
*/
void
{
int drv_instance;
char *drv_name;
if (status == HERMON_CMD_SUCCESS) {
}
port_num);
return;
}
"enable", KSTAT_DATA_UINT32);
"xmit_data", KSTAT_DATA_UINT64);
"recv_data", KSTAT_DATA_UINT64);
"xmit_pkts", KSTAT_DATA_UINT64);
"recv_pkts", KSTAT_DATA_UINT64);
/* Install the kstat */
}
/*
* hermon_kstat_perfcntr64_read()
*
* Read the values of 32 bit hardware counters.
*
* If reset is true, reset the 32 bit hardware counters. Add the values of the
* 32 bit hardware counters to the 64 bit software counters.
*
* If reset is false, just save the values read from the 32 bit hardware
* counters in hki64_last_read[].
*
* See the general comment on the 64 bit performance counters
* regarding the use of last read 32 bit hardware counter values.
*/
static int
{
int status, i;
/* read the 32 bit hardware counters */
if (status != HERMON_CMD_SUCCESS) {
return (status);
}
if (reset) {
/* reset the hardware counters */
if (status != HERMON_CMD_SUCCESS) {
return (status);
}
/*
* Update 64 bit software counters
*/
for (i = 0; i < HERMON_PERFCNTR64_NUM_COUNTERS; i++)
ksi64->hki64_last_read[i] = 0;
} else {
/*
* Update ksi64->hki64_last_read[]
*/
}
return (HERMON_CMD_SUCCESS);
}
/*
* hermon_kstat_perfcntr64_update_thread()
* Context: Entry point for a kernel thread
*
* Maintain 64 bit performance counters in software using the 32 bit
* hardware counters.
*/
static void
{
uint_t i;
/*
* Every one second update the values 64 bit software counters
* for all ports. Exit if HERMON_PERFCNTR64_THREAD_EXIT flag is set.
*/
(void) hermon_kstat_perfcntr64_read(state,
i + 1, 1);
}
}
/* sleep for a second */
}
ksi->hki_perfcntr64_flags = 0;
}
/*
* hermon_kstat_perfcntr64_thread_create()
* Context: Called from the kstat context
*
* Create a thread that maintains 64 bit performance counters in software.
*/
static void
{
/*
* One thread per hermon instance. Don't create a thread if already
* created.
*/
}
}
/*
* hermon_kstat_perfcntr64_thread_exit()
* Context: Called from attach, detach or kstat context
*/
static void
{
/*
* Signal the thread to exit and wait until the thread exits.
*/
}
}
/*
* hermon_kstat_perfcntr64_update_ext()
* Context: Called from the kstat context
*
* Update perf_counters kstats with the values of the extended port counters
* from the hardware.
*/
static int
struct kstat_named *data)
{
/*
* The "enable" component of the kstat is the only writable kstat.
* It is a no-op when the hardware supports extended port counters.
*/
if (rw == KSTAT_WRITE)
return (0);
/*
* Read the counters and update kstats.
*/
return (EIO);
}
return (0);
}
/*
* hermon_kstat_perfcntr64_update()
* Context: Called from the kstat context
*
* See the general comment on 64 bit kstats for performance counters:
*/
static int
{
int i, thr_exit;
int rv;
return (rv);
}
/*
* 64 bit performance counters maintained by the software is not
* enabled by default. Enable them upon a writing a non-zero value
* to "enable" kstat. Disable them upon a writing zero to the
* "enable" kstat.
*/
if (rw == KSTAT_WRITE) {
if (ksi64->hki64_enabled == 0) {
/*
* Reset the hardware counters to ensure that
* the hardware counter doesn't max out
* (and hence stop counting) before we get
* a chance to reset the counter in
* hermon_kstat_perfcntr64_update_thread.
*/
return (EIO);
}
/* Enable 64 bit software counters */
for (i = 0;
i < HERMON_PERFCNTR64_NUM_COUNTERS; i++) {
ksi64->hki64_counters[i] = 0;
ksi64->hki64_last_read[i] = 0;
}
}
} else if (ksi64->hki64_enabled) {
/* Disable 64 bit software counters */
ksi64->hki64_enabled = 0;
thr_exit = 1;
i++) {
thr_exit = 0;
break;
}
}
if (thr_exit)
}
} else if (ksi64->hki64_enabled) {
/*
* Read the counters and update kstats.
*/
0) != HERMON_CMD_SUCCESS) {
return (EIO);
}
} else {
/* return 0 in kstats if not enabled */
for (i = 1; i < HERMON_PERFCNTR64_NUM_COUNTERS; i++)
}
return (0);
}