modules/genunix/cyclic.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident   "%Z%%M% %I% %E% SMI"

#include "cyclic.h"

#define CYCLIC_TRACE

#include <mdb/mdb_modapi.h>
#include <sys/timer.h>
#include <sys/cyclic_impl.h>
#include <sys/sysmacros.h>
#include <stdio.h>

int
cyccpu_vread(cyc_cpu_t *cpu, uintptr_t addr)
{
    static int inited = 0;
    static int cyc_trace_enabled = 0;
    static size_t cyccpu_size;

    if (!inited) {
        inited = 1;
        (void) mdb_readvar(&cyc_trace_enabled, "cyc_trace_enabled");
        cyccpu_size = (cyc_trace_enabled) ? sizeof (*cpu) :
            OFFSETOF(cyc_cpu_t, cyp_trace);
    }

    if (mdb_vread(cpu, cyccpu_size, addr) == -1)
        return (-1);

    if (!cyc_trace_enabled)
        bzero(cpu->cyp_trace, sizeof (cpu->cyp_trace));

    return (0);
}

int
cyccpu_walk_init(mdb_walk_state_t *wsp)
{
    if (mdb_layered_walk("cpu", wsp) == -1) {
        mdb_warn("couldn't walk 'cpu'");
        return (WALK_ERR);
    }

    return (WALK_NEXT);
}

int
cyccpu_walk_step(mdb_walk_state_t *wsp)
{
    uintptr_t addr = (uintptr_t)((cpu_t *)wsp->walk_layer)->cpu_cyclic;
    cyc_cpu_t cpu;

    if (cyccpu_vread(&cpu, addr) == -1) {
        mdb_warn("couldn't read cyc_cpu at %p", addr);
        return (WALK_ERR);
    }

    return (wsp->walk_callback(addr, &cpu, wsp->walk_cbdata));
}

int
cycomni_walk_init(mdb_walk_state_t *wsp)
{
    cyc_id_t id;

    if (wsp->walk_addr == NULL) {
        mdb_warn("must provide a cyclic id\n");
        return (WALK_ERR);
    }

    if (mdb_vread(&id, sizeof (id), wsp->walk_addr) == -1) {
        mdb_warn("couldn't read cyc_id_t at %p", wsp->walk_addr);
        return (WALK_ERR);
    }

    if (id.cyi_cpu != NULL || id.cyi_omni_list == NULL ||
        id.cyi_omni_hdlr.cyo_online == NULL) {
        mdb_warn("%p is not an omnipresent cyclic.\n", wsp->walk_addr);
        return (WALK_ERR);
    }

    wsp->walk_addr = (uintptr_t)id.cyi_omni_list;

    return (WALK_NEXT);
}

int
cycomni_walk_step(mdb_walk_state_t *wsp)
{
    uintptr_t addr = wsp->walk_addr;
    cyc_omni_cpu_t omni;

    if (addr == NULL)
        return (WALK_DONE);

    if (mdb_vread(&omni, sizeof (omni), addr) == -1) {
        mdb_warn("couldn't read cyc_omni_cpu at %p", addr);
        return (WALK_ERR);
    }

    wsp->walk_addr = (uintptr_t)omni.cyo_next;

    return (wsp->walk_callback(addr, &omni, wsp->walk_cbdata));
}

void
cyclic_dump_node(cyc_cpu_t *cpu, cyc_index_t *heap, char **c, size_t w,
    int ndx, int l, int r, int depth)
{
    int heap_left, heap_right;
    int me;
    int i, x = l + (r - l) / 2;
    size_t n = w - (x - 1); /* n bytes left for snprintf after c[][x - 1] */

    heap_left = CYC_HEAP_LEFT(ndx);
    heap_right = CYC_HEAP_RIGHT(ndx);
    me = heap[ndx];

    if (ndx >= cpu->cyp_nelems)
        return;

    if (me < 10) {
        (void) mdb_snprintf(&c[depth][x - 1], n, " %d", me);
    } else if (me >= 100) {
        (void) mdb_snprintf(&c[depth][x - 1], n, "%3d", me);
    } else {
        (void) mdb_snprintf(&c[depth][x - 1], n, "%s%2d%s",
            CYC_HEAP_LEFT(CYC_HEAP_PARENT(ndx)) == ndx ? " " : "", me,
            CYC_HEAP_LEFT(CYC_HEAP_PARENT(ndx)) == ndx ? "" : " ");
    }

    if (r - l > 5) {
        c[++depth][x] = '|';
        depth++;

        for (i = l + (r - l) / 4; i < r - (r - l) / 4; i++)
            c[depth][i] = '-';
        c[depth][l + (r - l) / 4] = '+';
        c[depth][r - (r - l) / 4 - 1] = '+';
        c[depth][x] = '+';
    } else {

        if (heap_left >= cpu->cyp_nelems)
            return;

        (void) mdb_snprintf(&c[++depth][x - 1], n, "L%d",
            heap[heap_left]);

        if (heap_right >= cpu->cyp_nelems)
            return;

        (void) mdb_snprintf(&c[++depth][x - 1], n, "R%d",
            heap[heap_right]);
        return;
    }

    if (heap_left < cpu->cyp_nelems)
        cyclic_dump_node(cpu, heap, c, w, heap_left, l, x, depth + 1);

    if (heap_right < cpu->cyp_nelems)
        cyclic_dump_node(cpu, heap, c, w, heap_right, x, r, depth + 1);
}

#define LINES_PER_LEVEL 3

void
cyclic_pretty_dump(cyc_cpu_t *cpu)
{
    char **c;
    int i, j;
    int width = 80;
    int depth;
    cyc_index_t *heap;
    size_t hsize = sizeof (cyc_index_t) * cpu->cyp_size;

    heap = mdb_alloc(hsize, UM_SLEEP | UM_GC);

    if (mdb_vread(heap, hsize, (uintptr_t)cpu->cyp_heap) == -1) {
        mdb_warn("couldn't read heap at %p", (uintptr_t)cpu->cyp_heap);
        return;
    }

    for (depth = 0; (1 << depth) < cpu->cyp_nelems; depth++)
        continue;
    depth++;
    depth = (depth + 1) * LINES_PER_LEVEL;

    c = mdb_zalloc(sizeof (char *) * depth, UM_SLEEP|UM_GC);

    for (i = 0; i < depth; i++)
        c[i] = mdb_zalloc(width, UM_SLEEP|UM_GC);

    cyclic_dump_node(cpu, heap, c, width, 0, 1, width - 2, 0);

    for (i = 0; i < depth; i++) {
        int dump = 0;
        for (j = 0; j < width - 1; j++) {
            if (c[i][j] == '\0')
                c[i][j] = ' ';
            else
                dump = 1;
        }
        c[i][width - 2] = '\n';

        if (dump)
            mdb_printf(c[i]);
    }
}

/*
 * Yes, this is very weak.  Full 16-column-wide 64-bit addresses screw up
 * ::cycinfo's very carefully planned 80-column layout.  We set the column
 * width for addresses to be 11 (instead of 16), knowing that the kernel
 * heap (from which these data structures are allocated) starts at
 * 0x0000030000000000, and isn't likely to extend to 0x0000100000000000.
 */
#ifdef _LP64
#define CYC_ADDR_WIDTH  11
#else
#define CYC_ADDR_WIDTH  8
#endif

int
cycinfo(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
    cyc_cpu_t cpu;
    cpu_t c;
    cyc_index_t root, i, *heap;
    size_t hsize;
    cyclic_t *cyc;
    uintptr_t caddr;
    uint_t verbose = FALSE, Verbose = FALSE;
    int header = 0;
    cyc_level_t lev;

    if (!(flags & DCMD_ADDRSPEC)) {
        if (mdb_walk_dcmd("cyccpu", "cycinfo", argc, argv) == -1) {
            mdb_warn("can't walk 'cyccpu'");
            return (DCMD_ERR);
        }
        return (DCMD_OK);
    }

    if (mdb_getopts(argc, argv,
        'v', MDB_OPT_SETBITS, TRUE, &verbose,
        'V', MDB_OPT_SETBITS, TRUE, &Verbose, NULL) != argc)
        return (DCMD_USAGE);

    if (!DCMD_HDRSPEC(flags) && (verbose || Verbose))
        mdb_printf("\n\n");

    if (DCMD_HDRSPEC(flags) || verbose || Verbose)
        mdb_printf("%3s %*s %7s %6s %*s %15s %s\n", "CPU",
            CYC_ADDR_WIDTH, "CYC_CPU", "STATE", "NELEMS",
            CYC_ADDR_WIDTH, "ROOT", "FIRE", "HANDLER");

    if (cyccpu_vread(&cpu, addr) == -1) {
        mdb_warn("couldn't read cyc_cpu at %p", addr);
        return (DCMD_ERR);
    }

    if (mdb_vread(&c, sizeof (c), (uintptr_t)cpu.cyp_cpu) == -1) {
        mdb_warn("couldn't read cpu at %p", cpu.cyp_cpu);
        return (DCMD_ERR);
    }

    cyc = mdb_alloc(sizeof (cyclic_t) * cpu.cyp_size, UM_SLEEP | UM_GC);
    caddr = (uintptr_t)cpu.cyp_cyclics;

    if (mdb_vread(cyc, sizeof (cyclic_t) * cpu.cyp_size, caddr) == -1) {
        mdb_warn("couldn't read cyclic at %p", caddr);
        return (DCMD_ERR);
    }

    hsize = sizeof (cyc_index_t) * cpu.cyp_size;
    heap = mdb_alloc(hsize, UM_SLEEP | UM_GC);

    if (mdb_vread(heap, hsize, (uintptr_t)cpu.cyp_heap) == -1) {
        mdb_warn("couldn't read heap at %p", cpu.cyp_heap);
        return (DCMD_ERR);
    }

    root = heap[0];

    mdb_printf("%3d %0*p %7s %6d ", c.cpu_id, CYC_ADDR_WIDTH, addr,
        cpu.cyp_state == CYS_ONLINE ? "online" :
        cpu.cyp_state == CYS_OFFLINE ? "offline" :
        cpu.cyp_state == CYS_EXPANDING ? "expand" :
        cpu.cyp_state == CYS_REMOVING ? "remove" :
        cpu.cyp_state == CYS_SUSPENDED ? "suspend" : "????",
        cpu.cyp_nelems);

    if (cpu.cyp_nelems > 0)
        mdb_printf("%0*p %15llx %a\n", CYC_ADDR_WIDTH,
            caddr, cyc[root].cy_expire, cyc[root].cy_handler);
    else
        mdb_printf("%*s %15s %s\n", CYC_ADDR_WIDTH, "-", "-", "-");

    if (!verbose && !Verbose)
        return (DCMD_OK);

    mdb_printf("\n");

    cyclic_pretty_dump(&cpu);

    mdb_inc_indent(2);

    for (i = 0; i < cpu.cyp_size; i++) {
        int j;

        for (j = 0; j < cpu.cyp_size; j++) {
            if (heap[j] == i)
                break;
        }

        if (!Verbose && j >= cpu.cyp_nelems)
            continue;

        if (!header) {
            header = 1;
            mdb_printf("\n%*s %3s %4s %4s %5s %15s %7s %s\n",
                CYC_ADDR_WIDTH, "ADDR", "NDX", "HEAP", "LEVL",
                "PEND", "FIRE", "USECINT", "HANDLER");
        }

        mdb_printf("%0*p %3d ", CYC_ADDR_WIDTH,
            caddr + i * sizeof (cyclic_t), i);

        mdb_printf("%4d ", j);

        if (j >= cpu.cyp_nelems) {
            mdb_printf("%4s %5s %15s %7s %s\n", "-", "-",
                "-", "-", "-");
            continue;
        }

        mdb_printf("%4s %5d %15llx ",
            cyc[i].cy_level == CY_HIGH_LEVEL ? "high" :
            cyc[i].cy_level == CY_LOCK_LEVEL ? "lock" :
            cyc[i].cy_level == CY_LOW_LEVEL ? "low" : "????",
            cyc[i].cy_pend, cyc[i].cy_expire);

        if (cyc[i].cy_interval + cyc[i].cy_expire != INT64_MAX)
            mdb_printf("%7lld ", cyc[i].cy_interval /
                (uint64_t)(NANOSEC / MICROSEC));
        else
            mdb_printf("%7s ", "-");

        mdb_printf("%a\n", cyc[i].cy_handler);
    }


    if (!Verbose)
        goto out;

    for (lev = CY_LOW_LEVEL; lev < CY_LOW_LEVEL + CY_SOFT_LEVELS; lev++) {
        cyc_softbuf_t *softbuf = &cpu.cyp_softbuf[lev];
        char which = softbuf->cys_hard, shared = 1;
        cyc_pcbuffer_t *pc;
        size_t bufsiz;
        cyc_index_t *buf;

        if (softbuf->cys_hard != softbuf->cys_soft)
            shared = 0;

again:
        pc = &softbuf->cys_buf[which];
        bufsiz = (pc->cypc_sizemask + 1) * sizeof (cyc_index_t);
        buf = mdb_alloc(bufsiz, UM_SLEEP | UM_GC);

        if (mdb_vread(buf, bufsiz, (uintptr_t)pc->cypc_buf) == -1) {
            mdb_warn("couldn't read cypc_buf at %p", pc->cypc_buf);
            continue;
        }

        mdb_printf("\n%3s %4s %4s %4s %*s %4s %*s\n", "CPU",
            "LEVL", "USER", "NDX", CYC_ADDR_WIDTH, "ADDR", "CYC",
            CYC_ADDR_WIDTH, "CYC_ADDR", "PEND");

        for (i = 0; i <= pc->cypc_sizemask &&
            i <= pc->cypc_prodndx; i++) {
            uintptr_t cyc_addr = caddr + buf[i] * sizeof (cyclic_t);

            mdb_printf("%3d %4s %4s ", c.cpu_id,
                lev == CY_HIGH_LEVEL ? "high" :
                lev == CY_LOCK_LEVEL ? "lock" :
                lev == CY_LOW_LEVEL ? "low" : "????",
                shared ? "shrd" : which == softbuf->cys_hard ?
                "hard" : "soft");

            mdb_printf("%4d %0*p ", i, CYC_ADDR_WIDTH,
                (uintptr_t)&buf[i] - (uintptr_t)&buf[0] +
                (uintptr_t)pc->cypc_buf, buf[i],
                caddr + buf[i] * sizeof (cyclic_t));

            if (i >= pc->cypc_prodndx)
                mdb_printf("%4s %*s %5s  ",
                    "-", CYC_ADDR_WIDTH, "-", "-");
            else {
                cyclic_t c;

                if (mdb_vread(&c, sizeof (c), cyc_addr) == -1) {
                    mdb_warn("\ncouldn't read cyclic at "
                        "%p", cyc_addr);
                    continue;
                }

                mdb_printf("%4d %0*p %5d  ", buf[i],
                    CYC_ADDR_WIDTH, cyc_addr, c.cy_pend);
            }

            if (i == (pc->cypc_consndx & pc->cypc_sizemask)) {
                mdb_printf("<-- consndx");
                if (i == (pc->cypc_prodndx & pc->cypc_sizemask))
                    mdb_printf(",prodndx");
                mdb_printf("\n");
                continue;
            }

            if (i == (pc->cypc_prodndx & pc->cypc_sizemask)) {
                mdb_printf("<-- prodndx\n");
                continue;
            }
            mdb_printf("\n");

            if (i >= pc->cypc_prodndx)
                break;
        }

        if (!shared && which == softbuf->cys_hard) {
            which = softbuf->cys_soft;
            goto again;
        }
    }

out:
    mdb_dec_indent(2);
    return (DCMD_OK);
}

int
cyctrace_walk_init(mdb_walk_state_t *wsp)
{
    cyc_cpu_t *cpu;
    int i;

    cpu = mdb_zalloc(sizeof (cyc_cpu_t), UM_SLEEP);

    if (wsp->walk_addr == NULL) {
        /*
         * If an address isn't provided, we'll use the passive buffer.
         */
        GElf_Sym sym;
        cyc_tracebuf_t *tr = &cpu->cyp_trace[0];
        uintptr_t addr;

        if (mdb_lookup_by_name("cyc_ptrace", &sym) == -1) {
            mdb_warn("couldn't find passive buffer");
            return (-1);
        }

        addr = (uintptr_t)sym.st_value;

        if (mdb_vread(tr, sizeof (cyc_tracebuf_t), addr) == -1) {
            mdb_warn("couldn't read passive buffer");
            return (-1);
        }

        wsp->walk_addr = addr - offsetof(cyc_cpu_t, cyp_trace[0]);
    } else {
        if (cyccpu_vread(cpu, wsp->walk_addr) == -1) {
            mdb_warn("couldn't read cyc_cpu at %p", wsp->walk_addr);
            mdb_free(cpu, sizeof (cyc_cpu_t));
            return (-1);
        }
    }

    for (i = 0; i < CY_LEVELS; i++) {
        if (cpu->cyp_trace[i].cyt_ndx-- == 0)
            cpu->cyp_trace[i].cyt_ndx = CY_NTRACEREC - 1;
    }

    wsp->walk_data = cpu;

    return (0);
}

int
cyctrace_walk_step(mdb_walk_state_t *wsp)
{
    cyc_cpu_t *cpu = wsp->walk_data;
    cyc_tracebuf_t *buf = cpu->cyp_trace;
    hrtime_t latest = 0;
    int i, ndx, new_ndx, lev, rval;
    uintptr_t addr;

    for (i = 0; i < CY_LEVELS; i++) {
        if ((ndx = buf[i].cyt_ndx) == -1)
            continue;

        /*
         * Account for NPT.
         */
        buf[i].cyt_buf[ndx].cyt_tstamp <<= 1;
        buf[i].cyt_buf[ndx].cyt_tstamp >>= 1;

        if (buf[i].cyt_buf[ndx].cyt_tstamp > latest) {
            latest = buf[i].cyt_buf[ndx].cyt_tstamp;
            lev = i;
        }
    }

    /*
     * If we didn't find one, we're done.
     */
    if (latest == 0)
        return (-1);

    buf = &buf[lev];
    ndx = buf->cyt_ndx;
    addr = wsp->walk_addr +
        (uintptr_t)&(buf->cyt_buf[ndx]) - (uintptr_t)cpu;

    rval = wsp->walk_callback(addr, &buf->cyt_buf[ndx], wsp->walk_cbdata);

    new_ndx = ndx == 0 ? CY_NTRACEREC - 1 : ndx - 1;

    if (buf->cyt_buf[new_ndx].cyt_tstamp != 0 &&
        buf->cyt_buf[new_ndx].cyt_tstamp > buf->cyt_buf[ndx].cyt_tstamp)
        new_ndx = -1;

    buf->cyt_ndx = new_ndx;

    return (rval);
}

void
cyctrace_walk_fini(mdb_walk_state_t *wsp)
{
    cyc_cpu_t *cpu = wsp->walk_data;

    mdb_free(cpu, sizeof (cyc_cpu_t));
}

#define WHYLEN  17

int
cyctrace_walk(uintptr_t addr, const cyc_tracerec_t *rec, cyc_cpu_t *cpu)
{
    int i;
    char c[WHYLEN];

    for (i = 0; cpu != NULL && i < CY_LEVELS; i++)
        if (addr < (uintptr_t)&cpu->cyp_trace[i + 1].cyt_buf[0])
            break;

    (void) mdb_readstr(c, WHYLEN, (uintptr_t)rec->cyt_why);

    mdb_printf("%08p %4s %15llx %-*s %15llx %15llx\n",
        addr & UINT_MAX, cpu == NULL ? "pasv" :
        i == CY_HIGH_LEVEL ? "high" : i == CY_LOCK_LEVEL ? "lock" :
        i == CY_LOW_LEVEL ? "low" : "????", rec->cyt_tstamp, WHYLEN, c,
        rec->cyt_arg0, rec->cyt_arg1);

    return (0);
}

/*ARGSUSED*/
int
cyctrace(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
    if (!(flags & DCMD_ADDRSPEC) || argc != 0)
        addr = NULL;

    if (mdb_pwalk("cyctrace", (mdb_walk_cb_t)cyctrace_walk,
        (void *)addr, addr) == -1) {
        mdb_warn("couldn't walk cyctrace");
        return (DCMD_ERR);
    }

    return (DCMD_OK);
}

int
cyccover_comp(const void *l, const void *r)
{
    cyc_coverage_t *lhs = (cyc_coverage_t *)l;
    cyc_coverage_t *rhs = (cyc_coverage_t *)r;

    char ly[WHYLEN], ry[WHYLEN];

    if (rhs->cyv_why == lhs->cyv_why)
        return (0);

    if (rhs->cyv_why == NULL)
        return (-1);

    if (lhs->cyv_why == NULL)
        return (1);

    (void) mdb_readstr(ly, WHYLEN, (uintptr_t)lhs->cyv_why);
    (void) mdb_readstr(ry, WHYLEN, (uintptr_t)rhs->cyv_why);

    return (strcmp(ly, ry));
}

/*ARGSUSED*/
int
cyccover(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
    cyc_coverage_t cv[CY_NCOVERAGE];
    char c[WHYLEN];
    GElf_Sym sym;
    int i;

    if ((flags & DCMD_ADDRSPEC) || argc != 0)
        return (DCMD_USAGE);

    if (mdb_lookup_by_name("cyc_coverage", &sym) == -1) {
        mdb_warn("couldn't find coverage information");
        return (DCMD_ABORT);
    }

    addr = (uintptr_t)sym.st_value;

    if (mdb_vread(cv, sizeof (cyc_coverage_t) * CY_NCOVERAGE, addr) == -1) {
        mdb_warn("couldn't read coverage array at %p", addr);
        return (DCMD_ABORT);
    }

    mdb_printf("%-*s %8s %8s %8s %15s %15s\n",
        WHYLEN, "POINT", "HIGH", "LOCK", "LOW/PASV", "ARG0", "ARG1");

    qsort(cv, CY_NCOVERAGE, sizeof (cyc_coverage_t), cyccover_comp);

    for (i = 0; i < CY_NCOVERAGE; i++) {
        if (cv[i].cyv_why != NULL) {
            (void) mdb_readstr(c, WHYLEN, (uintptr_t)cv[i].cyv_why);
            mdb_printf("%-*s %8d %8d %8d %15llx %15llx\n",
                WHYLEN, c,
                cv[i].cyv_count[CY_HIGH_LEVEL],
                cv[i].cyv_count[CY_LOCK_LEVEL],
                cv[i].cyv_passive_count != 0 ?
                cv[i].cyv_passive_count :
                cv[i].cyv_count[CY_LOW_LEVEL],
                cv[i].cyv_arg0, cv[i].cyv_arg1);
        }
    }

    return (DCMD_OK);
}

/*ARGSUSED*/
int
cyclic(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
    cyclic_t cyc;

    if (!(flags & DCMD_ADDRSPEC) || argc != 0)
        return (DCMD_USAGE);

    if (DCMD_HDRSPEC(flags))
        mdb_printf("%?s %4s %5s %5s %15s %7s %s\n", "ADDR", "LEVL",
            "PEND", "FLAGS", "FIRE", "USECINT", "HANDLER");

    if (mdb_vread(&cyc, sizeof (cyclic_t), addr) == -1) {
        mdb_warn("couldn't read cyclic at %p", addr);
        return (DCMD_ERR);
    }

    mdb_printf("%0?p %4s %5d  %04x %15llx %7lld %a\n", addr,
        cyc.cy_level == CY_HIGH_LEVEL ? "high" :
        cyc.cy_level == CY_LOCK_LEVEL ? "lock" :
        cyc.cy_level == CY_LOW_LEVEL ? "low" : "????",
        cyc.cy_pend, cyc.cy_flags, cyc.cy_expire,
        cyc.cy_interval / (uint64_t)(NANOSEC / MICROSEC),
        cyc.cy_handler);

    return (DCMD_OK);
}

static int
cycid_cpu(cyc_cpu_t *addr, int ndx)
{
    cyc_cpu_t cpu;
    cpu_t c;
    uintptr_t caddr;
    cyclic_t cyc;

    if (cyccpu_vread(&cpu, (uintptr_t)addr) == -1) {
        mdb_warn("couldn't read cyc_cpu at %p", addr);
        return (DCMD_ERR);
    }

    if (mdb_vread(&c, sizeof (c), (uintptr_t)cpu.cyp_cpu) == -1) {
        mdb_warn("couldn't read cpu at %p", cpu.cyp_cpu);
        return (DCMD_ERR);
    }

    caddr = (uintptr_t)cpu.cyp_cyclics + ndx * sizeof (cyclic_t);

    if (mdb_vread(&cyc, sizeof (cyc), caddr) == -1) {
        mdb_warn("couldn't read cyclic at %p", caddr);
        return (DCMD_ERR);
    }

    mdb_printf("%4d %3d %?p %a\n", c.cpu_id, ndx, caddr, cyc.cy_handler);

    return (DCMD_OK);
}

/*ARGSUSED*/
static int
cycid_walk_omni(uintptr_t addr, const cyc_omni_cpu_t *omni, int *ignored)
{
    mdb_printf("%?s        ");
    cycid_cpu(omni->cyo_cpu, omni->cyo_ndx);

    return (WALK_NEXT);
}

/*ARGSUSED*/
int
cycid(uintptr_t addr, uint_t flags, int ac, const mdb_arg_t *av)
{
    cyc_id_t id;

    if (!(flags & DCMD_ADDRSPEC)) {
        if (mdb_walk_dcmd("cyclic_id_cache", "cycid", ac, av) == -1) {
            mdb_warn("can't walk cyclic_id_cache");
            return (DCMD_ERR);
        }

        return (DCMD_OK);
    }

    if (DCMD_HDRSPEC(flags)) {
        mdb_printf("%?s %4s %3s %?s %s\n", "ADDR", "CPU", "NDX",
            "CYCLIC", "HANDLER");
    }

    if (mdb_vread(&id, sizeof (id), addr) == -1) {
        mdb_warn("couldn't read cyc_id_t at %p", addr);
        return (DCMD_ERR);
    }

    if (id.cyi_cpu == NULL) {
        /*
         * This is an omnipresent cyclic.
         */
        mdb_printf("%?p %4s %3s %?s %a\n", addr, "omni", "-", "-",
            id.cyi_omni_hdlr.cyo_online);
        mdb_printf("%?s    |\n", "");
        mdb_printf("%?s    +-->%4s %3s %?s %s\n", "",
            "CPU", "NDX", "CYCLIC", "HANDLER");

        if (mdb_pwalk("cycomni",
            (mdb_walk_cb_t)cycid_walk_omni, NULL, addr) == -1) {
            mdb_warn("couldn't walk cycomni for %p", addr);
            return (DCMD_ERR);
        }

        mdb_printf("\n");

        return (DCMD_OK);
    }

    mdb_printf("%?p ", addr);

    return (cycid_cpu(id.cyi_cpu, id.cyi_ndx));
}