fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * CDDL HEADER START
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * The contents of this file are subject to the terms of the
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Common Development and Distribution License (the "License").
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * You may not use this file except in compliance with the License.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * See the License for the specific language governing permissions
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * and limitations under the License.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * When distributing Covered Code, include this CDDL HEADER in each
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * If applicable, add the following below this CDDL HEADER, with the
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * fields enclosed by brackets "[]" replaced with your own identifying
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * information: Portions Copyright [yyyy] [name of copyright owner]
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * CDDL HEADER END
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * Copyright (c) 2009-2010, Intel Corporation.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * All rights reserved.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Introduction
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * This file implements a CPU event notification mechanism to signal clients
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * which are interested in CPU related events.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Currently it only supports CPU idle state change events which will be
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * triggered just before CPU entering hardware idle state and just after CPU
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * wakes up from hardware idle state.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Please refer to PSARC/2009/115 for detail information.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Lock Strategy
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * 1) cpu_idle_prop_busy/free are protected by cpu_idle_prop_lock.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * 2) No protection for cpu_idle_cb_state because it's per-CPU data.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * 3) cpu_idle_cb_busy is protected by cpu_idle_cb_lock.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * 4) cpu_idle_cb_array is protected by pause_cpus/start_cpus logic.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * 5) cpu_idle_cb_max/curr are protected by both cpu_idle_cb_lock and
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * We have optimized the algorithm for hot path on read side access.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * In the current algorithm, it's lock free on read side access.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * On write side, we use pause_cpus() to keep other CPUs in the pause thread,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * which will guarantee that no other threads will access
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * cpu_idle_cb_max/curr/array data structure.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel/* Define normal state for CPU on different platforms. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * At the time of this implementation IDLE_STATE_NORMAL is defined
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * in mach_startup.c, and not in a header file. So if we find it is
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * undefined, then we set it to the value as defined in mach_startup.c
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Should it eventually be defined, we will pick it up.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * To improve cache efficiency and avoid cache false sharing, CPU idle
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * properties are grouped into cache lines as below:
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * | CPU0 | CPU1 |.........| CPUn |
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * | cache line 0 | cache line 1 |.........| cache line n |
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * | v0 | ... | vm | v0 | ... | vm |.........| v0 | ... | vm |
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * To access value of property m for CPU n, using following value as index:
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * index = seq_id_of_CPUn * CPU_IDLE_VALUE_GROUP_SIZE + m.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel (CPU_CACHE_COHERENCE_SIZE / sizeof (cpu_idle_prop_value_t))
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel/* Get callback context handle for current CPU. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ((cpu_idle_callback_context_t)(intptr_t)((cp)->cpu_seqid))
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel/* Get CPU sequential id from ctx. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel#define CPU_IDLE_CTX2CPUID(ctx) ((processorid_t)(intptr_t)(ctx))
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel/* Compute index from callback context handle. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel (((int)(intptr_t)(ctx)) * CPU_IDLE_VALUE_GROUP_SIZE)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel (&((cpu_idle_prop_impl_t *)(hdl))->value[(idx)])
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * When cpu_idle_cb_array is NULL or full, increase CPU_IDLE_ARRAY_CAPACITY_INC
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * entries every time. Here we prefer linear growth instead of exponential.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel/* Structure to maintain registered callbacks in list. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Structure to maintain registered callbacks in priority order and also
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * optimized for cache efficiency for reading access.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel/* Per-CPU state aligned to CPU_CACHE_COHERENCE_SIZE to avoid false sharing. */
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu /* Index of already invoked callbacks. */
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu /* Invoke registered callbacks if true. */
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu /* Property values are valid if true. */
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu /* Pointers to per-CPU properties. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_prop_impl_t *cpu_idle_prop_busy = NULL;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_prop_impl_t *cpu_idle_prop_free = NULL;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_cb_impl_t *cpu_idle_cb_busy = NULL;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_cb_item_t *cpu_idle_cb_array = NULL;
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * cpuset used to intercept CPUs before powering them off.
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * The control CPU sets the bit corresponding to the target CPU and waits
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * until the bit is cleared.
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * The target CPU disables interrupts before clearing corresponding bit and
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * then loops for ever.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic int cpu_idle_prop_update_intr_cnt(void *arg, uint64_t seqnum,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_prop_item_t cpu_idle_prop_array[] = {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel CPU_IDLE_PROP_TYPE_INTPTR, CPU_IDLE_PROP_IDLE_STATE,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_ENTER_TIMESTAMP,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_EXIT_TIMESTAMP,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_LAST_IDLE_TIME,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_LAST_BUSY_TIME,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_TOTAL_IDLE_TIME,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_TOTAL_BUSY_TIME,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel CPU_IDLE_PROP_TYPE_UINT64, CPU_IDLE_PROP_INTERRUPT_COUNT,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_dtrace_enter(void *arg, cpu_idle_callback_context_t ctx,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_check_wakeup_t check_func, void *check_arg)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_array[CPU_IDLE_PROP_IDX_IDLE_STATE].handle, ctx);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel DTRACE_PROBE1(idle__state__transition, uint_t, state);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_dtrace_exit(void *arg, cpu_idle_callback_context_t ctx, int flag)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel DTRACE_PROBE1(idle__state__transition, uint_t, CPU_IDLE_STATE_NORMAL);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_callback_handle_t cpu_idle_cb_handle_dtrace;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_callback_t cpu_idle_callback_dtrace = {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelextern void tlb_going_idle(void);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelextern void tlb_service(void);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_callback_handle_t cpu_idle_cb_handle_tlb;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic cpu_idle_callback_t cpu_idle_callback_tlb = {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel mutex_init(&cpu_idle_cb_lock, NULL, MUTEX_DRIVER, NULL);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel mutex_init(&cpu_idle_prop_lock, NULL, MUTEX_DRIVER, NULL);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Create internal properties. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel i < sizeof (cpu_idle_prop_array) / sizeof (cpu_idle_prop_array[0]);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel (void) cpu_idle_prop_create_property(ip->name, ip->type,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Allocate buffer and align to CPU_CACHE_COHERENCE_SIZE. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_cb_state = (cpu_idle_cb_state_t *)P2ROUNDUP(buf,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Cache frequently used property value pointers. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (sp = cpu_idle_cb_state, i = 0; i < max_ncpus; i++, sp++) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sp->v.f = CPU_IDLE_HDL2VALP(cpu_idle_prop_array[(i)].handle, idx)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ___INIT_P(idle_state, CPU_IDLE_PROP_IDX_IDLE_STATE);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ___INIT_P(enter_ts, CPU_IDLE_PROP_IDX_ENTER_TS);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ___INIT_P(last_idle, CPU_IDLE_PROP_IDX_LAST_IDLE);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ___INIT_P(last_busy, CPU_IDLE_PROP_IDX_LAST_BUSY);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ___INIT_P(total_idle, CPU_IDLE_PROP_IDX_TOTAL_IDLE);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ___INIT_P(total_busy, CPU_IDLE_PROP_IDX_TOTAL_BUSY);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ___INIT_P(last_idle, CPU_IDLE_PROP_IDX_INTR_CNT);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Register built-in callbacks. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel if (cpu_idle_register_callback(CPU_IDLE_CB_PRIO_DTRACE,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel &cpu_idle_callback_dtrace, NULL, &cpu_idle_cb_handle_dtrace) != 0) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "cpu_idle: failed to register callback for dtrace.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel if (cpu_idle_register_callback(CPU_IDLE_CB_PRIO_TLB,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel &cpu_idle_callback_tlb, NULL, &cpu_idle_cb_handle_tlb) != 0) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "cpu_idle: failed to register callback for tlb_flush.");
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * This function is called to initialize per CPU state when starting CPUs.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_cb_state[cp->cpu_seqid].v.ready = B_FALSE;
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu cpu_idle_cb_state[cp->cpu_seqid].v.enabled = B_TRUE;
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * This function is called to clean up per CPU state when stopping CPUs.
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu cpu_idle_cb_state[cp->cpu_seqid].v.enabled = B_FALSE;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_cb_state[cp->cpu_seqid].v.ready = B_FALSE;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_insert_callback(cpu_idle_cb_impl_t *cip)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Expand array if it's full.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Memory must be allocated out of pause/start_cpus() scope because
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * kmem_zalloc() can't be called with KM_SLEEP flag within that scope.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cnt_new = cpu_idle_cb_max + CPU_IDLE_ARRAY_CAPACITY_INC;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Try to acquire cpu_lock if not held yet. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Pause all other CPUs (and let them run pause thread).
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * It's guaranteed that no other threads will access cpu_idle_cb_array
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * after pause_cpus().
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Copy content to new buffer if needed. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_cb_array = (cpu_idle_cb_item_t *)buf_new;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Insert into array according to priority. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (i = cpu_idle_cb_curr; i > 0; i--) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel if (cpu_idle_cb_array[i - 1].impl->priority >= cip->priority) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_cb_array[i] = cpu_idle_cb_array[i - 1];
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_cb_array[i].enter = cip->callback->idle_enter;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_cb_array[i].exit = cip->callback->idle_exit;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Resume other CPUs from paused state if needed. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Free old resource if needed. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel kmem_free(buf_old, cnt_old * sizeof (cpu_idle_cb_item_t));
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_remove_callback(cpu_idle_cb_impl_t *cip)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Try to acquire cpu_lock if not held yet. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Pause all other CPUs.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * It's guaranteed that no other threads will access cpu_idle_cb_array
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * after pause_cpus().
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Remove cip from array. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (i = 0; i < cpu_idle_cb_curr; i++) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_cb_array[i - 1] = cpu_idle_cb_array[i];
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Reset property ready flag for all CPUs if no registered callback
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * left because cpu_idle_enter/exit will stop updating property if
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * there's no callback registered.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (sp = cpu_idle_cb_state, i = 0; i < max_ncpus; i++, sp++) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Resume other CPUs from paused state if needed. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_register_callback(uint_t prio, cpu_idle_callback_t *cbp,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* First validate parameters. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: register_callback called from callback.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: NULL parameters in register_callback.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: priority 0x%x out of range.", prio);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel } else if (cbp->version != CPU_IDLE_CALLBACK_VERS) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: callback version %d is not supported.",
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Check whether callback with priority exists if not dynamic. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_unregister_callback(cpu_idle_callback_handle_t hdl)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: unregister_callback called from callback.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: hdl is NULL in unregister_callback.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (ipp = &cpu_idle_cb_busy; *ipp != NULL; ipp = &(*ipp)->next) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: callback handle %p not found.", (void *)hdl);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_enter_state(cpu_idle_cb_state_t *sp, intptr_t state)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sp->v.enter_ts->cipv_hrtime = gethrtime_unscaled();
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sp->v.last_busy->cipv_hrtime = sp->v.enter_ts->cipv_hrtime -
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sp->v.total_busy->cipv_hrtime += sp->v.last_busy->cipv_hrtime;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sp->v.idle_state->cipv_intptr = CPU_IDLE_STATE_NORMAL;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sp->v.exit_ts->cipv_hrtime = gethrtime_unscaled();
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sp->v.last_idle->cipv_hrtime = sp->v.exit_ts->cipv_hrtime -
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sp->v.total_idle->cipv_hrtime += sp->v.last_idle->cipv_hrtime;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_check_wakeup_t check_func, void *check_arg)
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu /* Intercept CPU at a safe point before powering off it. */
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu if (CPU_IN_SET(cpu_idle_intercept_set, CPU->cpu_id)) {
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu CPUSET_ATOMIC_DEL(cpu_idle_intercept_set, CPU->cpu_id);
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu /*CONSTCOND*/
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * On x86, cpu_idle_enter can be called from idle thread with either
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * interrupts enabled or disabled, so we need to make sure interrupts
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * are disabled here.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * On SPARC, cpu_idle_enter will be called from idle thread with
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * interrupt disabled, so no special handling necessary.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Skip calling callback if state is not ready for current CPU. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (i = 0, cip = cpu_idle_cb_array; i < cpu_idle_cb_curr; i++, cip++) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Increase index so corresponding idle_exit callback
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * will be invoked should interrupt happen during
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * idle_enter callback.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Call idle_enter callback function if it's not NULL. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cip->enter(cip->arg, ctx, check_func, check_arg);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * cpu_idle_enter runs with interrupts
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * disabled, so the idle_enter callbacks will
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * also be called with interrupts disabled.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * It is permissible for the callbacks to
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * enable the interrupts, if they can also
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * handle the condition if the interrupt
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * However, if an interrupt occurs and we
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * return here without dealing with it, we
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * return to the cpu_idle_enter() caller
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * with an EBUSY, and the caller will not
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * enter the idle state.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * We detect the interrupt, by checking the
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * index value of the state pointer. If it
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * is not the index we incremented above,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * then it was cleared while processing
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * the interrupt.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Also note, that at this point of the code
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * the normal index value will be one greater
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * than the variable 'i' in the loop, as it
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * hasn't yet been incremented.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * On SPARC, cpu_idle_exit will only be called from idle thread
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * with interrupt disabled.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * On x86, cpu_idle_exit will be called from idle thread or interrupt
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * handler. When called from interrupt handler, interrupts will be
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * disabled. When called from idle thread, interrupts may be disabled
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * or enabled.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Called from interrupt, interrupts are already disabled. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * return if cpu_idle_exit already called or
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * there is no registered callback.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Called from idle thread, need to disable interrupt. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * Allocate property structure in group of CPU_IDLE_VALUE_GROUP_SIZE to improve
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * cache efficiency. To simplify implementation, allocated memory for property
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel * structure won't be freed.
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel prop = kmem_zalloc(sizeof (*prop) * CPU_IDLE_VALUE_GROUP_SIZE,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel sz = sizeof (*valp) * CPU_IDLE_VALUE_GROUP_SIZE * max_ncpus;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (i = 0; i < CPU_IDLE_VALUE_GROUP_SIZE; i++, prop++, valp++) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_create_property(const char *name, cpu_idle_prop_type_t type,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_update_t update, void *arg, cpu_idle_prop_handle_t *hdlp)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: NULL parameters in create_property.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: property %s already exists.", name);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_destroy_property(cpu_idle_prop_handle_t hdl)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: hdl is NULL in destroy_property.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (propp = &cpu_idle_prop_busy; *propp != NULL;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_create_handle(const char *name, cpu_idle_prop_handle_t *hdlp)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: NULL parameters in create_handle.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Hold one refcount on object. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_destroy_handle(cpu_idle_prop_handle_t hdl)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel "!cpu_event: hdl is NULL in destroy_handle.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* Release refcnt held in create_handle. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_get_type(cpu_idle_prop_handle_t hdl)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_get_name(cpu_idle_prop_handle_t hdl)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_get_value(cpu_idle_prop_handle_t hdl,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_callback_context_t ctx, cpu_idle_prop_value_t *valp)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cmn_err(CE_NOTE, "!cpu_event: NULL parameters in prop_get.");
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel /* CPU's idle enter timestamp as sequence number. */
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel (uint64_t)sp->v.enter_ts->cipv_hrtime, &prop->value[idx]);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_get_uint32(cpu_idle_prop_handle_t hdl,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_get_uint64(cpu_idle_prop_handle_t hdl,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_get_intptr(cpu_idle_prop_handle_t hdl,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_get_hrtime(cpu_idle_prop_handle_t hdl,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_set_value(cpu_idle_prop_handle_t hdl,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_callback_context_t ctx, cpu_idle_prop_value_t val)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelcpu_idle_prop_set_all(cpu_idle_prop_handle_t hdl, cpu_idle_prop_value_t val)
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel for (i = 0; i < max_ncpus; i++) {
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishelstatic int cpu_idle_prop_update_intr_cnt(void *arg, uint64_t seqnum,
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel ASSERT(cp != NULL && cp->cpu_seqid < max_ncpus);
fb2caebe9e38ee2e6e469d5136fb247faaa7299bRandy Fishel cpu_idle_prop_array[CPU_IDLE_PROP_IDX_IDLE_STATE].handle,
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * Intercept CPU at a safe point in idle() before powering it off.
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu ASSERT(cpu_idle_cb_state[cp->cpu_seqid].v.enabled == B_FALSE);
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu /* Set flag to intercept CPU. */
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu CPUSET_ATOMIC_ADD(cpu_idle_intercept_set, cp->cpu_id);
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu /* Wake up CPU from possible sleep state. */
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu while (CPU_IN_SET(cpu_idle_intercept_set, cp->cpu_id)) {
a31148363f598def767ac48c5d82e1572e44b935Gerry Liu * Now target CPU is spinning in a pause loop with interrupts disabled.