os/cpupm/cpu_idle.c

	cpu_idle.c revision f34a71784df3fbc5d1227a7b6201fd318ad1667e
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright (c) 2009, Intel Corporation.
 * All rights reserved.
 */

#include <sys/x86_archext.h>
#include <sys/machsystm.h>
#include <sys/x_call.h>
#include <sys/stat.h>
#include <sys/acpi/acpi.h>
#include <sys/acpica.h>
#include <sys/cpu_acpi.h>
#include <sys/cpu_idle.h>
#include <sys/cpupm.h>
#include <sys/hpet.h>
#include <sys/archsystm.h>
#include <vm/hat_i86.h>
#include <sys/dtrace.h>
#include <sys/sdt.h>
#include <sys/callb.h>

#define CSTATE_USING_HPET       1
#define CSTATE_USING_LAT        2

extern void cpu_idle_adaptive(void);
extern uint32_t cpupm_next_cstate(cma_c_state_t *cs_data,
    cpu_acpi_cstate_t *cstates, uint32_t cs_count, hrtime_t start);

static int cpu_idle_init(cpu_t *);
static void cpu_idle_fini(cpu_t *);
static boolean_t cpu_deep_idle_callb(void *arg, int code);
static boolean_t cpu_idle_cpr_callb(void *arg, int code);
static void acpi_cpu_cstate(cpu_acpi_cstate_t *cstate);

static boolean_t cstate_use_timer(hrtime_t *lapic_expire, int timer);

/*
 * the flag of always-running local APIC timer.
 * the flag of HPET Timer use in deep cstate.
 */
static boolean_t cpu_cstate_arat = B_FALSE;
static boolean_t cpu_cstate_hpet = B_FALSE;

/*
 * Interfaces for modules implementing Intel's deep c-state.
 */
cpupm_state_ops_t cpu_idle_ops = {
    "Generic ACPI C-state Support",
    cpu_idle_init,
    cpu_idle_fini,
    NULL
};

static kmutex_t     cpu_idle_callb_mutex;
static callb_id_t   cpu_deep_idle_callb_id;
static callb_id_t   cpu_idle_cpr_callb_id;
static uint_t       cpu_idle_cfg_state;

static kmutex_t cpu_idle_mutex;

cpu_idle_kstat_t cpu_idle_kstat = {
    { "address_space_id",   KSTAT_DATA_STRING },
    { "latency",        KSTAT_DATA_UINT32 },
    { "power",      KSTAT_DATA_UINT32 },
};

/*
 * kstat update function of the c-state info
 */
static int
cpu_idle_kstat_update(kstat_t *ksp, int flag)
{
    cpu_acpi_cstate_t *cstate = ksp->ks_private;

    if (flag == KSTAT_WRITE) {
        return (EACCES);
    }

    if (cstate->cs_addrspace_id == ACPI_ADR_SPACE_FIXED_HARDWARE) {
        kstat_named_setstr(&cpu_idle_kstat.addr_space_id,
        "FFixedHW");
    } else if (cstate->cs_addrspace_id == ACPI_ADR_SPACE_SYSTEM_IO) {
        kstat_named_setstr(&cpu_idle_kstat.addr_space_id,
        "SystemIO");
    } else {
        kstat_named_setstr(&cpu_idle_kstat.addr_space_id,
        "Unsupported");
    }

    cpu_idle_kstat.cs_latency.value.ui32 = cstate->cs_latency;
    cpu_idle_kstat.cs_power.value.ui32 = cstate->cs_power;

    return (0);
}

/*
 * Used during configuration callbacks to manage implementation specific
 * details of the hardware timer used during Deep C-state.
 */
boolean_t
cstate_timer_callback(int code)
{
    if (cpu_cstate_arat) {
        return (B_TRUE);
    } else if (cpu_cstate_hpet) {
        return (hpet.callback(code));
    }
    return (B_FALSE);
}

/*
 * Some Local APIC Timers do not work during Deep C-states.
 * The Deep C-state idle function uses this function to ensure it is using a
 * hardware timer that works during Deep C-states.  This function also
 * switches the timer back to the LACPI Timer after Deep C-state.
 */
static boolean_t
cstate_use_timer(hrtime_t *lapic_expire, int timer)
{
    if (cpu_cstate_arat)
        return (B_TRUE);

    /*
     * We have to return B_FALSE if no arat or hpet support
     */
    if (!cpu_cstate_hpet)
        return (B_FALSE);

    switch (timer) {
    case CSTATE_USING_HPET:
        return (hpet.use_hpet_timer(lapic_expire));
    case CSTATE_USING_LAT:
        hpet.use_lapic_timer(*lapic_expire);
        return (B_TRUE);
    default:
        return (B_FALSE);
    }
}

/*
 * c-state wakeup function.
 * Similar to cpu_wakeup and cpu_wakeup_mwait except this function deals
 * with CPUs asleep in MWAIT, HLT, or ACPI Deep C-State.
 */
void
cstate_wakeup(cpu_t *cp, int bound)
{
    struct machcpu  *mcpu = &(cp->cpu_m);
    volatile uint32_t *mcpu_mwait = mcpu->mcpu_mwait;
    cpupart_t   *cpu_part;
    uint_t      cpu_found;
    processorid_t   cpu_sid;

    cpu_part = cp->cpu_part;
    cpu_sid = cp->cpu_seqid;
    /*
     * Clear the halted bit for that CPU since it will be woken up
     * in a moment.
     */
    if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) {
        /*
         * Clear the halted bit for that CPU since it will be
         * poked in a moment.
         */
        bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid);

        /*
         * We may find the current CPU present in the halted cpuset
         * if we're in the context of an interrupt that occurred
         * before we had a chance to clear our bit in cpu_idle().
         * Waking ourself is obviously unnecessary, since if
         * we're here, we're not halted.
         */
        if (cp != CPU) {
            /*
             * Use correct wakeup mechanism
             */
            if ((mcpu_mwait != NULL) &&
                (*mcpu_mwait == MWAIT_HALTED))
                MWAIT_WAKEUP(cp);
            else
                poke_cpu(cp->cpu_id);
        }
        return;
    } else {
        /*
         * This cpu isn't halted, but it's idle or undergoing a
         * context switch. No need to awaken anyone else.
         */
        if (cp->cpu_thread == cp->cpu_idle_thread ||
            cp->cpu_disp_flags & CPU_DISP_DONTSTEAL)
            return;
    }

    /*
     * No need to wake up other CPUs if the thread we just enqueued
     * is bound.
     */
    if (bound)
        return;


    /*
     * See if there's any other halted CPUs. If there are, then
     * select one, and awaken it.
     * It's possible that after we find a CPU, somebody else
     * will awaken it before we get the chance.
     * In that case, look again.
     */
    do {
        cpu_found = bitset_find(&cpu_part->cp_haltset);
        if (cpu_found == (uint_t)-1)
            return;

    } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset,
        cpu_found) < 0);

    /*
     * Must use correct wakeup mechanism to avoid lost wakeup of
     * alternate cpu.
     */
    if (cpu_found != CPU->cpu_seqid) {
        mcpu_mwait = cpu[cpu_found]->cpu_m.mcpu_mwait;
        if ((mcpu_mwait != NULL) && (*mcpu_mwait == MWAIT_HALTED))
            MWAIT_WAKEUP(cpu_seq[cpu_found]);
        else
            poke_cpu(cpu_seq[cpu_found]->cpu_id);
    }
}

/*
 * enter deep c-state handler
 */
static void
acpi_cpu_cstate(cpu_acpi_cstate_t *cstate)
{
    volatile uint32_t   *mcpu_mwait = CPU->cpu_m.mcpu_mwait;
    cpu_t           *cpup = CPU;
    processorid_t       cpu_sid = cpup->cpu_seqid;
    cpupart_t       *cp = cpup->cpu_part;
    hrtime_t        lapic_expire;
    uint8_t         type = cstate->cs_addrspace_id;
    uint32_t        cs_type = cstate->cs_type;
    int         hset_update = 1;
    boolean_t       using_timer;

    /*
     * Set our mcpu_mwait here, so we can tell if anyone tries to
     * wake us between now and when we call mwait.  No other cpu will
     * attempt to set our mcpu_mwait until we add ourself to the haltset.
     */
    if (mcpu_mwait) {
        if (type == ACPI_ADR_SPACE_SYSTEM_IO)
            *mcpu_mwait = MWAIT_WAKEUP_IPI;
        else
            *mcpu_mwait = MWAIT_HALTED;
    }

    /*
     * If this CPU is online, and there are multiple CPUs
     * in the system, then we should note our halting
     * by adding ourselves to the partition's halted CPU
     * bitmap. This allows other CPUs to find/awaken us when
     * work becomes available.
     */
    if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
        hset_update = 0;

    /*
     * Add ourselves to the partition's halted CPUs bitmask
     * and set our HALTED flag, if necessary.
     *
     * When a thread becomes runnable, it is placed on the queue
     * and then the halted cpuset is checked to determine who
     * (if anyone) should be awakened. We therefore need to first
     * add ourselves to the halted cpuset, and and then check if there
     * is any work available.
     *
     * Note that memory barriers after updating the HALTED flag
     * are not necessary since an atomic operation (updating the bitmap)
     * immediately follows. On x86 the atomic operation acts as a
     * memory barrier for the update of cpu_disp_flags.
     */
    if (hset_update) {
        cpup->cpu_disp_flags |= CPU_DISP_HALTED;
        bitset_atomic_add(&cp->cp_haltset, cpu_sid);
    }

    /*
     * Check to make sure there's really nothing to do.
     * Work destined for this CPU may become available after
     * this check. We'll be notified through the clearing of our
     * bit in the halted CPU bitmask, and a write to our mcpu_mwait.
     *
     * disp_anywork() checks disp_nrunnable, so we do not have to later.
     */
    if (disp_anywork()) {
        if (hset_update) {
            cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
            bitset_atomic_del(&cp->cp_haltset, cpu_sid);
        }
        return;
    }

    /*
     * We're on our way to being halted.
     *
     * The local APIC timer can stop in ACPI C2 and deeper c-states.
     * Try to program the HPET hardware to substitute for this CPU's
     * LAPIC timer.
     * cstate_use_timer() could disable the LAPIC Timer.  Make sure
     * to start the LAPIC Timer again before leaving this function.
     *
     * Disable interrupts here so we will awaken immediately after halting
     * if someone tries to poke us between now and the time we actually
     * halt.
     */
    cli();
    using_timer = cstate_use_timer(&lapic_expire, CSTATE_USING_HPET);

    /*
     * We check for the presence of our bit after disabling interrupts.
     * If it's cleared, we'll return. If the bit is cleared after
     * we check then the cstate_wakeup() will pop us out of the halted
     * state.
     *
     * This means that the ordering of the cstate_wakeup() and the clearing
     * of the bit by cpu_wakeup is important.
     * cpu_wakeup() must clear our mc_haltset bit, and then call
     * cstate_wakeup().
     * acpi_cpu_cstate() must disable interrupts, then check for the bit.
     */
    if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) {
        (void) cstate_use_timer(&lapic_expire,
            CSTATE_USING_LAT);
        sti();
        cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
        return;
    }

    /*
     * The check for anything locally runnable is here for performance
     * and isn't needed for correctness. disp_nrunnable ought to be
     * in our cache still, so it's inexpensive to check, and if there
     * is anything runnable we won't have to wait for the poke.
     */
    if (cpup->cpu_disp->disp_nrunnable != 0) {
        (void) cstate_use_timer(&lapic_expire,
            CSTATE_USING_LAT);
        sti();
        if (hset_update) {
            cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
            bitset_atomic_del(&cp->cp_haltset, cpu_sid);
        }
        return;
    }

    if (using_timer == B_FALSE) {

        (void) cstate_use_timer(&lapic_expire,
            CSTATE_USING_LAT);
        sti();

        /*
         * We are currently unable to program the HPET to act as this
         * CPU's proxy LAPIC timer.  This CPU cannot enter C2 or deeper
         * because no timer is set to wake it up while its LAPIC timer
         * stalls in deep C-States.
         * Enter C1 instead.
         *
         * cstate_wake_cpu() will wake this CPU with an IPI which
         * works with MWAIT.
         */
        i86_monitor(mcpu_mwait, 0, 0);
        if ((*mcpu_mwait & ~MWAIT_WAKEUP_IPI) == MWAIT_HALTED) {
            cpu_dtrace_idle_probe(CPU_ACPI_C1);

            tlb_going_idle();
            i86_mwait(0, 0);
            tlb_service();

            cpu_dtrace_idle_probe(CPU_ACPI_C0);
        }

        /*
         * We're no longer halted
         */
        if (hset_update) {
            cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
            bitset_atomic_del(&cp->cp_haltset, cpu_sid);
        }
        return;
    }

    cpu_dtrace_idle_probe((uint_t)cs_type);

    if (type == ACPI_ADR_SPACE_FIXED_HARDWARE) {
        /*
         * We're on our way to being halted.
         * To avoid a lost wakeup, arm the monitor before checking
         * if another cpu wrote to mcpu_mwait to wake us up.
         */
        i86_monitor(mcpu_mwait, 0, 0);
        if (*mcpu_mwait == MWAIT_HALTED) {
            uint32_t eax = cstate->cs_address;
            uint32_t ecx = 1;

            tlb_going_idle();
            i86_mwait(eax, ecx);
            tlb_service();
        }
    } else if (type == ACPI_ADR_SPACE_SYSTEM_IO) {
        uint32_t value;
        ACPI_TABLE_FADT *gbl_FADT;

        if (*mcpu_mwait == MWAIT_WAKEUP_IPI) {
            tlb_going_idle();
            (void) cpu_acpi_read_port(cstate->cs_address,
                &value, 8);
            acpica_get_global_FADT(&gbl_FADT);
            (void) cpu_acpi_read_port(
                gbl_FADT->XPmTimerBlock.Address, &value, 32);
            tlb_service();
        }
    }

    /*
     * The LAPIC timer may have stopped in deep c-state.
     * Reprogram this CPU's LAPIC here before enabling interrupts.
     */
    (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT);
    sti();

    cpu_dtrace_idle_probe(CPU_ACPI_C0);

    /*
     * We're no longer halted
     */
    if (hset_update) {
        cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
        bitset_atomic_del(&cp->cp_haltset, cpu_sid);
    }
}

/*
 * indicate when bus masters are active
 */
static uint32_t
cpu_acpi_bm_sts(void)
{
    uint32_t bm_sts = 0;

    cpu_acpi_get_register(ACPI_BITREG_BUS_MASTER_STATUS, &bm_sts);

    if (bm_sts)
        cpu_acpi_set_register(ACPI_BITREG_BUS_MASTER_STATUS, 1);

    return (bm_sts);
}

/*
 * Idle the present CPU, deep c-state is supported
 */
void
cpu_acpi_idle(void)
{
    cpu_t *cp = CPU;
    cpu_acpi_handle_t handle;
    cma_c_state_t *cs_data;
    cpu_acpi_cstate_t *cstates;
    hrtime_t start, end;
    int cpu_max_cstates;
    uint32_t cs_indx;
    uint16_t cs_type;

    cpupm_mach_state_t *mach_state =
        (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state;
    handle = mach_state->ms_acpi_handle;
    ASSERT(CPU_ACPI_CSTATES(handle) != NULL);

    cs_data = mach_state->ms_cstate.cma_state.cstate;
    cstates = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle);
    ASSERT(cstates != NULL);
    cpu_max_cstates = cpu_acpi_get_max_cstates(handle);
    if (cpu_max_cstates > CPU_MAX_CSTATES)
        cpu_max_cstates = CPU_MAX_CSTATES;
    if (cpu_max_cstates == 1) { /* no ACPI c-state data */
        (*non_deep_idle_cpu)();
        return;
    }

    start = gethrtime_unscaled();

    cs_indx = cpupm_next_cstate(cs_data, cstates, cpu_max_cstates, start);

    /*
     * OSPM uses the BM_STS bit to determine the power state to enter
     * when considering a transition to or from the C2/C3 power state.
     * if C3 is determined, bus master activity demotes the power state
     * to C2.
     */
    if ((cstates[cs_indx].cs_type >= CPU_ACPI_C3) && cpu_acpi_bm_sts())
        --cs_indx;
    cs_type = cstates[cs_indx].cs_type;

    /*
     * BM_RLD determines if the Cx power state was exited as a result of
     * bus master requests. Set this bit when using a C3 power state, and
     * clear it when using a C1 or C2 power state.
     */
    if ((CPU_ACPI_BM_INFO(handle) & BM_RLD) && (cs_type < CPU_ACPI_C3)) {
        cpu_acpi_set_register(ACPI_BITREG_BUS_MASTER_RLD, 0);
        CPU_ACPI_BM_INFO(handle) &= ~BM_RLD;
    }

    if ((!(CPU_ACPI_BM_INFO(handle) & BM_RLD)) &&
        (cs_type >= CPU_ACPI_C3)) {
        cpu_acpi_set_register(ACPI_BITREG_BUS_MASTER_RLD, 1);
        CPU_ACPI_BM_INFO(handle) |= BM_RLD;
    }

    switch (cs_type) {
    default:
        /* FALLTHROUGH */
    case CPU_ACPI_C1:
        (*non_deep_idle_cpu)();
        break;

    case CPU_ACPI_C2:
        acpi_cpu_cstate(&cstates[cs_indx]);
        break;

    case CPU_ACPI_C3:
        /*
         * recommended in ACPI spec, providing hardware mechanisms
         * to prevent master from writing to memory (UP-only)
         */
        if ((ncpus_online == 1) &&
            (CPU_ACPI_BM_INFO(handle) & BM_CTL)) {
            cpu_acpi_set_register(ACPI_BITREG_ARB_DISABLE, 1);
            CPU_ACPI_BM_INFO(handle) |= BM_ARB_DIS;
        /*
         * Today all Intel's processor support C3 share cache.
         */
        } else if (x86_vendor != X86_VENDOR_Intel) {
            __acpi_wbinvd();
        }
        acpi_cpu_cstate(&cstates[cs_indx]);
        if (CPU_ACPI_BM_INFO(handle) & BM_ARB_DIS) {
            cpu_acpi_set_register(ACPI_BITREG_ARB_DISABLE, 0);
            CPU_ACPI_BM_INFO(handle) &= ~BM_ARB_DIS;
        }
        break;
    }

    end = gethrtime_unscaled();

    /*
     * Update statistics
     */
    cpupm_wakeup_cstate_data(cs_data, end);
}

boolean_t
cpu_deep_cstates_supported(void)
{
    extern int  idle_cpu_no_deep_c;

    if (idle_cpu_no_deep_c)
        return (B_FALSE);

    if (!cpuid_deep_cstates_supported())
        return (B_FALSE);

    if (cpuid_arat_supported()) {
        cpu_cstate_arat = B_TRUE;
        return (B_TRUE);
    }

    if ((hpet.supported == HPET_FULL_SUPPORT) &&
        hpet.install_proxy()) {
        cpu_cstate_hpet = B_TRUE;
        return (B_TRUE);
    }

    return (B_FALSE);
}

/*
 * Validate that this processor supports deep cstate and if so,
 * get the c-state data from ACPI and cache it.
 */
static int
cpu_idle_init(cpu_t *cp)
{
    cpupm_mach_state_t *mach_state =
        (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state;
    cpu_acpi_handle_t handle = mach_state->ms_acpi_handle;
    cpu_acpi_cstate_t *cstate;
    char name[KSTAT_STRLEN];
    int cpu_max_cstates, i;
    ACPI_TABLE_FADT *gbl_FADT;

    /*
     * Cache the C-state specific ACPI data.
     */
    if (cpu_acpi_cache_cstate_data(handle) != 0) {
        cmn_err(CE_NOTE,
            "!cpu_idle_init: Failed to cache ACPI C-state data\n");
        cpu_idle_fini(cp);
        return (-1);
    }

    /*
     * Check the bus master arbitration control ability.
     */
    acpica_get_global_FADT(&gbl_FADT);
    if (gbl_FADT->Pm2ControlBlock && gbl_FADT->Pm2ControlLength)
        CPU_ACPI_BM_INFO(handle) |= BM_CTL;

    cstate = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle);

    cpu_max_cstates = cpu_acpi_get_max_cstates(handle);

    for (i = CPU_ACPI_C1; i <= cpu_max_cstates; i++) {
        (void) snprintf(name, KSTAT_STRLEN - 1, "c%d", cstate->cs_type);
        /*
         * Allocate, initialize and install cstate kstat
         */
        cstate->cs_ksp = kstat_create("cstate", CPU->cpu_id,
            name, "misc",
            KSTAT_TYPE_NAMED,
            sizeof (cpu_idle_kstat) / sizeof (kstat_named_t),
            KSTAT_FLAG_VIRTUAL);

        if (cstate->cs_ksp == NULL) {
            cmn_err(CE_NOTE, "kstat_create(c_state) fail");
        } else {
            cstate->cs_ksp->ks_data = &cpu_idle_kstat;
            cstate->cs_ksp->ks_lock = &cpu_idle_mutex;
            cstate->cs_ksp->ks_update = cpu_idle_kstat_update;
            cstate->cs_ksp->ks_data_size += MAXNAMELEN;
            cstate->cs_ksp->ks_private = cstate;
            kstat_install(cstate->cs_ksp);
            cstate++;
        }
    }

    cpupm_alloc_domains(cp, CPUPM_C_STATES);
    cpupm_alloc_ms_cstate(cp);

    if (cpu_deep_cstates_supported()) {
        mutex_enter(&cpu_idle_callb_mutex);
        if (cpu_deep_idle_callb_id == (callb_id_t)0)
            cpu_deep_idle_callb_id = callb_add(&cpu_deep_idle_callb,
                (void *)NULL, CB_CL_CPU_DEEP_IDLE, "cpu_deep_idle");
        if (cpu_idle_cpr_callb_id == (callb_id_t)0)
            cpu_idle_cpr_callb_id = callb_add(&cpu_idle_cpr_callb,
                (void *)NULL, CB_CL_CPR_PM, "cpu_idle_cpr");
        mutex_exit(&cpu_idle_callb_mutex);
    }

    return (0);
}

/*
 * Free resources allocated by cpu_idle_init().
 */
static void
cpu_idle_fini(cpu_t *cp)
{
    cpupm_mach_state_t *mach_state =
        (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state);
    cpu_acpi_handle_t handle = mach_state->ms_acpi_handle;
    cpu_acpi_cstate_t *cstate;
    uint_t  cpu_max_cstates, i;

    /*
     * idle cpu points back to the generic one
     */
    idle_cpu = CPU->cpu_m.mcpu_idle_cpu = non_deep_idle_cpu;
    disp_enq_thread = non_deep_idle_disp_enq_thread;

    cstate = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle);
    if (cstate) {
        cpu_max_cstates = cpu_acpi_get_max_cstates(handle);

        for (i = CPU_ACPI_C1; i <= cpu_max_cstates; i++) {
            if (cstate->cs_ksp != NULL)
                kstat_delete(cstate->cs_ksp);
            cstate++;
        }
    }

    cpupm_free_ms_cstate(cp);
    cpupm_free_domains(&cpupm_cstate_domains);
    cpu_acpi_free_cstate_data(handle);

    mutex_enter(&cpu_idle_callb_mutex);
    if (cpu_deep_idle_callb_id != (callb_id_t)0) {
        (void) callb_delete(cpu_deep_idle_callb_id);
        cpu_deep_idle_callb_id = (callb_id_t)0;
    }
    if (cpu_idle_cpr_callb_id != (callb_id_t)0) {
        (void) callb_delete(cpu_idle_cpr_callb_id);
        cpu_idle_cpr_callb_id = (callb_id_t)0;
    }
    mutex_exit(&cpu_idle_callb_mutex);
}

/*ARGSUSED*/
static boolean_t
cpu_deep_idle_callb(void *arg, int code)
{
    boolean_t rslt = B_TRUE;

    mutex_enter(&cpu_idle_callb_mutex);
    switch (code) {
    case PM_DEFAULT_CPU_DEEP_IDLE:
        /*
         * Default policy is same as enable
         */
        /*FALLTHROUGH*/
    case PM_ENABLE_CPU_DEEP_IDLE:
        if ((cpu_idle_cfg_state & CPU_IDLE_DEEP_CFG) == 0)
            break;

        if (cstate_timer_callback(PM_ENABLE_CPU_DEEP_IDLE)) {
            disp_enq_thread = cstate_wakeup;
            idle_cpu = cpu_idle_adaptive;
            cpu_idle_cfg_state &= ~CPU_IDLE_DEEP_CFG;
        } else {
            rslt = B_FALSE;
        }
        break;

    case PM_DISABLE_CPU_DEEP_IDLE:
        if (cpu_idle_cfg_state & CPU_IDLE_DEEP_CFG)
            break;

        idle_cpu = non_deep_idle_cpu;
        if (cstate_timer_callback(PM_DISABLE_CPU_DEEP_IDLE)) {
            disp_enq_thread = non_deep_idle_disp_enq_thread;
            cpu_idle_cfg_state |= CPU_IDLE_DEEP_CFG;
        }
        break;

    default:
        cmn_err(CE_NOTE, "!cpu deep_idle_callb: invalid code %d\n",
            code);
        break;
    }
    mutex_exit(&cpu_idle_callb_mutex);
    return (rslt);
}

/*ARGSUSED*/
static boolean_t
cpu_idle_cpr_callb(void *arg, int code)
{
    boolean_t rslt = B_TRUE;

    mutex_enter(&cpu_idle_callb_mutex);
    switch (code) {
    case CB_CODE_CPR_RESUME:
        if (cstate_timer_callback(CB_CODE_CPR_RESUME)) {
            /*
             * Do not enable dispatcher hooks if disabled by user.
             */
            if (cpu_idle_cfg_state & CPU_IDLE_DEEP_CFG)
                break;

            disp_enq_thread = cstate_wakeup;
            idle_cpu = cpu_idle_adaptive;
        } else {
            rslt = B_FALSE;
        }
        break;

    case CB_CODE_CPR_CHKPT:
        idle_cpu = non_deep_idle_cpu;
        disp_enq_thread = non_deep_idle_disp_enq_thread;
        (void) cstate_timer_callback(CB_CODE_CPR_CHKPT);
        break;

    default:
        cmn_err(CE_NOTE, "!cpudvr cpr_callb: invalid code %d\n", code);
        break;
    }
    mutex_exit(&cpu_idle_callb_mutex);
    return (rslt);
}

/*
 * handle _CST notification
 */
void
cpuidle_cstate_instance(cpu_t *cp)
{
#ifndef __xpv
    cpupm_mach_state_t  *mach_state =
        (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state;
    cpu_acpi_handle_t   handle;
    struct machcpu      *mcpu;
    cpuset_t        dom_cpu_set;
    kmutex_t        *pm_lock;
    int         result = 0;
    processorid_t       cpu_id;

    if (mach_state == NULL) {
        return;
    }

    ASSERT(mach_state->ms_cstate.cma_domain != NULL);
    dom_cpu_set = mach_state->ms_cstate.cma_domain->pm_cpus;
    pm_lock = &mach_state->ms_cstate.cma_domain->pm_lock;

    /*
     * Do for all the CPU's in the domain
     */
    mutex_enter(pm_lock);
    do {
        CPUSET_FIND(dom_cpu_set, cpu_id);
        if (cpu_id == CPUSET_NOTINSET)
            break;

        ASSERT(cpu_id >= 0 && cpu_id < NCPU);
        cp = cpu[cpu_id];
        mach_state = (cpupm_mach_state_t *)
            cp->cpu_m.mcpu_pm_mach_state;
        if (!(mach_state->ms_caps & CPUPM_C_STATES)) {
            mutex_exit(pm_lock);
            return;
        }
        handle = mach_state->ms_acpi_handle;
        ASSERT(handle != NULL);

        /*
         * re-evaluate cstate object
         */
        if (cpu_acpi_cache_cstate_data(handle) != 0) {
            cmn_err(CE_WARN, "Cannot re-evaluate the cpu c-state"
                " object Instance: %d", cpu_id);
        }
        mutex_enter(&cpu_lock);
        mcpu = &(cp->cpu_m);
        mcpu->max_cstates = cpu_acpi_get_max_cstates(handle);
        if (mcpu->max_cstates > CPU_ACPI_C1) {
            (void) cstate_timer_callback(
                CST_EVENT_MULTIPLE_CSTATES);
            disp_enq_thread = cstate_wakeup;
            cp->cpu_m.mcpu_idle_cpu = cpu_acpi_idle;
        } else if (mcpu->max_cstates == CPU_ACPI_C1) {
            disp_enq_thread = non_deep_idle_disp_enq_thread;
            cp->cpu_m.mcpu_idle_cpu = non_deep_idle_cpu;
            (void) cstate_timer_callback(CST_EVENT_ONE_CSTATE);
        }
        mutex_exit(&cpu_lock);

        CPUSET_ATOMIC_XDEL(dom_cpu_set, cpu_id, result);
        mutex_exit(pm_lock);
    } while (result < 0);
#endif
}

/*
 * handle the number or the type of available processor power states change
 */
void
cpuidle_manage_cstates(void *ctx)
{
    cpu_t           *cp = ctx;
    cpupm_mach_state_t  *mach_state =
        (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state;
    boolean_t       is_ready;

    if (mach_state == NULL) {
        return;
    }

    /*
     * We currently refuse to power manage if the CPU is not ready to
     * take cross calls (cross calls fail silently if CPU is not ready
     * for it).
     *
     * Additionally, for x86 platforms we cannot power manage
     * any one instance, until all instances have been initialized.
     * That's because we don't know what the CPU domains look like
     * until all instances have been initialized.
     */
    is_ready = (cp->cpu_flags & CPU_READY) && cpupm_cstate_ready();
    if (!is_ready)
        return;

    cpuidle_cstate_instance(cp);
}