common/os/mem_cage.c

	mem_cage.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident   "%Z%%M% %I% %E% SMI"

#include <sys/types.h>
#include <sys/param.h>
#include <sys/thread.h>
#include <sys/proc.h>
#include <sys/callb.h>
#include <sys/vnode.h>
#include <sys/debug.h>
#include <sys/systm.h>      /* for bzero */
#include <sys/memlist.h>
#include <sys/cmn_err.h>
#include <sys/sysmacros.h>
#include <sys/vmsystm.h>    /* for NOMEMWAIT() */
#include <sys/atomic.h>     /* used to update kcage_freemem */
#include <sys/kmem.h>       /* for kmem_reap */
#include <sys/errno.h>
#include <sys/mem_cage.h>
#include <vm/seg_kmem.h>
#include <vm/page.h>
#include <vm/hat.h>
#include <sys/mem_config.h>
#include <sys/lgrp.h>

extern pri_t maxclsyspri;

#ifdef DEBUG
#define KCAGE_STATS
#endif

#ifdef KCAGE_STATS

#define KCAGE_STATS_VERSION 9   /* can help report generators */
#define KCAGE_STATS_NSCANS 256  /* depth of scan statistics buffer */

struct kcage_stats_scan {
    /* managed by KCAGE_STAT_* macros */
    clock_t scan_lbolt;
    uint_t  scan_id;

    /* set in kcage_cageout() */
    uint_t  kt_passes;
    clock_t kt_ticks;
    pgcnt_t kt_kcage_freemem_start;
    pgcnt_t kt_kcage_freemem_end;
    pgcnt_t kt_freemem_start;
    pgcnt_t kt_freemem_end;
    uint_t  kt_examined;
    uint_t  kt_cantlock;
    uint_t  kt_gotone;
    uint_t  kt_gotonefree;
    uint_t  kt_skiplevel;
    uint_t  kt_skipshared;
    uint_t  kt_skiprefd;
    uint_t  kt_destroy;

    /* set in kcage_invalidate_page() */
    uint_t  kip_reloclocked;
    uint_t  kip_relocmod;
    uint_t  kip_destroy;
    uint_t  kip_nomem;
    uint_t  kip_demotefailed;

    /* set in kcage_expand() */
    uint_t  ke_wanted;
    uint_t  ke_examined;
    uint_t  ke_lefthole;
    uint_t  ke_gotone;
    uint_t  ke_gotonefree;
};

struct kcage_stats {
    /* managed by KCAGE_STAT_* macros */
    uint_t  version;
    uint_t  size;

    /* set in kcage_cageout */
    uint_t  kt_wakeups;
    uint_t  kt_scans;
    uint_t  kt_cageout_break;

    /* set in kcage_expand */
    uint_t  ke_calls;
    uint_t  ke_nopfn;
    uint_t  ke_nopaget;
    uint_t  ke_isnoreloc;
    uint_t  ke_deleting;
    uint_t  ke_lowfreemem;
    uint_t  ke_terminate;

    /* set in kcage_freemem_add() */
    uint_t  kfa_trottlewake;

    /* set in kcage_freemem_sub() */
    uint_t  kfs_cagewake;

    /* set in kcage_create_throttle */
    uint_t  kct_calls;
    uint_t  kct_cageout;
    uint_t  kct_critical;
    uint_t  kct_exempt;
    uint_t  kct_cagewake;
    uint_t  kct_wait;
    uint_t  kct_progress;
    uint_t  kct_noprogress;
    uint_t  kct_timeout;

    /* set in kcage_cageout_wakeup */
    uint_t  kcw_expandearly;

    /* managed by KCAGE_STAT_* macros */
    uint_t  scan_array_size;
    uint_t  scan_index;
    struct kcage_stats_scan scans[KCAGE_STATS_NSCANS];
};

static struct kcage_stats kcage_stats;
static struct kcage_stats_scan kcage_stats_scan_zero;

/*
 * No real need for atomics here. For the most part the incs and sets are
 * done by the kernel cage thread. There are a few that are done by any
 * number of other threads. Those cases are noted by comments.
 */
#define KCAGE_STAT_INCR(m)  kcage_stats.m++

#define KCAGE_STAT_NINCR(m, v) kcage_stats.m += (v)

#define KCAGE_STAT_INCR_SCAN(m) \
    KCAGE_STAT_INCR(scans[kcage_stats.scan_index].m)

#define KCAGE_STAT_NINCR_SCAN(m, v) \
    KCAGE_STAT_NINCR(scans[kcage_stats.scan_index].m, v)

#define KCAGE_STAT_SET(m, v)    kcage_stats.m = (v)

#define KCAGE_STAT_SETZ(m, v)   \
    if (kcage_stats.m == 0) kcage_stats.m = (v)

#define KCAGE_STAT_SET_SCAN(m, v)   \
    KCAGE_STAT_SET(scans[kcage_stats.scan_index].m, v)

#define KCAGE_STAT_SETZ_SCAN(m, v)  \
    KCAGE_STAT_SETZ(scans[kcage_stats.scan_index].m, v)

#define KCAGE_STAT_INC_SCAN_INDEX \
    KCAGE_STAT_SET_SCAN(scan_lbolt, lbolt); \
    KCAGE_STAT_SET_SCAN(scan_id, kcage_stats.scan_index); \
    kcage_stats.scan_index = \
    (kcage_stats.scan_index + 1) % KCAGE_STATS_NSCANS; \
    kcage_stats.scans[kcage_stats.scan_index] = kcage_stats_scan_zero

#define KCAGE_STAT_INIT_SCAN_INDEX \
    kcage_stats.version = KCAGE_STATS_VERSION; \
    kcage_stats.size = sizeof (kcage_stats); \
    kcage_stats.scan_array_size = KCAGE_STATS_NSCANS; \
    kcage_stats.scan_index = 0

#else /* KCAGE_STATS */

#define KCAGE_STAT_INCR(v)
#define KCAGE_STAT_NINCR(m, v)
#define KCAGE_STAT_INCR_SCAN(v)
#define KCAGE_STAT_NINCR_SCAN(m, v)
#define KCAGE_STAT_SET(m, v)
#define KCAGE_STAT_SETZ(m, v)
#define KCAGE_STAT_SET_SCAN(m, v)
#define KCAGE_STAT_SETZ_SCAN(m, v)
#define KCAGE_STAT_INC_SCAN_INDEX
#define KCAGE_STAT_INIT_SCAN_INDEX

#endif /* KCAGE_STATS */

static kmutex_t kcage_throttle_mutex;   /* protects kcage_throttle_cv */
static kcondvar_t kcage_throttle_cv;

static kmutex_t kcage_cageout_mutex;    /* protects cv and ready flag */
static kcondvar_t kcage_cageout_cv; /* cageout thread naps here */
static int kcage_cageout_ready;     /* nonzero when cageout thread ready */
kthread_id_t kcage_cageout_thread;  /* to aid debugging */

static kmutex_t kcage_range_mutex;  /* proctects kcage_glist elements */

/*
 * Cage expansion happens within a range.
 */
struct kcage_glist {
    struct kcage_glist  *next;
    pfn_t           base;
    pfn_t           lim;
    pfn_t           curr;
    int         decr;
};

static struct kcage_glist *kcage_glist;
static struct kcage_glist *kcage_current_glist;

/*
 * The firstfree element is provided so that kmem_alloc can be avoided
 * until that cage has somewhere to go. This is not currently a problem
 * as early kmem_alloc's use BOP_ALLOC instead of page_create_va.
 */
static struct kcage_glist kcage_glist_firstfree;
static struct kcage_glist *kcage_glist_freelist = &kcage_glist_firstfree;

/*
 * Miscellaneous forward references
 */
static struct kcage_glist *kcage_glist_alloc(void);
static int kcage_glist_delete(pfn_t, pfn_t, struct kcage_glist **);
static void kcage_cageout(void);
static int kcage_invalidate_page(page_t *, pgcnt_t *);
static int kcage_setnoreloc_pages(page_t *, se_t);

/*
 * Kernel Memory Cage counters and thresholds.
 */
int kcage_on = 0;
pgcnt_t kcage_freemem;
pgcnt_t kcage_needfree;
pgcnt_t kcage_lotsfree;
pgcnt_t kcage_desfree;
pgcnt_t kcage_minfree;
pgcnt_t kcage_throttlefree;
int kcage_maxwait = 10; /* in seconds */

/* when we use lp for kmem we start the cage at a higher initial value */
pgcnt_t kcage_kmemlp_mincage;

#ifdef DEBUG
pgcnt_t kcage_pagets;
#define KCAGEPAGETS_INC()   kcage_pagets++
#else
#define KCAGEPAGETS_INC()
#endif

/*
 * Startup and Dynamic Reconfiguration interfaces.
 * kcage_range_lock()
 * kcage_range_unlock()
 * kcage_range_islocked()
 * kcage_range_add()
 * kcage_range_del()
 * kcage_init()
 * kcage_set_thresholds()
 */

int
kcage_range_trylock(void)
{
    return (mutex_tryenter(&kcage_range_mutex));
}

void
kcage_range_lock(void)
{
    mutex_enter(&kcage_range_mutex);
}

void
kcage_range_unlock(void)
{
    mutex_exit(&kcage_range_mutex);
}

int
kcage_range_islocked(void)
{
    return (MUTEX_HELD(&kcage_range_mutex));
}

/*
 * Called from page_get_contig_pages to get the approximate kcage pfn range
 * for exclusion from search for contiguous pages. This routine is called
 * without kcage_range lock (kcage routines can call page_get_contig_pages
 * through page_relocate) and with the assumption, based on kcage_range_add,
 * that kcage_current_glist always contain a valid pointer.
 */

int
kcage_current_pfn(pfn_t *pfncur)
{
    struct kcage_glist *lp = kcage_current_glist;

    ASSERT(kcage_on);

    ASSERT(lp != NULL);

    *pfncur = lp->curr;

    return (lp->decr);
}

int
kcage_range_init(struct memlist *ml, int decr)
{
    int ret = 0;

    ASSERT(kcage_range_islocked());

    if (decr) {
        while (ml->next != NULL)
            ml = ml->next;
    }

    while (ml != NULL) {
        ret = kcage_range_add(btop(ml->address), btop(ml->size), decr);
        if (ret)
            break;

        ml = (decr ? ml->prev : ml->next);
    }

    return (ret);
}

/*
 * Third arg controls direction of growth: 0: increasing pfns,
 * 1: decreasing.
 * Calls to add and delete must be protected by calls to
 * kcage_range_lock() and kcage_range_unlock().
 */
int
kcage_range_add(pfn_t base, pgcnt_t npgs, int decr)
{
    struct kcage_glist *new, **lpp;
    pfn_t lim;

    ASSERT(kcage_range_islocked());

    ASSERT(npgs != 0);
    if (npgs == 0)
        return (EINVAL);

    lim = base + npgs;

    ASSERT(lim > base);
    if (lim <= base)
        return (EINVAL);

    new = kcage_glist_alloc();
    if (new == NULL) {
        return (ENOMEM);
    }

    new->base = base;
    new->lim = lim;
    new->decr = decr;
    if (new->decr != 0)
        new->curr = new->lim;
    else
        new->curr = new->base;
    /*
     * Any overlapping existing ranges are removed by deleting
     * from the new list as we search for the tail.
     */
    lpp = &kcage_glist;
    while (*lpp != NULL) {
        int ret;
        ret = kcage_glist_delete((*lpp)->base, (*lpp)->lim, &new);
        if (ret != 0)
            return (ret);
        lpp = &(*lpp)->next;
    }

    *lpp = new;

    if (kcage_current_glist == NULL) {
        kcage_current_glist = kcage_glist;
    }

    return (0);
}

/*
 * Calls to add and delete must be protected by calls to
 * kcage_range_lock() and kcage_range_unlock().
 */
int
kcage_range_delete(pfn_t base, pgcnt_t npgs)
{
    struct kcage_glist *lp;
    pfn_t lim;

    ASSERT(kcage_range_islocked());

    ASSERT(npgs != 0);
    if (npgs == 0)
        return (EINVAL);

    lim = base + npgs;

    ASSERT(lim > base);
    if (lim <= base)
        return (EINVAL);

    /*
     * Check if the delete is OK first as a number of elements
     * might be involved and it will be difficult to go
     * back and undo (can't just add the range back in).
     */
    for (lp = kcage_glist; lp != NULL; lp = lp->next) {
        /*
         * If there have been no pages allocated from this
         * element, we don't need to check it.
         */
        if ((lp->decr == 0 && lp->curr == lp->base) ||
            (lp->decr != 0 && lp->curr == lp->lim))
            continue;
        /*
         * If the element does not overlap, its OK.
         */
        if (base >= lp->lim || lim <= lp->base)
            continue;
        /*
         * Overlapping element: Does the range to be deleted
         * overlap the area already used? If so fail.
         */
        if (lp->decr == 0 && base < lp->curr && lim >= lp->base) {
            return (EBUSY);
        }
        if (lp->decr != 0 && base < lp->lim && lim >= lp->curr) {
            return (EBUSY);
        }
    }
    return (kcage_glist_delete(base, lim, &kcage_glist));
}

/*
 * Calls to add and delete must be protected by calls to
 * kcage_range_lock() and kcage_range_unlock().
 * This routine gets called after successful Solaris memory
 * delete operation from DR post memory delete routines.
 */
int
kcage_range_delete_post_mem_del(pfn_t base, pgcnt_t npgs)
{
    pfn_t lim;

    ASSERT(kcage_range_islocked());

    ASSERT(npgs != 0);
    if (npgs == 0)
        return (EINVAL);

    lim = base + npgs;

    ASSERT(lim > base);
    if (lim <= base)
        return (EINVAL);

    return (kcage_glist_delete(base, lim, &kcage_glist));
}

/*
 * No locking is required here as the whole operation is covered
 * by the kcage_range_lock().
 */
static struct kcage_glist *
kcage_glist_alloc(void)
{
    struct kcage_glist *new;

    if ((new = kcage_glist_freelist) != NULL) {
        kcage_glist_freelist = new->next;
        bzero(new, sizeof (*new));
    } else {
        new = kmem_zalloc(sizeof (struct kcage_glist), KM_NOSLEEP);
    }
    return (new);
}

static void
kcage_glist_free(struct kcage_glist *lp)
{
    lp->next = kcage_glist_freelist;
    kcage_glist_freelist = lp;
}

static int
kcage_glist_delete(pfn_t base, pfn_t lim, struct kcage_glist **lpp)
{
    struct kcage_glist *lp, *prev = *lpp;

    while ((lp = *lpp) != NULL) {
        if (lim > lp->base && base < lp->lim) {
            /* The delete range overlaps this element. */
            if (base <= lp->base && lim >= lp->lim) {
                /* Delete whole element. */
                *lpp = lp->next;
                if (lp == kcage_current_glist) {
                    /* This can never happen. */
                    ASSERT(kcage_current_glist != prev);
                    kcage_current_glist = prev;
                }
                kcage_glist_free(lp);
                continue;
            }

            /* Partial delete. */
            if (base > lp->base && lim < lp->lim) {
                struct kcage_glist *new;

                /*
                 * Remove a section from the middle,
                 * need to allocate a new element.
                 */
                new = kcage_glist_alloc();
                if (new == NULL) {
                    return (ENOMEM);
                }

                /*
                 * Tranfser unused range to new.
                 * Edit lp in place to preserve
                 * kcage_current_glist.
                 */
                new->decr = lp->decr;
                if (new->decr != 0) {
                    new->base = lp->base;
                    new->lim = base;
                    new->curr = base;

                    lp->base = lim;
                } else {
                    new->base = lim;
                    new->lim = lp->lim;
                    new->curr = new->base;

                    lp->lim = base;
                }

                /* Insert new. */
                new->next = lp->next;
                lp->next = new;
                lpp = &lp->next;
            } else {
                /* Delete part of current block. */
                if (base > lp->base) {
                    ASSERT(lim >= lp->lim);
                    ASSERT(base < lp->lim);
                    if (lp->decr != 0 &&
                        lp->curr == lp->lim)
                        lp->curr = base;
                    lp->lim = base;
                } else {
                    ASSERT(base <= lp->base);
                    ASSERT(lim > lp->base);
                    if (lp->decr == 0 &&
                        lp->curr == lp->base)
                        lp->curr = lim;
                    lp->base = lim;
                }
            }
        }
        prev = *lpp;
        lpp = &(*lpp)->next;
    }

    return (0);
}

/*
 * The caller of kcage_get_pfn must hold the kcage_range_lock to make
 * sure that there are no concurrent calls. The same lock
 * must be obtained for range add and delete by calling
 * kcage_range_lock() and kcage_range_unlock().
 */
static pfn_t
kcage_get_pfn(void)
{
    struct kcage_glist *lp;
    pfn_t pfn;

    ASSERT(kcage_range_islocked());

    lp = kcage_current_glist;
    while (lp != NULL) {
        if (lp->decr != 0) {
            if (lp->curr != lp->base) {
                pfn = --lp->curr;
                return (pfn);
            }
        } else {
            if (lp->curr != lp->lim) {
                pfn = lp->curr++;
                return (pfn);
            }
        }

        lp = lp->next;
        if (lp)
            kcage_current_glist = lp;
    }

    return (PFN_INVALID);
}

/*
 * Walk the physical address space of the cage.
 * This routine does not guarantee to return PFNs in the order
 * in which they were allocated to the cage. Instead, it walks
 * each range as they appear on the growth list returning the PFNs
 * range in ascending order.
 *
 * To begin scanning at lower edge of cage, reset should be nonzero.
 * To step through cage, reset should be zero.
 *
 * PFN_INVALID will be returned when the upper end of the cage is
 * reached -- indicating a full scan of the cage has been completed since
 * previous reset. PFN_INVALID will continue to be returned until
 * kcage_walk_cage is reset.
 *
 * It is possible to receive a PFN_INVALID result on reset if a growth
 * list is not installed or if none of the PFNs in the installed list have
 * been allocated to the cage. In otherwords, there is no cage.
 *
 * Caller need not hold kcage_range_lock while calling this function
 * as the front part of the list is static - pages never come out of
 * the cage.
 *
 * The caller is expected to only be kcage_cageout().
 */
static pfn_t
kcage_walk_cage(int reset)
{
    static struct kcage_glist *lp = NULL;
    static pfn_t pfn;

    if (reset)
        lp = NULL;
    if (lp == NULL) {
        lp = kcage_glist;
        pfn = PFN_INVALID;
    }
again:
    if (pfn == PFN_INVALID) {
        if (lp == NULL)
            return (PFN_INVALID);

        if (lp->decr != 0) {
            /*
             * In this range the cage grows from the highest
             * address towards the lowest.
             * Arrange to return pfns from curr to lim-1,
             * inclusive, in ascending order.
             */

            pfn = lp->curr;
        } else {
            /*
             * In this range the cage grows from the lowest
             * address towards the highest.
             * Arrange to return pfns from base to curr,
             * inclusive, in ascending order.
             */

            pfn = lp->base;
        }
    }

    if (lp->decr != 0) {        /* decrementing pfn */
        if (pfn == lp->lim) {
            /* Don't go beyond the static part of the glist. */
            if (lp == kcage_current_glist)
                lp = NULL;
            else
                lp = lp->next;
            pfn = PFN_INVALID;
            goto again;
        }

        ASSERT(pfn >= lp->curr && pfn < lp->lim);
    } else {            /* incrementing pfn */
        if (pfn == lp->curr) {
            /* Don't go beyond the static part of the glist. */
            if (lp == kcage_current_glist)
                lp = NULL;
            else
                lp = lp->next;
            pfn = PFN_INVALID;
            goto again;
        }

        ASSERT(pfn >= lp->base && pfn < lp->curr);
    }

    return (pfn++);
}

/*
 * Callback functions for to recalc cage thresholds after
 * Kphysm memory add/delete operations.
 */
/*ARGSUSED*/
static void
kcage_kphysm_postadd_cb(void *arg, pgcnt_t delta_pages)
{
    kcage_recalc_thresholds();
}

/*ARGSUSED*/
static int
kcage_kphysm_predel_cb(void *arg, pgcnt_t delta_pages)
{
    /* TODO: when should cage refuse memory delete requests? */
    return (0);
}

/*ARGSUSED*/
static  void
kcage_kphysm_postdel_cb(void *arg, pgcnt_t delta_pages, int cancelled)
{
    kcage_recalc_thresholds();
}

static kphysm_setup_vector_t kcage_kphysm_vectors = {
    KPHYSM_SETUP_VECTOR_VERSION,
    kcage_kphysm_postadd_cb,
    kcage_kphysm_predel_cb,
    kcage_kphysm_postdel_cb
};

/*
 * This is called before a CPR suspend and after a CPR resume.  We have to
 * turn off kcage_cageout_ready before a suspend, and turn it back on after a
 * restart.
 */
/*ARGSUSED*/
static boolean_t
kcage_cageout_cpr(void *arg, int code)
{
    if (code == CB_CODE_CPR_CHKPT) {
        ASSERT(kcage_cageout_ready);
        kcage_cageout_ready = 0;
        return (B_TRUE);
    } else if (code == CB_CODE_CPR_RESUME) {
        ASSERT(kcage_cageout_ready == 0);
        kcage_cageout_ready = 1;
        return (B_TRUE);
    }
    return (B_FALSE);
}

/*
 * kcage_recalc_preferred_size() increases initial cage size to improve large
 * page availability when lp for kmem is enabled and kpr is disabled
 */
static pgcnt_t
kcage_recalc_preferred_size(pgcnt_t preferred_size)
{
    if (SEGKMEM_USE_LARGEPAGES && segkmem_reloc == 0) {
        pgcnt_t lpmincage = kcage_kmemlp_mincage;
        if (lpmincage == 0) {
            lpmincage = MIN(P2ROUNDUP(((physmem * PAGESIZE) / 8),
                segkmem_heaplp_quantum), 0x40000000UL) / PAGESIZE;
        }
        kcage_kmemlp_mincage = MIN(lpmincage,
                (segkmem_kmemlp_max / PAGESIZE));
        preferred_size = MAX(kcage_kmemlp_mincage, preferred_size);
    }
    return (preferred_size);
}

/*
 * Kcage_init() builds the cage and initializes the cage thresholds.
 * The size of the cage is determined by the argument preferred_size.
 * or the actual amount of memory, whichever is smaller.
 */
void
kcage_init(pgcnt_t preferred_size)
{
    pgcnt_t wanted;
    pfn_t pfn;
    page_t *pp;
    extern struct vnode kvp;
    extern void page_list_noreloc_startup(page_t *);

    ASSERT(!kcage_on);
    ASSERT(kcage_range_islocked());

    /* increase preferred cage size for lp for kmem */
    preferred_size = kcage_recalc_preferred_size(preferred_size);

    /* Debug note: initialize this now so early expansions can stat */
    KCAGE_STAT_INIT_SCAN_INDEX;

    /*
     * Initialize cage thresholds and install kphysm callback.
     * If we can't arrange to have the thresholds track with
     * available physical memory, then the cage thresholds may
     * end up over time at levels that adversly effect system
     * performance; so, bail out.
     */
    kcage_recalc_thresholds();
    if (kphysm_setup_func_register(&kcage_kphysm_vectors, NULL)) {
        ASSERT(0);      /* Catch this in DEBUG kernels. */
        return;
    }

    /*
     * Limit startup cage size within the range of kcage_minfree
     * and availrmem, inclusively.
     */
    wanted = MIN(MAX(preferred_size, kcage_minfree), availrmem);

    /*
     * Construct the cage. PFNs are allocated from the glist. It
     * is assumed that the list has been properly ordered for the
     * platform by the platform code. Typically, this is as simple
     * as calling kcage_range_init(phys_avail, decr), where decr is
     * 1 if the kernel has been loaded into upper end of physical
     * memory, or 0 if the kernel has been loaded at the low end.
     *
     * Note: it is assumed that we are in the startup flow, so there
     * is no reason to grab the page lock.
     */
    kcage_freemem = 0;
    pfn = PFN_INVALID;          /* prime for alignment test */
    while (wanted != 0) {
        if ((pfn = kcage_get_pfn()) == PFN_INVALID)
            break;

        if ((pp = page_numtopp_nolock(pfn)) != NULL) {
            KCAGEPAGETS_INC();
            /*
             * Set the noreloc state on the page.
             * If the page is free and not already
             * on the noreloc list then move it.
             */
            if (PP_ISFREE(pp)) {
                if (PP_ISNORELOC(pp) == 0)
                    page_list_noreloc_startup(pp);
            } else {
                ASSERT(pp->p_szc == 0);
                PP_SETNORELOC(pp);
            }
        }

        wanted -= 1;
    }

    /*
     * Need to go through and find kernel allocated pages
     * and capture them into the Cage.  These will primarily
     * be pages gotten through boot_alloc().
     */
    if (kvp.v_pages) {

        pp = kvp.v_pages;
        do {
            ASSERT(!PP_ISFREE(pp));
            ASSERT(pp->p_szc == 0);
            PP_SETNORELOC(pp);
        } while ((pp = pp->p_vpnext) != kvp.v_pages);

    }

    kcage_on = 1;

    /*
     * CB_CL_CPR_POST_KERNEL is the class that executes from cpr_suspend()
     * after the cageout thread is blocked, and executes from cpr_resume()
     * before the cageout thread is restarted.  By executing in this class,
     * we are assured that the kernel cage thread won't miss wakeup calls
     * and also CPR's larger kmem_alloc requests will not fail after
     * CPR shuts down the cageout kernel thread.
     */
    (void) callb_add(kcage_cageout_cpr, NULL, CB_CL_CPR_POST_KERNEL,
        "cageout");

    /*
     * Coalesce pages to improve large page availability. A better fix
     * would to coalesce pages as they are included in the cage
     */
    if (SEGKMEM_USE_LARGEPAGES) {
        extern void page_freelist_coalesce_all(int mnode);
        extern int max_mem_nodes;
        int mnode, max_mnodes = max_mem_nodes;
        for (mnode = 0; mnode < max_mnodes; mnode++) {
            page_freelist_coalesce_all(mnode);
        }
    }
}

void
kcage_recalc_thresholds()
{
    static int first = 1;
    static pgcnt_t init_lotsfree;
    static pgcnt_t init_desfree;
    static pgcnt_t init_minfree;
    static pgcnt_t init_throttlefree;

    /* TODO: any reason to take more care than this with live editing? */
    mutex_enter(&kcage_cageout_mutex);
    mutex_enter(&freemem_lock);

    if (first) {
        first = 0;
        init_lotsfree = kcage_lotsfree;
        init_desfree = kcage_desfree;
        init_minfree = kcage_minfree;
        init_throttlefree = kcage_throttlefree;
    } else {
        kcage_lotsfree = init_lotsfree;
        kcage_desfree = init_desfree;
        kcage_minfree = init_minfree;
        kcage_throttlefree = init_throttlefree;
    }

    if (kcage_lotsfree == 0)
        kcage_lotsfree = MAX(32, total_pages / 256);

    if (kcage_minfree == 0)
        kcage_minfree = MAX(32, kcage_lotsfree / 2);

    if (kcage_desfree == 0)
        kcage_desfree = MAX(32, kcage_minfree);

    if (kcage_throttlefree == 0)
        kcage_throttlefree = MAX(32, kcage_minfree / 2);

    mutex_exit(&freemem_lock);
    mutex_exit(&kcage_cageout_mutex);

    if (kcage_cageout_ready) {
        if (kcage_freemem < kcage_desfree)
            kcage_cageout_wakeup();

        if (kcage_needfree) {
            mutex_enter(&kcage_throttle_mutex);
            cv_broadcast(&kcage_throttle_cv);
            mutex_exit(&kcage_throttle_mutex);
        }
    }
}

/*
 * Pageout interface:
 * kcage_cageout_init()
 */
void
kcage_cageout_init()
{
    if (kcage_on) {
        mutex_enter(&kcage_cageout_mutex);

        kcage_cageout_thread = thread_create(NULL, 0, kcage_cageout,
            NULL, 0, proc_pageout, TS_RUN, maxclsyspri - 1);

        mutex_exit(&kcage_cageout_mutex);
    }
}


/*
 * VM Interfaces:
 * kcage_create_throttle()
 * kcage_freemem_add()
 * kcage_freemem_sub()
 */

/*
 * Wakeup cageout thread and throttle waiting for the number of pages
 * requested to become available.  For non-critical requests, a
 * timeout is added, since freemem accounting is separate from cage
 * freemem accounting: it's possible for us to get stuck and not make
 * forward progress even though there was sufficient freemem before
 * arriving here.
 */
int
kcage_create_throttle(pgcnt_t npages, int flags)
{
    int niter = 0;
    pgcnt_t lastfree;
    int enough = kcage_freemem > kcage_throttlefree + npages;

    KCAGE_STAT_INCR(kct_calls);     /* unprotected incr. */

    kcage_cageout_wakeup();         /* just to be sure */
    KCAGE_STAT_INCR(kct_cagewake);      /* unprotected incr. */

    /*
     * Obviously, we can't throttle the cageout thread since
     * we depend on it.  We also can't throttle the panic thread.
     */
    if (curthread == kcage_cageout_thread || panicstr) {
        KCAGE_STAT_INCR(kct_cageout);   /* unprotected incr. */
        return (KCT_CRIT);
    }

    /*
     * Don't throttle threads which are critical for proper
     * vm management if we're above kcage_throttlefree or
     * if freemem is very low.
     */
    if (NOMEMWAIT()) {
        if (enough) {
            KCAGE_STAT_INCR(kct_exempt);    /* unprotected incr. */
            return (KCT_CRIT);
        } else if (freemem < minfree) {
            KCAGE_STAT_INCR(kct_critical);  /* unprotected incr. */
            return (KCT_CRIT);
        }
    }

    /*
     * Don't throttle real-time threads.
     */
    if (DISP_PRIO(curthread) > maxclsyspri) {
        KCAGE_STAT_INCR(kct_exempt);    /* unprotected incr. */
        return (KCT_CRIT);
    }

    /*
     * Cause all other threads (which are assumed to not be
     * critical to cageout) to wait here until their request
     * can be satisfied. Be a little paranoid and wake the
     * kernel cage on each loop through this logic.
     */
    while (kcage_freemem < kcage_throttlefree + npages) {
        ASSERT(kcage_on);

        lastfree = kcage_freemem;

        if (kcage_cageout_ready) {
            mutex_enter(&kcage_throttle_mutex);

            kcage_needfree += npages;
            KCAGE_STAT_INCR(kct_wait);

            kcage_cageout_wakeup();
            KCAGE_STAT_INCR(kct_cagewake);

            cv_wait(&kcage_throttle_cv, &kcage_throttle_mutex);

            kcage_needfree -= npages;

            mutex_exit(&kcage_throttle_mutex);
        } else {
            /*
             * NOTE: atomics are used just in case we enter
             * mp operation before the cageout thread is ready.
             */
            atomic_add_long(&kcage_needfree, npages);

            kcage_cageout_wakeup();
            KCAGE_STAT_INCR(kct_cagewake);  /* unprotected incr. */

            atomic_add_long(&kcage_needfree, -npages);
        }

        if ((flags & PG_WAIT) == 0) {
            if (kcage_freemem > lastfree) {
                KCAGE_STAT_INCR(kct_progress);
                niter = 0;
            } else {
                KCAGE_STAT_INCR(kct_noprogress);
                if (++niter >= kcage_maxwait) {
                    KCAGE_STAT_INCR(kct_timeout);
                    return (KCT_FAILURE);
                }
            }
        }
    }
    return (KCT_NONCRIT);
}

void
kcage_freemem_add(pgcnt_t npages)
{
    extern void wakeup_pcgs(void);

    atomic_add_long(&kcage_freemem, npages);

    wakeup_pcgs();  /* wakeup threads in pcgs() */

    if (kcage_needfree != 0 &&
        kcage_freemem >= (kcage_throttlefree + kcage_needfree)) {

        mutex_enter(&kcage_throttle_mutex);
        cv_broadcast(&kcage_throttle_cv);
        KCAGE_STAT_INCR(kfa_trottlewake);
        mutex_exit(&kcage_throttle_mutex);
    }
}

void
kcage_freemem_sub(pgcnt_t npages)
{
    atomic_add_long(&kcage_freemem, -npages);

    if (kcage_freemem < kcage_desfree) {
        kcage_cageout_wakeup();
        KCAGE_STAT_INCR(kfs_cagewake); /* unprotected incr. */
    }
}

/*
 * return 0 on failure and 1 on success.
 */
static int
kcage_setnoreloc_pages(page_t *rootpp, se_t se)
{
    pgcnt_t npgs, i;
    page_t *pp;
    pfn_t rootpfn = page_pptonum(rootpp);
    uint_t szc;

    ASSERT(!PP_ISFREE(rootpp));
    ASSERT(PAGE_LOCKED_SE(rootpp, se));
    if (!group_page_trylock(rootpp, se)) {
        return (0);
    }
    szc = rootpp->p_szc;
    if (szc == 0) {
        /*
         * The szc of a locked page can only change for pages that are
         * non-swapfs (i.e. anonymous memory) file system pages.
         */
        ASSERT(rootpp->p_vnode != NULL &&
            rootpp->p_vnode != &kvp &&
            !IS_SWAPFSVP(rootpp->p_vnode));
        PP_SETNORELOC(rootpp);
        return (1);
    }
    npgs = page_get_pagecnt(szc);
    ASSERT(IS_P2ALIGNED(rootpfn, npgs));
    pp = rootpp;
    for (i = 0; i < npgs; i++, pp = page_next(pp)) {
        ASSERT(PAGE_LOCKED_SE(pp, se));
        ASSERT(!PP_ISFREE(pp));
        ASSERT(pp->p_szc == szc);
        PP_SETNORELOC(pp);
    }
    group_page_unlock(rootpp);
    return (1);
}

/*
 * Attempt to convert page to a caged page (set the P_NORELOC flag).
 * If successful and pages is free, move page to the tail of whichever
 * list it is on.
 * Returns:
 *   EBUSY  page already locked, assimilated but not free.
 *   ENOMEM page assimilated, but memory too low to relocate. Page not free.
 *   EAGAIN page not assimilated. Page not free.
 *   ERANGE page assimilated. Page not root.
 *   0      page assimilated. Page free.
 *   *nfreedp number of pages freed.
 * NOTE: With error codes ENOMEM, EBUSY, and 0 (zero), there is no way
 * to distinguish between a page that was already a NORELOC page from
 * those newly converted to NORELOC pages by this invocation of
 * kcage_assimilate_page.
 */
static int
kcage_assimilate_page(page_t *pp, pgcnt_t *nfreedp)
{
    if (page_trylock(pp, SE_EXCL)) {
        if (PP_ISNORELOC(pp)) {
check_free_and_return:
            if (PP_ISFREE(pp)) {
                page_unlock(pp);
                *nfreedp = 0;
                return (0);
            } else {
                page_unlock(pp);
                return (EBUSY);
            }
            /*NOTREACHED*/
        }
    } else {
        if (page_trylock(pp, SE_SHARED)) {
            if (PP_ISNORELOC(pp))
                goto check_free_and_return;
        } else
            return (EAGAIN);

        if (!PP_ISFREE(pp)) {
            page_unlock(pp);
            return (EAGAIN);
        }

        /*
         * Need to upgrade the lock on it and set the NORELOC
         * bit. If it is free then remove it from the free
         * list so that the platform free list code can keep
         * NORELOC pages where they should be.
         */
        /*
         * Before doing anything, get the exclusive lock.
         * This may fail (eg ISM pages are left shared locked).
         * If the page is free this will leave a hole in the
         * cage. There is no solution yet to this.
         */
        if (!page_tryupgrade(pp)) {
            page_unlock(pp);
            return (EAGAIN);
        }
    }

    ASSERT(PAGE_EXCL(pp));

    if (PP_ISFREE(pp)) {
        int which = PP_ISAGED(pp) ? PG_FREE_LIST : PG_CACHE_LIST;

        page_list_sub(pp, which | PG_LIST_ISCAGE);
        ASSERT(pp->p_szc == 0);
        PP_SETNORELOC(pp);
        page_list_add(pp, which | PG_LIST_TAIL | PG_LIST_ISCAGE);

        page_unlock(pp);
        *nfreedp = 1;
        return (0);
    } else {
        if (pp->p_szc != 0) {
            if (!kcage_setnoreloc_pages(pp, SE_EXCL)) {
                page_unlock(pp);
                return (EAGAIN);
            }
            ASSERT(PP_ISNORELOC(pp));
        } else {
            PP_SETNORELOC(pp);
        }
        return (kcage_invalidate_page(pp, nfreedp));
    }
    /*NOTREACHED*/
}

static int
kcage_expand()
{
    int did_something = 0;

    spgcnt_t wanted;
    pfn_t pfn;
    page_t *pp;
    /* TODO: we don't really need n any more? */
    pgcnt_t n;
    pgcnt_t nf, nfreed;

    /*
     * Expand the cage if available cage memory is really low. Calculate
     * the amount required to return kcage_freemem to the level of
     * kcage_lotsfree, or to satisfy throttled requests, whichever is
     * more.  It is rare for their sum to create an artificial threshold
     * above kcage_lotsfree, but it is possible.
     *
     * Exit early if expansion amount is equal to or less than zero.
     * (<0 is possible if kcage_freemem rises suddenly.)
     *
     * Exit early when the global page pool (apparently) does not
     * have enough free pages to page_relocate() even a single page.
     */
    wanted = MAX(kcage_lotsfree, kcage_throttlefree + kcage_needfree)
        - kcage_freemem;
    if (wanted <= 0)
        return (0);
    else if (freemem < pageout_reserve + 1) {
        KCAGE_STAT_INCR(ke_lowfreemem);
        return (0);
    }

    /*
     * Try to get the range list lock. If the lock is already
     * held, then don't get stuck here waiting for it.
     */
    if (!kcage_range_trylock())
        return (0);

    KCAGE_STAT_INCR(ke_calls);
    KCAGE_STAT_SET_SCAN(ke_wanted, (uint_t)wanted);

    /*
     * Assimilate more pages from the global page pool into the cage.
     */
    n = 0;              /* number of pages PP_SETNORELOC'd */
    nf = 0;             /* number of those actually free */
    while (kcage_on && nf < wanted) {
        pfn = kcage_get_pfn();
        if (pfn == PFN_INVALID) {   /* eek! no where to grow */
            KCAGE_STAT_INCR(ke_nopfn);
            goto terminate;
        }

        KCAGE_STAT_INCR_SCAN(ke_examined);

        if ((pp = page_numtopp_nolock(pfn)) == NULL) {
            KCAGE_STAT_INCR(ke_nopaget);
            continue;
        }
        KCAGEPAGETS_INC();
        /*
         * Sanity check. Skip this pfn if it is
         * being deleted.
         */
        if (pfn_is_being_deleted(pfn)) {
            KCAGE_STAT_INCR(ke_deleting);
            continue;
        }

        /*
         * NORELOC is only set at boot-time or by this routine
         * under the kcage_range_mutex lock which is currently
         * held. This means we can do a fast check here before
         * locking the page in kcage_assimilate_page.
         */
        if (PP_ISNORELOC(pp)) {
            KCAGE_STAT_INCR(ke_isnoreloc);
            continue;
        }

        switch (kcage_assimilate_page(pp, &nfreed)) {
            case 0:     /* assimilated, page is free */
                KCAGE_STAT_NINCR_SCAN(ke_gotonefree, nfreed);
                did_something = 1;
                nf += nfreed;
                n++;
                break;

            case EBUSY: /* assimilated, page not free */
            case ERANGE:    /* assimilated, page not root */
                KCAGE_STAT_INCR_SCAN(ke_gotone);
                did_something = 1;
                n++;
                break;

            case ENOMEM:    /* assimilated, but no mem */
                KCAGE_STAT_INCR(ke_terminate);
                did_something = 1;
                n++;
                goto terminate;

            case EAGAIN:    /* can't assimilate */
                KCAGE_STAT_INCR_SCAN(ke_lefthole);
                break;

            default:    /* catch this with debug kernels */
                ASSERT(0);
                break;
        }
    }

    /*
     * Realign cage edge with the nearest physical address
     * boundry for big pages. This is done to give us a
     * better chance of actually getting usable big pages
     * in the cage.
     */

terminate:
    kcage_range_unlock();

    return (did_something);
}

/*
 * Relocate page opp (Original Page Pointer) from cage pool to page rpp
 * (Replacement Page Pointer) in the global pool. Page opp will be freed
 * if relocation is successful, otherwise it is only unlocked.
 * On entry, page opp must be exclusively locked and not free.
 * *nfreedp: number of pages freed.
 */
static int
kcage_relocate_page(page_t *pp, pgcnt_t *nfreedp)
{
    page_t *opp = pp;
    page_t *rpp = NULL;
    spgcnt_t npgs;
    int result;

    ASSERT(!PP_ISFREE(opp));
    ASSERT(PAGE_EXCL(opp));

    result = page_relocate(&opp, &rpp, 1, 1, &npgs, NULL);
    *nfreedp = npgs;
    if (result == 0) {
        while (npgs-- > 0) {
            page_t *tpp;

            ASSERT(rpp != NULL);
            tpp = rpp;
            page_sub(&rpp, tpp);
            page_unlock(tpp);
        }

        ASSERT(rpp == NULL);

        return (0);     /* success */
    }

    page_unlock(opp);
    return (result);
}

/*
 * Based on page_invalidate_pages()
 *
 * Kcage_invalidate_page() uses page_relocate() twice. Both instances
 * of use must be updated to match the new page_relocate() when it
 * becomes available.
 *
 * Return result of kcage_relocate_page or zero if page was directly freed.
 * *nfreedp: number of pages freed.
 */
static int
kcage_invalidate_page(page_t *pp, pgcnt_t *nfreedp)
{
    int result;

#if defined(__sparc)
    extern struct vnode prom_ppages;
    ASSERT(pp->p_vnode != &prom_ppages);
#endif /* __sparc */

    ASSERT(!PP_ISFREE(pp));
    ASSERT(PAGE_EXCL(pp));

    /*
     * Is this page involved in some I/O? shared?
     * The page_struct_lock need not be acquired to
     * examine these fields since the page has an
     * "exclusive" lock.
     */
    if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
        result = kcage_relocate_page(pp, nfreedp);
#ifdef KCAGE_STATS
        if (result == 0)
            KCAGE_STAT_INCR_SCAN(kip_reloclocked);
        else if (result == ENOMEM)
            KCAGE_STAT_INCR_SCAN(kip_nomem);
#endif
        return (result);
    }

    ASSERT(pp->p_vnode->v_type != VCHR);

    /*
     * Unload the mappings and check if mod bit is set.
     */
    (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);

    if (hat_ismod(pp)) {
        result = kcage_relocate_page(pp, nfreedp);
#ifdef KCAGE_STATS
        if (result == 0)
            KCAGE_STAT_INCR_SCAN(kip_relocmod);
        else if (result == ENOMEM)
            KCAGE_STAT_INCR_SCAN(kip_nomem);
#endif
        return (result);
    }

    if (!page_try_demote_pages(pp)) {
        KCAGE_STAT_INCR_SCAN(kip_demotefailed);
        page_unlock(pp);
        return (EAGAIN);
    }

    page_destroy(pp, 0);
    KCAGE_STAT_INCR_SCAN(kip_destroy);
    *nfreedp = 1;
    return (0);
}

static void
kcage_cageout()
{
    pfn_t pfn;
    page_t *pp;
    callb_cpr_t cprinfo;
    int did_something;
    int scan_again;
    pfn_t start_pfn;
    int pass;
    int last_pass;
    int pages_skipped;
    int shared_skipped;
    uint_t shared_level = 8;
    pgcnt_t nfreed;
#ifdef KCAGE_STATS
    clock_t scan_start;
#endif

    CALLB_CPR_INIT(&cprinfo, &kcage_cageout_mutex,
        callb_generic_cpr, "cageout");

    mutex_enter(&kcage_cageout_mutex);

    pfn = PFN_INVALID;      /* force scan reset */
    start_pfn = PFN_INVALID;    /* force init with 1st cage pfn */
    kcage_cageout_ready = 1;    /* switch kcage_cageout_wakeup mode */

loop:
    /*
     * Wait here. Sooner or later, kcage_freemem_sub() will notice
     * that kcage_freemem is less than kcage_desfree. When it does
     * notice, kcage_freemem_sub() will wake us up via call to
     * kcage_cageout_wakeup().
     */
    CALLB_CPR_SAFE_BEGIN(&cprinfo);
    cv_wait(&kcage_cageout_cv, &kcage_cageout_mutex);
    CALLB_CPR_SAFE_END(&cprinfo, &kcage_cageout_mutex);

    KCAGE_STAT_INCR(kt_wakeups);
    KCAGE_STAT_SET_SCAN(kt_freemem_start, freemem);
    KCAGE_STAT_SET_SCAN(kt_kcage_freemem_start, kcage_freemem);
    pass = 0;
    last_pass = 0;

#ifdef KCAGE_STATS
    scan_start = lbolt;
#endif

again:
    if (!kcage_on)
        goto loop;

    KCAGE_STAT_INCR(kt_scans);
    KCAGE_STAT_INCR_SCAN(kt_passes);

    did_something = 0;
    pages_skipped = 0;
    shared_skipped = 0;
    while ((kcage_freemem < kcage_lotsfree || kcage_needfree) &&
        (pfn = kcage_walk_cage(pfn == PFN_INVALID)) != PFN_INVALID) {

        if (start_pfn == PFN_INVALID)
            start_pfn = pfn;
        else if (start_pfn == pfn) {
            last_pass = pass;
            pass += 1;
            /*
             * Did a complete walk of kernel cage, but didn't free
             * any pages.  If only one cpu is online then
             * stop kernel cage walk and try expanding.
             */
            if (ncpus_online == 1 && did_something == 0) {
                KCAGE_STAT_INCR(kt_cageout_break);
                break;
            }
        }

        pp = page_numtopp_nolock(pfn);
        if (pp == NULL) {
            continue;
        }

        KCAGE_STAT_INCR_SCAN(kt_examined);

        /*
         * Do a quick PP_ISNORELOC() and PP_ISFREE test outside
         * of the lock. If one is missed it will be seen next
         * time through.
         *
         * Skip non-caged-pages. These pages can exist in the cage
         * because, if during cage expansion, a page is
         * encountered that is long-term locked the lock prevents the
         * expansion logic from setting the P_NORELOC flag. Hence,
         * non-caged-pages surrounded by caged-pages.
         */
        if (!PP_ISNORELOC(pp)) {
            switch (kcage_assimilate_page(pp, &nfreed)) {
                case 0:
                    did_something = 1;
                    KCAGE_STAT_NINCR_SCAN(kt_gotonefree,
                        nfreed);
                    break;

                case EBUSY:
                case ERANGE:
                    did_something = 1;
                    KCAGE_STAT_INCR_SCAN(kt_gotone);
                    break;

                case EAGAIN:
                case ENOMEM:
                    break;

                default:
                    /* catch this with debug kernels */
                    ASSERT(0);
                    break;
            }

            continue;
        } else {
            int prm;

            if (PP_ISFREE(pp)) {
                continue;
            }

            if ((pp->p_vnode == &kvp && pp->p_lckcnt > 0) ||
                !page_trylock(pp, SE_EXCL)) {
                KCAGE_STAT_INCR_SCAN(kt_cantlock);
                continue;
            }

            /* P_NORELOC bit should not have gone away. */
            ASSERT(PP_ISNORELOC(pp));
            if (PP_ISFREE(pp) || (pp->p_vnode == &kvp &&
                pp->p_lckcnt > 0)) {
                page_unlock(pp);
                continue;
            }

            KCAGE_STAT_SET_SCAN(kt_skiplevel, shared_level);
            if (hat_page_getshare(pp) > shared_level) {
                page_unlock(pp);
                pages_skipped = 1;
                shared_skipped = 1;
                KCAGE_STAT_INCR_SCAN(kt_skipshared);
                continue;
            }

            /*
             * In pass {0, 1}, skip page if ref bit is set.
             * In pass {0, 1, 2}, skip page if mod bit is set.
             */
            prm = hat_pagesync(pp,
                HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD);

            /* On first pass ignore ref'd pages */
            if (pass <= 1 && (prm & P_REF)) {
                KCAGE_STAT_INCR_SCAN(kt_skiprefd);
                pages_skipped = 1;
                page_unlock(pp);
                continue;
            }

            /* On pass 2, page_destroy if mod bit is not set */
            if (pass <= 2) {
                if (pp->p_szc != 0 || (prm & P_MOD) ||
                    pp->p_lckcnt || pp->p_cowcnt) {
                    pages_skipped = 1;
                    page_unlock(pp);
                } else {

                    /*
                     * unload the mappings before
                     * checking if mod bit is set
                     */
                    (void) hat_pageunload(pp,
                        HAT_FORCE_PGUNLOAD);

                    /*
                     * skip this page if modified
                     */
                    if (hat_ismod(pp)) {
                        pages_skipped = 1;
                        page_unlock(pp);
                        continue;
                    }

                    KCAGE_STAT_INCR_SCAN(kt_destroy);
                    page_destroy(pp, 0);
                    did_something = 1;
                }
                continue;
            }

            if (kcage_invalidate_page(pp, &nfreed) == 0) {
                did_something = 1;
                KCAGE_STAT_NINCR_SCAN(kt_gotonefree, nfreed);
            }

            /*
             * No need to drop the page lock here.
             * Kcage_invalidate_page has done that for us
             * either explicitly or through a page_free.
             */
        }
    }

    /*
     * Expand the cage only if available cage memory is really low.
     * This test is done only after a complete scan of the cage.
     * The reason for not checking and expanding more often is to
     * avoid rapid expansion of the cage. Naturally, scanning the
     * cage takes time. So by scanning first, we use that work as a
     * delay loop in between expand decisions.
     */

    scan_again = 0;
    if (kcage_freemem < kcage_minfree || kcage_needfree) {
        /*
         * Kcage_expand() will return a non-zero value if it was
         * able to expand the cage -- whether or not the new
         * pages are free and immediately usable. If non-zero,
         * we do another scan of the cage. The pages might be
         * freed during that scan or by time we get back here.
         * If not, we will attempt another expansion.
         * However, if kcage_expand() returns zero, then it was
         * unable to expand the cage. This is the case when the
         * the growth list is exausted, therefore no work was done
         * and there is no reason to scan the cage again.
         * Note: Kernel cage scan is not repeated on single-cpu
         * system to avoid kernel cage thread hogging cpu.
         */
        if (pass <= 3 && pages_skipped && ncpus_online > 1)
            scan_again = 1;
        else
            (void) kcage_expand(); /* don't scan again */
    } else if (kcage_freemem < kcage_lotsfree) {
        /*
         * If available cage memory is less than abundant
         * and a full scan of the cage has not yet been completed,
         * or a scan has completed and some work was performed,
         * or pages were skipped because of sharing,
         * or we simply have not yet completed two passes,
         * then do another scan.
         */
        if (pass <= 2 && pages_skipped)
            scan_again = 1;
        if (pass == last_pass || did_something)
            scan_again = 1;
        else if (shared_skipped && shared_level < (8<<24)) {
            shared_level <<= 1;
            scan_again = 1;
        }
    }

    if (scan_again && ncpus_online > 1)
        goto again;
    else {
        if (shared_level > 8)
            shared_level >>= 1;

        KCAGE_STAT_SET_SCAN(kt_freemem_end, freemem);
        KCAGE_STAT_SET_SCAN(kt_kcage_freemem_end, kcage_freemem);
        KCAGE_STAT_SET_SCAN(kt_ticks, lbolt - scan_start);
        KCAGE_STAT_INC_SCAN_INDEX;
        goto loop;
    }

    /*NOTREACHED*/
}

void
kcage_cageout_wakeup()
{
    if (mutex_tryenter(&kcage_cageout_mutex)) {
        if (kcage_cageout_ready) {
            cv_signal(&kcage_cageout_cv);
        } else if (kcage_freemem < kcage_minfree || kcage_needfree) {
            /*
             * Available cage memory is really low. Time to
             * start expanding the cage. However, the
             * kernel cage thread is not yet ready to
             * do the work. Use *this* thread, which is
             * most likely to be t0, to do the work.
             */
            KCAGE_STAT_INCR(kcw_expandearly);
            (void) kcage_expand();
            KCAGE_STAT_INC_SCAN_INDEX;
        }

        mutex_exit(&kcage_cageout_mutex);
    }
    /* else, kernel cage thread is already running */
}

void
kcage_tick()
{
    /*
     * Once per second we wake up all the threads throttled
     * waiting for cage memory, in case we've become stuck
     * and haven't made forward progress expanding the cage.
     */
    if (kcage_on && kcage_cageout_ready)
        cv_broadcast(&kcage_throttle_cv);
}