common/vm/vm_seg.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright (c) 2015, Joyent, Inc.
 */

/*  Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/*    All Rights Reserved   */

/*
 * University Copyright- Copyright (c) 1982, 1986, 1988
 * The Regents of the University of California
 * All Rights Reserved
 *
 * University Acknowledgment- Portions of this document are derived from
 * software developed by the University of California, Berkeley, and its
 * contributors.
 */

/*
 * VM - segment management.
 */

#include <sys/types.h>
#include <sys/inttypes.h>
#include <sys/t_lock.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kmem.h>
#include <sys/sysmacros.h>
#include <sys/vmsystm.h>
#include <sys/tuneable.h>
#include <sys/debug.h>
#include <sys/fs/swapnode.h>
#include <sys/cmn_err.h>
#include <sys/callb.h>
#include <sys/mem_config.h>
#include <sys/mman.h>

#include <vm/hat.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <vm/seg_kmem.h>
#include <vm/seg_spt.h>
#include <vm/seg_vn.h>
#include <vm/anon.h>

/*
 * kstats for segment advise
 */
segadvstat_t segadvstat = {
    { "MADV_FREE_hit",  KSTAT_DATA_ULONG },
    { "MADV_FREE_miss", KSTAT_DATA_ULONG },
};

kstat_named_t *segadvstat_ptr = (kstat_named_t *)&segadvstat;
uint_t segadvstat_ndata = sizeof (segadvstat) / sizeof (kstat_named_t);

/*
 * entry in the segment page cache
 */
struct seg_pcache {
    struct seg_pcache   *p_hnext;   /* list for hashed blocks */
    struct seg_pcache   *p_hprev;
    pcache_link_t       p_plink;    /* per segment/amp list */
    void            *p_htag0;   /* segment/amp pointer */
    caddr_t         p_addr;     /* base address/anon_idx */
    size_t          p_len;      /* total bytes */
    size_t          p_wlen;     /* writtable bytes at p_addr */
    struct page     **p_pp;     /* pp shadow list */
    seg_preclaim_cbfunc_t   p_callback; /* reclaim callback function */
    clock_t         p_lbolt;    /* lbolt from last use */
    struct seg_phash    *p_hashp;   /* our pcache hash bucket */
    uint_t          p_active;   /* active count */
    uchar_t         p_write;    /* true if S_WRITE */
    uchar_t         p_ref;      /* reference byte */
    ushort_t        p_flags;    /* bit flags */
};

struct seg_phash {
    struct seg_pcache   *p_hnext;   /* list for hashed blocks */
    struct seg_pcache   *p_hprev;
    kmutex_t        p_hmutex;   /* protects hash bucket */
    pcache_link_t       p_halink[2];    /* active bucket linkages */
};

struct seg_phash_wired {
    struct seg_pcache   *p_hnext;   /* list for hashed blocks */
    struct seg_pcache   *p_hprev;
    kmutex_t        p_hmutex;   /* protects hash bucket */
};

/*
 * A parameter to control a maximum number of bytes that can be
 * purged from pcache at a time.
 */
#define P_MAX_APURGE_BYTES  (1024 * 1024 * 1024)

/*
 * log2(fraction of pcache to reclaim at a time).
 */
#define P_SHRINK_SHFT       (5)

/*
 * The following variables can be tuned via /etc/system.
 */

int segpcache_enabled = 1;      /* if 1, shadow lists are cached */
pgcnt_t segpcache_maxwindow = 0;    /* max # of pages that can be cached */
ulong_t segpcache_hashsize_win = 0; /* # of non wired buckets */
ulong_t segpcache_hashsize_wired = 0;   /* # of wired buckets */
int segpcache_reap_sec = 1;     /* reap check rate in secs */
clock_t segpcache_reap_ticks = 0;   /* reap interval in ticks */
int segpcache_pcp_maxage_sec = 1;   /* pcp max age in secs */
clock_t segpcache_pcp_maxage_ticks = 0; /* pcp max age in ticks */
int segpcache_shrink_shift = P_SHRINK_SHFT; /* log2 reap fraction */
pgcnt_t segpcache_maxapurge_bytes = P_MAX_APURGE_BYTES; /* max purge bytes */

static kmutex_t seg_pcache_mtx; /* protects seg_pdisabled counter */
static kmutex_t seg_pasync_mtx; /* protects async thread scheduling */
static kcondvar_t seg_pasync_cv;

#pragma align 64(pctrl1)
#pragma align 64(pctrl2)
#pragma align 64(pctrl3)

/*
 * Keep frequently used variables together in one cache line.
 */
static struct p_ctrl1 {
    uint_t p_disabled;      /* if not 0, caching temporarily off */
    pgcnt_t p_maxwin;       /* max # of pages that can be cached */
    size_t p_hashwin_sz;        /* # of non wired buckets */
    struct seg_phash *p_htabwin;    /* hash table for non wired entries */
    size_t p_hashwired_sz;      /* # of wired buckets */
    struct seg_phash_wired *p_htabwired; /* hash table for wired entries */
    kmem_cache_t *p_kmcache;    /* kmem cache for seg_pcache structs */
#ifdef _LP64
    ulong_t pad[1];
#endif /* _LP64 */
} pctrl1;

static struct p_ctrl2 {
    kmutex_t p_mem_mtx; /* protects window counter and p_halinks */
    pgcnt_t  p_locked_win;  /* # pages from window */
    pgcnt_t  p_locked;  /* # of pages cached by pagelock */
    uchar_t  p_ahcur;   /* current active links for insert/delete */
    uchar_t  p_athr_on; /* async reclaim thread is running. */
    pcache_link_t p_ahhead[2]; /* active buckets linkages */
} pctrl2;

static struct p_ctrl3 {
    clock_t p_pcp_maxage;       /* max pcp age in ticks */
    ulong_t p_athr_empty_ahb;   /* athread walk stats */
    ulong_t p_athr_full_ahb;    /* athread walk stats */
    pgcnt_t p_maxapurge_npages; /* max pages to purge at a time */
    int p_shrink_shft;      /* reap shift factor */
#ifdef _LP64
    ulong_t pad[3];
#endif /* _LP64 */
} pctrl3;

#define seg_pdisabled           pctrl1.p_disabled
#define seg_pmaxwindow          pctrl1.p_maxwin
#define seg_phashsize_win       pctrl1.p_hashwin_sz
#define seg_phashtab_win        pctrl1.p_htabwin
#define seg_phashsize_wired     pctrl1.p_hashwired_sz
#define seg_phashtab_wired      pctrl1.p_htabwired
#define seg_pkmcache            pctrl1.p_kmcache
#define seg_pmem_mtx            pctrl2.p_mem_mtx
#define seg_plocked_window      pctrl2.p_locked_win
#define seg_plocked         pctrl2.p_locked
#define seg_pahcur          pctrl2.p_ahcur
#define seg_pathr_on            pctrl2.p_athr_on
#define seg_pahhead         pctrl2.p_ahhead
#define seg_pmax_pcpage         pctrl3.p_pcp_maxage
#define seg_pathr_empty_ahb     pctrl3.p_athr_empty_ahb
#define seg_pathr_full_ahb      pctrl3.p_athr_full_ahb
#define seg_pshrink_shift       pctrl3.p_shrink_shft
#define seg_pmaxapurge_npages       pctrl3.p_maxapurge_npages

#define P_HASHWIN_MASK          (seg_phashsize_win - 1)
#define P_HASHWIRED_MASK        (seg_phashsize_wired - 1)
#define P_BASESHIFT         (6)

kthread_t *seg_pasync_thr;

extern struct seg_ops segvn_ops;
extern struct seg_ops segspt_shmops;

#define IS_PFLAGS_WIRED(flags) ((flags) & SEGP_FORCE_WIRED)
#define IS_PCP_WIRED(pcp) IS_PFLAGS_WIRED((pcp)->p_flags)

#define LBOLT_DELTA(t)  ((ulong_t)(ddi_get_lbolt() - (t)))

#define PCP_AGE(pcp)    LBOLT_DELTA((pcp)->p_lbolt)

/*
 * htag0 argument can be a seg or amp pointer.
 */
#define P_HASHBP(seg, htag0, addr, flags)               \
    (IS_PFLAGS_WIRED((flags)) ?                 \
        ((struct seg_phash *)&seg_phashtab_wired[P_HASHWIRED_MASK & \
        ((uintptr_t)(htag0) >> P_BASESHIFT)]) :         \
        (&seg_phashtab_win[P_HASHWIN_MASK &             \
        (((uintptr_t)(htag0) >> 3) ^                \
        ((uintptr_t)(addr) >> ((flags & SEGP_PSHIFT) ?      \
        (flags >> 16) : page_get_shift((seg)->s_szc))))]))

/*
 * htag0 argument can be a seg or amp pointer.
 */
#define P_MATCH(pcp, htag0, addr, len)                  \
    ((pcp)->p_htag0 == (htag0) &&                   \
    (pcp)->p_addr == (addr) &&                  \
    (pcp)->p_len >= (len))

#define P_MATCH_PP(pcp, htag0, addr, len, pp)               \
    ((pcp)->p_pp == (pp) &&                     \
    (pcp)->p_htag0 == (htag0) &&                    \
    (pcp)->p_addr == (addr) &&                  \
    (pcp)->p_len >= (len))

#define plink2pcache(pl)    ((struct seg_pcache *)((uintptr_t)(pl) - \
    offsetof(struct seg_pcache, p_plink)))

#define hlink2phash(hl, l)  ((struct seg_phash *)((uintptr_t)(hl) - \
    offsetof(struct seg_phash, p_halink[l])))

/*
 * seg_padd_abuck()/seg_premove_abuck() link and unlink hash buckets from
 * active hash bucket lists. We maintain active bucket lists to reduce the
 * overhead of finding active buckets during asynchronous purging since there
 * can be 10s of millions of buckets on a large system but only a small subset
 * of them in actual use.
 *
 * There're 2 active bucket lists. Current active list (as per seg_pahcur) is
 * used by seg_pinsert()/seg_pinactive()/seg_ppurge() to add and delete
 * buckets. The other list is used by asynchronous purge thread. This allows
 * the purge thread to walk its active list without holding seg_pmem_mtx for a
 * long time. When asynchronous thread is done with its list it switches to
 * current active list and makes the list it just finished processing as
 * current active list.
 *
 * seg_padd_abuck() only adds the bucket to current list if the bucket is not
 * yet on any list.  seg_premove_abuck() may remove the bucket from either
 * list. If the bucket is on current list it will be always removed. Otherwise
 * the bucket is only removed if asynchronous purge thread is not currently
 * running or seg_premove_abuck() is called by asynchronous purge thread
 * itself. A given bucket can only be on one of active lists at a time. These
 * routines should be called with per bucket lock held.  The routines use
 * seg_pmem_mtx to protect list updates. seg_padd_abuck() must be called after
 * the first entry is added to the bucket chain and seg_premove_abuck() must
 * be called after the last pcp entry is deleted from its chain. Per bucket
 * lock should be held by the callers.  This avoids a potential race condition
 * when seg_premove_abuck() removes a bucket after pcp entries are added to
 * its list after the caller checked that the bucket has no entries. (this
 * race would cause a loss of an active bucket from the active lists).
 *
 * Both lists are circular doubly linked lists anchored at seg_pahhead heads.
 * New entries are added to the end of the list since LRU is used as the
 * purging policy.
 */
static void
seg_padd_abuck(struct seg_phash *hp)
{
    int lix;

    ASSERT(MUTEX_HELD(&hp->p_hmutex));
    ASSERT((struct seg_phash *)hp->p_hnext != hp);
    ASSERT((struct seg_phash *)hp->p_hprev != hp);
    ASSERT(hp->p_hnext == hp->p_hprev);
    ASSERT(!IS_PCP_WIRED(hp->p_hnext));
    ASSERT(hp->p_hnext->p_hnext == (struct seg_pcache *)hp);
    ASSERT(hp->p_hprev->p_hprev == (struct seg_pcache *)hp);
    ASSERT(hp >= seg_phashtab_win &&
        hp < &seg_phashtab_win[seg_phashsize_win]);

    /*
     * This bucket can already be on one of active lists
     * since seg_premove_abuck() may have failed to remove it
     * before.
     */
    mutex_enter(&seg_pmem_mtx);
    lix = seg_pahcur;
    ASSERT(lix >= 0 && lix <= 1);
    if (hp->p_halink[lix].p_lnext != NULL) {
        ASSERT(hp->p_halink[lix].p_lprev != NULL);
        ASSERT(hp->p_halink[!lix].p_lnext == NULL);
        ASSERT(hp->p_halink[!lix].p_lprev == NULL);
        mutex_exit(&seg_pmem_mtx);
        return;
    }
    ASSERT(hp->p_halink[lix].p_lprev == NULL);

    /*
     * If this bucket is still on list !lix async thread can't yet remove
     * it since we hold here per bucket lock. In this case just return
     * since async thread will eventually find and process this bucket.
     */
    if (hp->p_halink[!lix].p_lnext != NULL) {
        ASSERT(hp->p_halink[!lix].p_lprev != NULL);
        mutex_exit(&seg_pmem_mtx);
        return;
    }
    ASSERT(hp->p_halink[!lix].p_lprev == NULL);
    /*
     * This bucket is not on any active bucket list yet.
     * Add the bucket to the tail of current active list.
     */
    hp->p_halink[lix].p_lnext = &seg_pahhead[lix];
    hp->p_halink[lix].p_lprev = seg_pahhead[lix].p_lprev;
    seg_pahhead[lix].p_lprev->p_lnext = &hp->p_halink[lix];
    seg_pahhead[lix].p_lprev = &hp->p_halink[lix];
    mutex_exit(&seg_pmem_mtx);
}

static void
seg_premove_abuck(struct seg_phash *hp, int athr)
{
    int lix;

    ASSERT(MUTEX_HELD(&hp->p_hmutex));
    ASSERT((struct seg_phash *)hp->p_hnext == hp);
    ASSERT((struct seg_phash *)hp->p_hprev == hp);
    ASSERT(hp >= seg_phashtab_win &&
        hp < &seg_phashtab_win[seg_phashsize_win]);

    if (athr) {
        ASSERT(seg_pathr_on);
        ASSERT(seg_pahcur <= 1);
        /*
         * We are called by asynchronous thread that found this bucket
         * on not currently active (i.e. !seg_pahcur) list. Remove it
         * from there.  Per bucket lock we are holding makes sure
         * seg_pinsert() can't sneak in and add pcp entries to this
         * bucket right before we remove the bucket from its list.
         */
        lix = !seg_pahcur;
        ASSERT(hp->p_halink[lix].p_lnext != NULL);
        ASSERT(hp->p_halink[lix].p_lprev != NULL);
        ASSERT(hp->p_halink[!lix].p_lnext == NULL);
        ASSERT(hp->p_halink[!lix].p_lprev == NULL);
        hp->p_halink[lix].p_lnext->p_lprev = hp->p_halink[lix].p_lprev;
        hp->p_halink[lix].p_lprev->p_lnext = hp->p_halink[lix].p_lnext;
        hp->p_halink[lix].p_lnext = NULL;
        hp->p_halink[lix].p_lprev = NULL;
        return;
    }

    mutex_enter(&seg_pmem_mtx);
    lix = seg_pahcur;
    ASSERT(lix >= 0 && lix <= 1);

    /*
     * If the bucket is on currently active list just remove it from
     * there.
     */
    if (hp->p_halink[lix].p_lnext != NULL) {
        ASSERT(hp->p_halink[lix].p_lprev != NULL);
        ASSERT(hp->p_halink[!lix].p_lnext == NULL);
        ASSERT(hp->p_halink[!lix].p_lprev == NULL);
        hp->p_halink[lix].p_lnext->p_lprev = hp->p_halink[lix].p_lprev;
        hp->p_halink[lix].p_lprev->p_lnext = hp->p_halink[lix].p_lnext;
        hp->p_halink[lix].p_lnext = NULL;
        hp->p_halink[lix].p_lprev = NULL;
        mutex_exit(&seg_pmem_mtx);
        return;
    }
    ASSERT(hp->p_halink[lix].p_lprev == NULL);

    /*
     * If asynchronous thread is not running we can remove the bucket from
     * not currently active list. The bucket must be on this list since we
     * already checked that it's not on the other list and the bucket from
     * which we just deleted the last pcp entry must be still on one of the
     * active bucket lists.
     */
    lix = !lix;
    ASSERT(hp->p_halink[lix].p_lnext != NULL);
    ASSERT(hp->p_halink[lix].p_lprev != NULL);

    if (!seg_pathr_on) {
        hp->p_halink[lix].p_lnext->p_lprev = hp->p_halink[lix].p_lprev;
        hp->p_halink[lix].p_lprev->p_lnext = hp->p_halink[lix].p_lnext;
        hp->p_halink[lix].p_lnext = NULL;
        hp->p_halink[lix].p_lprev = NULL;
    }
    mutex_exit(&seg_pmem_mtx);
}

/*
 * Check if bucket pointed by hp already has a pcp entry that matches request
 * htag0, addr and len. Set *found to 1 if match is found and to 0 otherwise.
 * Also delete matching entries that cover smaller address range but start
 * at the same address as addr argument. Return the list of deleted entries if
 * any. This is an internal helper function called from seg_pinsert() only
 * for non wired shadow lists. The caller already holds a per seg/amp list
 * lock.
 */
static struct seg_pcache *
seg_plookup_checkdup(struct seg_phash *hp, void *htag0,
    caddr_t addr, size_t len, int *found)
{
    struct seg_pcache *pcp;
    struct seg_pcache *delcallb_list = NULL;

    ASSERT(MUTEX_HELD(&hp->p_hmutex));

    *found = 0;
    for (pcp = hp->p_hnext; pcp != (struct seg_pcache *)hp;
        pcp = pcp->p_hnext) {
        ASSERT(pcp->p_hashp == hp);
        if (pcp->p_htag0 == htag0 && pcp->p_addr == addr) {
            ASSERT(!IS_PCP_WIRED(pcp));
            if (pcp->p_len < len) {
                pcache_link_t *plinkp;
                if (pcp->p_active) {
                    continue;
                }
                plinkp = &pcp->p_plink;
                plinkp->p_lprev->p_lnext = plinkp->p_lnext;
                plinkp->p_lnext->p_lprev = plinkp->p_lprev;
                pcp->p_hprev->p_hnext = pcp->p_hnext;
                pcp->p_hnext->p_hprev = pcp->p_hprev;
                pcp->p_hprev = delcallb_list;
                delcallb_list = pcp;
            } else {
                *found = 1;
                break;
            }
        }
    }
    return (delcallb_list);
}

/*
 * lookup an address range in pagelock cache. Return shadow list and bump up
 * active count. If amp is not NULL use amp as a lookup tag otherwise use seg
 * as a lookup tag.
 */
struct page **
seg_plookup(struct seg *seg, struct anon_map *amp, caddr_t addr, size_t len,
    enum seg_rw rw, uint_t flags)
{
    struct seg_pcache *pcp;
    struct seg_phash *hp;
    void *htag0;

    ASSERT(seg != NULL);
    ASSERT(rw == S_READ || rw == S_WRITE);

    /*
     * Skip pagelock cache, while DR is in progress or
     * seg_pcache is off.
     */
    if (seg_pdisabled) {
        return (NULL);
    }
    ASSERT(seg_phashsize_win != 0);

    htag0 = (amp == NULL ? (void *)seg : (void *)amp);
    hp = P_HASHBP(seg, htag0, addr, flags);
    mutex_enter(&hp->p_hmutex);
    for (pcp = hp->p_hnext; pcp != (struct seg_pcache *)hp;
        pcp = pcp->p_hnext) {
        ASSERT(pcp->p_hashp == hp);
        if (P_MATCH(pcp, htag0, addr, len)) {
            ASSERT(IS_PFLAGS_WIRED(flags) == IS_PCP_WIRED(pcp));
            /*
             * If this request wants to write pages
             * but write permissions starting from
             * addr don't cover the entire length len
             * return lookup failure back to the caller.
             * It will check protections and fail this
             * pagelock operation with EACCESS error.
             */
            if (rw == S_WRITE && pcp->p_wlen < len) {
                break;
            }
            if (pcp->p_active == UINT_MAX) {
                break;
            }
            pcp->p_active++;
            if (rw == S_WRITE && !pcp->p_write) {
                pcp->p_write = 1;
            }
            mutex_exit(&hp->p_hmutex);
            return (pcp->p_pp);
        }
    }
    mutex_exit(&hp->p_hmutex);
    return (NULL);
}

/*
 * mark address range inactive. If the cache is off or the address range is
 * not in the cache or another shadow list that covers bigger range is found
 * we call the segment driver to reclaim the pages. Otherwise just decrement
 * active count and set ref bit.  If amp is not NULL use amp as a lookup tag
 * otherwise use seg as a lookup tag.
 */
void
seg_pinactive(struct seg *seg, struct anon_map *amp, caddr_t addr,
    size_t len, struct page **pp, enum seg_rw rw, uint_t flags,
    seg_preclaim_cbfunc_t callback)
{
    struct seg_pcache *pcp;
    struct seg_phash *hp;
    kmutex_t *pmtx = NULL;
    pcache_link_t *pheadp;
    void *htag0;
    pgcnt_t npages = 0;
    int keep = 0;

    ASSERT(seg != NULL);
    ASSERT(rw == S_READ || rw == S_WRITE);

    htag0 = (amp == NULL ? (void *)seg : (void *)amp);

    /*
     * Skip lookup if pcache is not configured.
     */
    if (seg_phashsize_win == 0) {
        goto out;
    }

    /*
     * Grab per seg/amp lock before hash lock if we are going to remove
     * inactive entry from pcache.
     */
    if (!IS_PFLAGS_WIRED(flags) && seg_pdisabled) {
        if (amp == NULL) {
            pheadp = &seg->s_phead;
            pmtx = &seg->s_pmtx;
        } else {
            pheadp = &amp->a_phead;
            pmtx = &amp->a_pmtx;
        }
        mutex_enter(pmtx);
    }

    hp = P_HASHBP(seg, htag0, addr, flags);
    mutex_enter(&hp->p_hmutex);
again:
    for (pcp = hp->p_hnext; pcp != (struct seg_pcache *)hp;
        pcp = pcp->p_hnext) {
        ASSERT(pcp->p_hashp == hp);
        if (P_MATCH_PP(pcp, htag0, addr, len, pp)) {
            ASSERT(IS_PFLAGS_WIRED(flags) == IS_PCP_WIRED(pcp));
            ASSERT(pcp->p_active);
            if (keep) {
                /*
                 * Don't remove this pcp entry
                 * if we didn't find duplicate
                 * shadow lists on second search.
                 * Somebody removed those duplicates
                 * since we dropped hash lock after first
                 * search.
                 */
                ASSERT(pmtx != NULL);
                ASSERT(!IS_PFLAGS_WIRED(flags));
                mutex_exit(pmtx);
                pmtx = NULL;
            }
            pcp->p_active--;
            if (pcp->p_active == 0 && (pmtx != NULL ||
                (seg_pdisabled && IS_PFLAGS_WIRED(flags)))) {

                /*
                 * This entry is no longer active.  Remove it
                 * now either because pcaching is temporarily
                 * disabled or there're other pcp entries that
                 * can match this pagelock request (i.e. this
                 * entry is a duplicate).
                 */

                ASSERT(callback == pcp->p_callback);
                if (pmtx != NULL) {
                    pcache_link_t *plinkp = &pcp->p_plink;
                    ASSERT(!IS_PCP_WIRED(pcp));
                    ASSERT(pheadp->p_lnext != pheadp);
                    ASSERT(pheadp->p_lprev != pheadp);
                    plinkp->p_lprev->p_lnext =
                        plinkp->p_lnext;
                    plinkp->p_lnext->p_lprev =
                        plinkp->p_lprev;
                }
                pcp->p_hprev->p_hnext = pcp->p_hnext;
                pcp->p_hnext->p_hprev = pcp->p_hprev;
                if (!IS_PCP_WIRED(pcp) &&
                    hp->p_hnext == (struct seg_pcache *)hp) {
                    /*
                     * We removed the last entry from this
                     * bucket.  Now remove the bucket from
                     * its active list.
                     */
                    seg_premove_abuck(hp, 0);
                }
                mutex_exit(&hp->p_hmutex);
                if (pmtx != NULL) {
                    mutex_exit(pmtx);
                }
                len = pcp->p_len;
                npages = btop(len);
                if (rw != S_WRITE && pcp->p_write) {
                    rw = S_WRITE;
                }
                kmem_cache_free(seg_pkmcache, pcp);
                goto out;
            } else {
                /*
                 * We found a matching pcp entry but will not
                 * free it right away even if it's no longer
                 * active.
                 */
                if (!pcp->p_active && !IS_PCP_WIRED(pcp)) {
                    /*
                     * Set the reference bit and mark the
                     * time of last access to this pcp
                     * so that asynchronous thread doesn't
                     * free it immediately since
                     * it may be reactivated very soon.
                     */
                    pcp->p_lbolt = ddi_get_lbolt();
                    pcp->p_ref = 1;
                }
                mutex_exit(&hp->p_hmutex);
                if (pmtx != NULL) {
                    mutex_exit(pmtx);
                }
                return;
            }
        } else if (!IS_PFLAGS_WIRED(flags) &&
            P_MATCH(pcp, htag0, addr, len)) {
            /*
             * This is a duplicate pcp entry.  This situation may
             * happen if a bigger shadow list that covers our
             * range was added while our entry was still active.
             * Now we can free our pcp entry if it becomes
             * inactive.
             */
            if (!pcp->p_active) {
                /*
                 * Mark this entry as referenced just in case
                 * we'll free our own pcp entry soon.
                 */
                pcp->p_lbolt = ddi_get_lbolt();
                pcp->p_ref = 1;
            }
            if (pmtx != NULL) {
                /*
                 * we are already holding pmtx and found a
                 * duplicate.  Don't keep our own pcp entry.
                 */
                keep = 0;
                continue;
            }
            /*
             * We have to use mutex_tryenter to attempt to lock
             * seg/amp list lock since we already hold hash lock
             * and seg/amp list lock is above hash lock in lock
             * order.  If mutex_tryenter fails drop hash lock and
             * retake both locks in correct order and research
             * this hash chain.
             */
            ASSERT(keep == 0);
            if (amp == NULL) {
                pheadp = &seg->s_phead;
                pmtx = &seg->s_pmtx;
            } else {
                pheadp = &amp->a_phead;
                pmtx = &amp->a_pmtx;
            }
            if (!mutex_tryenter(pmtx)) {
                mutex_exit(&hp->p_hmutex);
                mutex_enter(pmtx);
                mutex_enter(&hp->p_hmutex);
                /*
                 * If we don't find bigger shadow list on
                 * second search (it may happen since we
                 * dropped bucket lock) keep the entry that
                 * matches our own shadow list.
                 */
                keep = 1;
                goto again;
            }
        }
    }
    mutex_exit(&hp->p_hmutex);
    if (pmtx != NULL) {
        mutex_exit(pmtx);
    }
out:
    (*callback)(htag0, addr, len, pp, rw, 0);
    if (npages) {
        mutex_enter(&seg_pmem_mtx);
        ASSERT(seg_plocked >= npages);
        seg_plocked -= npages;
        if (!IS_PFLAGS_WIRED(flags)) {
            ASSERT(seg_plocked_window >= npages);
            seg_plocked_window -= npages;
        }
        mutex_exit(&seg_pmem_mtx);
    }

}

#ifdef DEBUG
static uint32_t p_insert_chk_mtbf = 0;
#endif

/*
 * The seg_pinsert_check() is used by segment drivers to predict whether
 * a call to seg_pinsert will fail and thereby avoid wasteful pre-processing.
 */
/*ARGSUSED*/
int
seg_pinsert_check(struct seg *seg, struct anon_map *amp, caddr_t addr,
    size_t len, uint_t flags)
{
    ASSERT(seg != NULL);

#ifdef DEBUG
    if (p_insert_chk_mtbf && !(gethrtime() % p_insert_chk_mtbf)) {
        return (SEGP_FAIL);
    }
#endif

    if (seg_pdisabled) {
        return (SEGP_FAIL);
    }
    ASSERT(seg_phashsize_win != 0);

    if (IS_PFLAGS_WIRED(flags)) {
        return (SEGP_SUCCESS);
    }

    if (seg_plocked_window + btop(len) > seg_pmaxwindow) {
        return (SEGP_FAIL);
    }

    if (freemem < desfree) {
        return (SEGP_FAIL);
    }

    return (SEGP_SUCCESS);
}

#ifdef DEBUG
static uint32_t p_insert_mtbf = 0;
#endif

/*
 * Insert address range with shadow list into pagelock cache if there's no
 * shadow list already cached for this address range. If the cache is off or
 * caching is temporarily disabled or the allowed 'window' is exceeded return
 * SEGP_FAIL. Otherwise return SEGP_SUCCESS.
 *
 * For non wired shadow lists (segvn case) include address in the hashing
 * function to avoid linking all the entries from the same segment or amp on
 * the same bucket.  amp is used instead of seg if amp is not NULL. Non wired
 * pcache entries are also linked on a per segment/amp list so that all
 * entries can be found quickly during seg/amp purge without walking the
 * entire pcache hash table.  For wired shadow lists (segspt case) we
 * don't use address hashing and per segment linking because the caller
 * currently inserts only one entry per segment that covers the entire
 * segment. If we used per segment linking even for segspt it would complicate
 * seg_ppurge_wiredpp() locking.
 *
 * Both hash bucket and per seg/amp locks need to be held before adding a non
 * wired entry to hash and per seg/amp lists. per seg/amp lock should be taken
 * first.
 *
 * This function will also remove from pcache old inactive shadow lists that
 * overlap with this request but cover smaller range for the same start
 * address.
 */
int
seg_pinsert(struct seg *seg, struct anon_map *amp, caddr_t addr, size_t len,
    size_t wlen, struct page **pp, enum seg_rw rw, uint_t flags,
    seg_preclaim_cbfunc_t callback)
{
    struct seg_pcache *pcp;
    struct seg_phash *hp;
    pgcnt_t npages;
    pcache_link_t *pheadp;
    kmutex_t *pmtx;
    struct seg_pcache *delcallb_list = NULL;

    ASSERT(seg != NULL);
    ASSERT(rw == S_READ || rw == S_WRITE);
    ASSERT(rw == S_READ || wlen == len);
    ASSERT(rw == S_WRITE || wlen <= len);
    ASSERT(amp == NULL || wlen == len);

#ifdef DEBUG
    if (p_insert_mtbf && !(gethrtime() % p_insert_mtbf)) {
        return (SEGP_FAIL);
    }
#endif

    if (seg_pdisabled) {
        return (SEGP_FAIL);
    }
    ASSERT(seg_phashsize_win != 0);

    ASSERT((len & PAGEOFFSET) == 0);
    npages = btop(len);
    mutex_enter(&seg_pmem_mtx);
    if (!IS_PFLAGS_WIRED(flags)) {
        if (seg_plocked_window + npages > seg_pmaxwindow) {
            mutex_exit(&seg_pmem_mtx);
            return (SEGP_FAIL);
        }
        seg_plocked_window += npages;
    }
    seg_plocked += npages;
    mutex_exit(&seg_pmem_mtx);

    pcp = kmem_cache_alloc(seg_pkmcache, KM_SLEEP);
    /*
     * If amp is not NULL set htag0 to amp otherwise set it to seg.
     */
    if (amp == NULL) {
        pcp->p_htag0 = (void *)seg;
        pcp->p_flags = flags & 0xffff;
    } else {
        pcp->p_htag0 = (void *)amp;
        pcp->p_flags = (flags & 0xffff) | SEGP_AMP;
    }
    pcp->p_addr = addr;
    pcp->p_len = len;
    pcp->p_wlen = wlen;
    pcp->p_pp = pp;
    pcp->p_write = (rw == S_WRITE);
    pcp->p_callback = callback;
    pcp->p_active = 1;

    hp = P_HASHBP(seg, pcp->p_htag0, addr, flags);
    if (!IS_PFLAGS_WIRED(flags)) {
        int found;
        void *htag0;
        if (amp == NULL) {
            pheadp = &seg->s_phead;
            pmtx = &seg->s_pmtx;
            htag0 = (void *)seg;
        } else {
            pheadp = &amp->a_phead;
            pmtx = &amp->a_pmtx;
            htag0 = (void *)amp;
        }
        mutex_enter(pmtx);
        mutex_enter(&hp->p_hmutex);
        delcallb_list = seg_plookup_checkdup(hp, htag0, addr,
            len, &found);
        if (found) {
            mutex_exit(&hp->p_hmutex);
            mutex_exit(pmtx);
            mutex_enter(&seg_pmem_mtx);
            seg_plocked -= npages;
            seg_plocked_window -= npages;
            mutex_exit(&seg_pmem_mtx);
            kmem_cache_free(seg_pkmcache, pcp);
            goto out;
        }
        pcp->p_plink.p_lnext = pheadp->p_lnext;
        pcp->p_plink.p_lprev = pheadp;
        pheadp->p_lnext->p_lprev = &pcp->p_plink;
        pheadp->p_lnext = &pcp->p_plink;
    } else {
        mutex_enter(&hp->p_hmutex);
    }
    pcp->p_hashp = hp;
    pcp->p_hnext = hp->p_hnext;
    pcp->p_hprev = (struct seg_pcache *)hp;
    hp->p_hnext->p_hprev = pcp;
    hp->p_hnext = pcp;
    if (!IS_PFLAGS_WIRED(flags) &&
        hp->p_hprev == pcp) {
        seg_padd_abuck(hp);
    }
    mutex_exit(&hp->p_hmutex);
    if (!IS_PFLAGS_WIRED(flags)) {
        mutex_exit(pmtx);
    }

out:
    npages = 0;
    while (delcallb_list != NULL) {
        pcp = delcallb_list;
        delcallb_list = pcp->p_hprev;
        ASSERT(!IS_PCP_WIRED(pcp) && !pcp->p_active);
        (void) (*pcp->p_callback)(pcp->p_htag0, pcp->p_addr,
            pcp->p_len, pcp->p_pp, pcp->p_write ? S_WRITE : S_READ, 0);
        npages += btop(pcp->p_len);
        kmem_cache_free(seg_pkmcache, pcp);
    }
    if (npages) {
        ASSERT(!IS_PFLAGS_WIRED(flags));
        mutex_enter(&seg_pmem_mtx);
        ASSERT(seg_plocked >= npages);
        ASSERT(seg_plocked_window >= npages);
        seg_plocked -= npages;
        seg_plocked_window -= npages;
        mutex_exit(&seg_pmem_mtx);
    }

    return (SEGP_SUCCESS);
}

/*
 * purge entries from the pagelock cache if not active
 * and not recently used.
 */
static void
seg_ppurge_async(int force)
{
    struct seg_pcache *delcallb_list = NULL;
    struct seg_pcache *pcp;
    struct seg_phash *hp;
    pgcnt_t npages = 0;
    pgcnt_t npages_window = 0;
    pgcnt_t npgs_to_purge;
    pgcnt_t npgs_purged = 0;
    int hlinks = 0;
    int hlix;
    pcache_link_t *hlinkp;
    pcache_link_t *hlnextp = NULL;
    int lowmem;
    int trim;

    ASSERT(seg_phashsize_win != 0);

    /*
     * if the cache is off or empty, return
     */
    if (seg_plocked == 0 || (!force && seg_plocked_window == 0)) {
        return;
    }

    if (!force) {
        lowmem = 0;
        trim = 0;
        if (freemem < lotsfree + needfree) {
            spgcnt_t fmem = MAX((spgcnt_t)(freemem - needfree), 0);
            if (fmem <= 5 * (desfree >> 2)) {
                lowmem = 1;
            } else if (fmem <= 7 * (lotsfree >> 3)) {
                if (seg_plocked_window >=
                    (availrmem_initial >> 1)) {
                    lowmem = 1;
                }
            } else if (fmem < lotsfree) {
                if (seg_plocked_window >=
                    3 * (availrmem_initial >> 2)) {
                    lowmem = 1;
                }
            }
        }
        if (seg_plocked_window >= 7 * (seg_pmaxwindow >> 3)) {
            trim = 1;
        }
        if (!lowmem && !trim) {
            return;
        }
        npgs_to_purge = seg_plocked_window >>
            seg_pshrink_shift;
        if (lowmem) {
            npgs_to_purge = MIN(npgs_to_purge,
                MAX(seg_pmaxapurge_npages, desfree));
        } else {
            npgs_to_purge = MIN(npgs_to_purge,
                seg_pmaxapurge_npages);
        }
        if (npgs_to_purge == 0) {
            return;
        }
    } else {
        struct seg_phash_wired *hpw;

        ASSERT(seg_phashsize_wired != 0);

        for (hpw = seg_phashtab_wired;
            hpw < &seg_phashtab_wired[seg_phashsize_wired]; hpw++) {

            if (hpw->p_hnext == (struct seg_pcache *)hpw) {
                continue;
            }

            mutex_enter(&hpw->p_hmutex);

            for (pcp = hpw->p_hnext;
                pcp != (struct seg_pcache *)hpw;
                pcp = pcp->p_hnext) {

                ASSERT(IS_PCP_WIRED(pcp));
                ASSERT(pcp->p_hashp ==
                    (struct seg_phash *)hpw);

                if (pcp->p_active) {
                    continue;
                }
                pcp->p_hprev->p_hnext = pcp->p_hnext;
                pcp->p_hnext->p_hprev = pcp->p_hprev;
                pcp->p_hprev = delcallb_list;
                delcallb_list = pcp;
            }
            mutex_exit(&hpw->p_hmutex);
        }
    }

    mutex_enter(&seg_pmem_mtx);
    if (seg_pathr_on) {
        mutex_exit(&seg_pmem_mtx);
        goto runcb;
    }
    seg_pathr_on = 1;
    mutex_exit(&seg_pmem_mtx);
    ASSERT(seg_pahcur <= 1);
    hlix = !seg_pahcur;

again:
    for (hlinkp = seg_pahhead[hlix].p_lnext; hlinkp != &seg_pahhead[hlix];
        hlinkp = hlnextp) {

        hlnextp = hlinkp->p_lnext;
        ASSERT(hlnextp != NULL);

        hp = hlink2phash(hlinkp, hlix);
        if (hp->p_hnext == (struct seg_pcache *)hp) {
            seg_pathr_empty_ahb++;
            continue;
        }
        seg_pathr_full_ahb++;
        mutex_enter(&hp->p_hmutex);

        for (pcp = hp->p_hnext; pcp != (struct seg_pcache *)hp;
            pcp = pcp->p_hnext) {
            pcache_link_t *pheadp;
            pcache_link_t *plinkp;
            void *htag0;
            kmutex_t *pmtx;

            ASSERT(!IS_PCP_WIRED(pcp));
            ASSERT(pcp->p_hashp == hp);

            if (pcp->p_active) {
                continue;
            }
            if (!force && pcp->p_ref &&
                PCP_AGE(pcp) < seg_pmax_pcpage) {
                pcp->p_ref = 0;
                continue;
            }
            plinkp = &pcp->p_plink;
            htag0 = pcp->p_htag0;
            if (pcp->p_flags & SEGP_AMP) {
                pheadp = &((amp_t *)htag0)->a_phead;
                pmtx = &((amp_t *)htag0)->a_pmtx;
            } else {
                pheadp = &((seg_t *)htag0)->s_phead;
                pmtx = &((seg_t *)htag0)->s_pmtx;
            }
            if (!mutex_tryenter(pmtx)) {
                continue;
            }
            ASSERT(pheadp->p_lnext != pheadp);
            ASSERT(pheadp->p_lprev != pheadp);
            plinkp->p_lprev->p_lnext =
                plinkp->p_lnext;
            plinkp->p_lnext->p_lprev =
                plinkp->p_lprev;
            pcp->p_hprev->p_hnext = pcp->p_hnext;
            pcp->p_hnext->p_hprev = pcp->p_hprev;
            mutex_exit(pmtx);
            pcp->p_hprev = delcallb_list;
            delcallb_list = pcp;
            npgs_purged += btop(pcp->p_len);
        }
        if (hp->p_hnext == (struct seg_pcache *)hp) {
            seg_premove_abuck(hp, 1);
        }
        mutex_exit(&hp->p_hmutex);
        if (npgs_purged >= seg_plocked_window) {
            break;
        }
        if (!force) {
            if (npgs_purged >= npgs_to_purge) {
                break;
            }
            if (!trim && !(seg_pathr_full_ahb & 15)) {
                ASSERT(lowmem);
                if (freemem >= lotsfree + needfree) {
                    break;
                }
            }
        }
    }

    if (hlinkp == &seg_pahhead[hlix]) {
        /*
         * We processed the entire hlix active bucket list
         * but didn't find enough pages to reclaim.
         * Switch the lists and walk the other list
         * if we haven't done it yet.
         */
        mutex_enter(&seg_pmem_mtx);
        ASSERT(seg_pathr_on);
        ASSERT(seg_pahcur == !hlix);
        seg_pahcur = hlix;
        mutex_exit(&seg_pmem_mtx);
        if (++hlinks < 2) {
            hlix = !hlix;
            goto again;
        }
    } else if ((hlinkp = hlnextp) != &seg_pahhead[hlix] &&
        seg_pahhead[hlix].p_lnext != hlinkp) {
        ASSERT(hlinkp != NULL);
        ASSERT(hlinkp->p_lprev != &seg_pahhead[hlix]);
        ASSERT(seg_pahhead[hlix].p_lnext != &seg_pahhead[hlix]);
        ASSERT(seg_pahhead[hlix].p_lprev != &seg_pahhead[hlix]);

        /*
         * Reinsert the header to point to hlinkp
         * so that we start from hlinkp bucket next time around.
         */
        seg_pahhead[hlix].p_lnext->p_lprev = seg_pahhead[hlix].p_lprev;
        seg_pahhead[hlix].p_lprev->p_lnext = seg_pahhead[hlix].p_lnext;
        seg_pahhead[hlix].p_lnext = hlinkp;
        seg_pahhead[hlix].p_lprev = hlinkp->p_lprev;
        hlinkp->p_lprev->p_lnext = &seg_pahhead[hlix];
        hlinkp->p_lprev = &seg_pahhead[hlix];
    }

    mutex_enter(&seg_pmem_mtx);
    ASSERT(seg_pathr_on);
    seg_pathr_on = 0;
    mutex_exit(&seg_pmem_mtx);

runcb:
    /*
     * Run the delayed callback list. segments/amps can't go away until
     * callback is executed since they must have non 0 softlockcnt. That's
     * why we don't need to hold as/seg/amp locks to execute the callback.
     */
    while (delcallb_list != NULL) {
        pcp = delcallb_list;
        delcallb_list = pcp->p_hprev;
        ASSERT(!pcp->p_active);
        (void) (*pcp->p_callback)(pcp->p_htag0, pcp->p_addr,
            pcp->p_len, pcp->p_pp, pcp->p_write ? S_WRITE : S_READ, 1);
        npages += btop(pcp->p_len);
        if (!IS_PCP_WIRED(pcp)) {
            npages_window += btop(pcp->p_len);
        }
        kmem_cache_free(seg_pkmcache, pcp);
    }
    if (npages) {
        mutex_enter(&seg_pmem_mtx);
        ASSERT(seg_plocked >= npages);
        ASSERT(seg_plocked_window >= npages_window);
        seg_plocked -= npages;
        seg_plocked_window -= npages_window;
        mutex_exit(&seg_pmem_mtx);
    }
}

/*
 * Remove cached pages for segment(s) entries from hashtable.  The segments
 * are identified by pp array. This is useful for multiple seg's cached on
 * behalf of dummy segment (ISM/DISM) with common pp array.
 */
void
seg_ppurge_wiredpp(struct page **pp)
{
    struct seg_pcache *pcp;
    struct seg_phash_wired *hp;
    pgcnt_t npages = 0;
    struct  seg_pcache *delcallb_list = NULL;

    /*
     * if the cache is empty, return
     */
    if (seg_plocked == 0) {
        return;
    }
    ASSERT(seg_phashsize_wired != 0);

    for (hp = seg_phashtab_wired;
        hp < &seg_phashtab_wired[seg_phashsize_wired]; hp++) {
        if (hp->p_hnext == (struct seg_pcache *)hp) {
            continue;
        }
        mutex_enter(&hp->p_hmutex);
        pcp = hp->p_hnext;
        while (pcp != (struct seg_pcache *)hp) {
            ASSERT(pcp->p_hashp == (struct seg_phash *)hp);
            ASSERT(IS_PCP_WIRED(pcp));
            /*
             * purge entries which are not active
             */
            if (!pcp->p_active && pcp->p_pp == pp) {
                ASSERT(pcp->p_htag0 != NULL);
                pcp->p_hprev->p_hnext = pcp->p_hnext;
                pcp->p_hnext->p_hprev = pcp->p_hprev;
                pcp->p_hprev = delcallb_list;
                delcallb_list = pcp;
            }
            pcp = pcp->p_hnext;
        }
        mutex_exit(&hp->p_hmutex);
        /*
         * segments can't go away until callback is executed since
         * they must have non 0 softlockcnt. That's why we don't
         * need to hold as/seg locks to execute the callback.
         */
        while (delcallb_list != NULL) {
            int done;
            pcp = delcallb_list;
            delcallb_list = pcp->p_hprev;
            ASSERT(!pcp->p_active);
            done = (*pcp->p_callback)(pcp->p_htag0, pcp->p_addr,
                pcp->p_len, pcp->p_pp,
                pcp->p_write ? S_WRITE : S_READ, 1);
            npages += btop(pcp->p_len);
            ASSERT(IS_PCP_WIRED(pcp));
            kmem_cache_free(seg_pkmcache, pcp);
            if (done) {
                ASSERT(delcallb_list == NULL);
                goto out;
            }
        }
    }

out:
    mutex_enter(&seg_pmem_mtx);
    ASSERT(seg_plocked >= npages);
    seg_plocked -= npages;
    mutex_exit(&seg_pmem_mtx);
}

/*
 * purge all entries for a given segment. Since we
 * callback into the segment driver directly for page
 * reclaim the caller needs to hold the right locks.
 */
void
seg_ppurge(struct seg *seg, struct anon_map *amp, uint_t flags)
{
    struct seg_pcache *delcallb_list = NULL;
    struct seg_pcache *pcp;
    struct seg_phash *hp;
    pgcnt_t npages = 0;
    void *htag0;

    if (seg_plocked == 0) {
        return;
    }
    ASSERT(seg_phashsize_win != 0);

    /*
     * If amp is not NULL use amp as a lookup tag otherwise use seg
     * as a lookup tag.
     */
    htag0 = (amp == NULL ? (void *)seg : (void *)amp);
    ASSERT(htag0 != NULL);
    if (IS_PFLAGS_WIRED(flags)) {
        hp = P_HASHBP(seg, htag0, 0, flags);
        mutex_enter(&hp->p_hmutex);
        pcp = hp->p_hnext;
        while (pcp != (struct seg_pcache *)hp) {
            ASSERT(pcp->p_hashp == hp);
            ASSERT(IS_PCP_WIRED(pcp));
            if (pcp->p_htag0 == htag0) {
                if (pcp->p_active) {
                    break;
                }
                pcp->p_hprev->p_hnext = pcp->p_hnext;
                pcp->p_hnext->p_hprev = pcp->p_hprev;
                pcp->p_hprev = delcallb_list;
                delcallb_list = pcp;
            }
            pcp = pcp->p_hnext;
        }
        mutex_exit(&hp->p_hmutex);
    } else {
        pcache_link_t *plinkp;
        pcache_link_t *pheadp;
        kmutex_t *pmtx;

        if (amp == NULL) {
            ASSERT(seg != NULL);
            pheadp = &seg->s_phead;
            pmtx = &seg->s_pmtx;
        } else {
            pheadp = &amp->a_phead;
            pmtx = &amp->a_pmtx;
        }
        mutex_enter(pmtx);
        while ((plinkp = pheadp->p_lnext) != pheadp) {
            pcp = plink2pcache(plinkp);
            ASSERT(!IS_PCP_WIRED(pcp));
            ASSERT(pcp->p_htag0 == htag0);
            hp = pcp->p_hashp;
            mutex_enter(&hp->p_hmutex);
            if (pcp->p_active) {
                mutex_exit(&hp->p_hmutex);
                break;
            }
            ASSERT(plinkp->p_lprev == pheadp);
            pheadp->p_lnext = plinkp->p_lnext;
            plinkp->p_lnext->p_lprev = pheadp;
            pcp->p_hprev->p_hnext = pcp->p_hnext;
            pcp->p_hnext->p_hprev = pcp->p_hprev;
            pcp->p_hprev = delcallb_list;
            delcallb_list = pcp;
            if (hp->p_hnext == (struct seg_pcache *)hp) {
                seg_premove_abuck(hp, 0);
            }
            mutex_exit(&hp->p_hmutex);
        }
        mutex_exit(pmtx);
    }
    while (delcallb_list != NULL) {
        pcp = delcallb_list;
        delcallb_list = pcp->p_hprev;
        ASSERT(!pcp->p_active);
        (void) (*pcp->p_callback)(pcp->p_htag0, pcp->p_addr, pcp->p_len,
            pcp->p_pp, pcp->p_write ? S_WRITE : S_READ, 0);
        npages += btop(pcp->p_len);
        kmem_cache_free(seg_pkmcache, pcp);
    }
    mutex_enter(&seg_pmem_mtx);
    ASSERT(seg_plocked >= npages);
    seg_plocked -= npages;
    if (!IS_PFLAGS_WIRED(flags)) {
        ASSERT(seg_plocked_window >= npages);
        seg_plocked_window -= npages;
    }
    mutex_exit(&seg_pmem_mtx);
}

static void seg_pinit_mem_config(void);

/*
 * setup the pagelock cache
 */
static void
seg_pinit(void)
{
    struct seg_phash *hp;
    ulong_t i;
    pgcnt_t physmegs;

    seg_plocked = 0;
    seg_plocked_window = 0;

    if (segpcache_enabled == 0) {
        seg_phashsize_win = 0;
        seg_phashsize_wired = 0;
        seg_pdisabled = 1;
        return;
    }

    seg_pdisabled = 0;
    seg_pkmcache = kmem_cache_create("seg_pcache",
        sizeof (struct seg_pcache), 0, NULL, NULL, NULL, NULL, NULL, 0);
    if (segpcache_pcp_maxage_ticks <= 0) {
        segpcache_pcp_maxage_ticks = segpcache_pcp_maxage_sec * hz;
    }
    seg_pmax_pcpage = segpcache_pcp_maxage_ticks;
    seg_pathr_empty_ahb = 0;
    seg_pathr_full_ahb = 0;
    seg_pshrink_shift = segpcache_shrink_shift;
    seg_pmaxapurge_npages = btop(segpcache_maxapurge_bytes);

    mutex_init(&seg_pcache_mtx, NULL, MUTEX_DEFAULT, NULL);
    mutex_init(&seg_pmem_mtx, NULL, MUTEX_DEFAULT, NULL);
    mutex_init(&seg_pasync_mtx, NULL, MUTEX_DEFAULT, NULL);
    cv_init(&seg_pasync_cv, NULL, CV_DEFAULT, NULL);

    physmegs = physmem >> (20 - PAGESHIFT);

    /*
     * If segpcache_hashsize_win was not set in /etc/system or it has
     * absurd value set it to a default.
     */
    if (segpcache_hashsize_win == 0 || segpcache_hashsize_win > physmem) {
        /*
         * Create one bucket per 32K (or at least per 8 pages) of
         * available memory.
         */
        pgcnt_t pages_per_bucket = MAX(btop(32 * 1024), 8);
        segpcache_hashsize_win = MAX(1024, physmem / pages_per_bucket);
    }
    if (!ISP2(segpcache_hashsize_win)) {
        ulong_t rndfac = ~(1UL <<
            (highbit(segpcache_hashsize_win) - 1));
        rndfac &= segpcache_hashsize_win;
        segpcache_hashsize_win += rndfac;
        segpcache_hashsize_win = 1 <<
            (highbit(segpcache_hashsize_win) - 1);
    }
    seg_phashsize_win = segpcache_hashsize_win;
    seg_phashtab_win = kmem_zalloc(
        seg_phashsize_win * sizeof (struct seg_phash),
        KM_SLEEP);
    for (i = 0; i < seg_phashsize_win; i++) {
        hp = &seg_phashtab_win[i];
        hp->p_hnext = (struct seg_pcache *)hp;
        hp->p_hprev = (struct seg_pcache *)hp;
        mutex_init(&hp->p_hmutex, NULL, MUTEX_DEFAULT, NULL);
    }

    seg_pahcur = 0;
    seg_pathr_on = 0;
    seg_pahhead[0].p_lnext = &seg_pahhead[0];
    seg_pahhead[0].p_lprev = &seg_pahhead[0];
    seg_pahhead[1].p_lnext = &seg_pahhead[1];
    seg_pahhead[1].p_lprev = &seg_pahhead[1];

    /*
     * If segpcache_hashsize_wired was not set in /etc/system or it has
     * absurd value set it to a default.
     */
    if (segpcache_hashsize_wired == 0 ||
        segpcache_hashsize_wired > physmem / 4) {
        /*
         * Choose segpcache_hashsize_wired based on physmem.
         * Create a bucket per 128K bytes upto 256K buckets.
         */
        if (physmegs < 20 * 1024) {
            segpcache_hashsize_wired = MAX(1024, physmegs << 3);
        } else {
            segpcache_hashsize_wired = 256 * 1024;
        }
    }
    if (!ISP2(segpcache_hashsize_wired)) {
        segpcache_hashsize_wired = 1 <<
            highbit(segpcache_hashsize_wired);
    }
    seg_phashsize_wired = segpcache_hashsize_wired;
    seg_phashtab_wired = kmem_zalloc(
        seg_phashsize_wired * sizeof (struct seg_phash_wired), KM_SLEEP);
    for (i = 0; i < seg_phashsize_wired; i++) {
        hp = (struct seg_phash *)&seg_phashtab_wired[i];
        hp->p_hnext = (struct seg_pcache *)hp;
        hp->p_hprev = (struct seg_pcache *)hp;
        mutex_init(&hp->p_hmutex, NULL, MUTEX_DEFAULT, NULL);
    }

    if (segpcache_maxwindow == 0) {
        if (physmegs < 64) {
            /* 3% of memory */
            segpcache_maxwindow = availrmem >> 5;
        } else if (physmegs < 512) {
            /* 12% of memory */
            segpcache_maxwindow = availrmem >> 3;
        } else if (physmegs < 1024) {
            /* 25% of memory */
            segpcache_maxwindow = availrmem >> 2;
        } else if (physmegs < 2048) {
            /* 50% of memory */
            segpcache_maxwindow = availrmem >> 1;
        } else {
            /* no limit */
            segpcache_maxwindow = (pgcnt_t)-1;
        }
    }
    seg_pmaxwindow = segpcache_maxwindow;
    seg_pinit_mem_config();
}

/*
 * called by pageout if memory is low
 */
void
seg_preap(void)
{
    /*
     * if the cache is off or empty, return
     */
    if (seg_plocked_window == 0) {
        return;
    }
    ASSERT(seg_phashsize_win != 0);

    /*
     * If somebody is already purging pcache
     * just return.
     */
    if (seg_pdisabled) {
        return;
    }

    cv_signal(&seg_pasync_cv);
}

/*
 * run as a backgroud thread and reclaim pagelock
 * pages which have not been used recently
 */
void
seg_pasync_thread(void)
{
    callb_cpr_t cpr_info;

    if (seg_phashsize_win == 0) {
        thread_exit();
        /*NOTREACHED*/
    }

    seg_pasync_thr = curthread;

    CALLB_CPR_INIT(&cpr_info, &seg_pasync_mtx,
        callb_generic_cpr, "seg_pasync");

    if (segpcache_reap_ticks <= 0) {
        segpcache_reap_ticks = segpcache_reap_sec * hz;
    }

    mutex_enter(&seg_pasync_mtx);
    for (;;) {
        CALLB_CPR_SAFE_BEGIN(&cpr_info);
        (void) cv_reltimedwait(&seg_pasync_cv, &seg_pasync_mtx,
            segpcache_reap_ticks, TR_CLOCK_TICK);
        CALLB_CPR_SAFE_END(&cpr_info, &seg_pasync_mtx);
        if (seg_pdisabled == 0) {
            seg_ppurge_async(0);
        }
    }
}

static struct kmem_cache *seg_cache;

/*
 * Initialize segment management data structures.
 */
void
seg_init(void)
{
    kstat_t *ksp;

    seg_cache = kmem_cache_create("seg_cache", sizeof (struct seg),
        0, NULL, NULL, NULL, NULL, NULL, 0);

    ksp = kstat_create("unix", 0, "segadvstat", "vm", KSTAT_TYPE_NAMED,
        segadvstat_ndata, KSTAT_FLAG_VIRTUAL);
    if (ksp) {
        ksp->ks_data = (void *)segadvstat_ptr;
        kstat_install(ksp);
    }

    seg_pinit();
}

/*
 * Allocate a segment to cover [base, base+size]
 * and attach it to the specified address space.
 */
struct seg *
seg_alloc(struct as *as, caddr_t base, size_t size)
{
    struct seg *new;
    caddr_t segbase;
    size_t segsize;

    segbase = (caddr_t)((uintptr_t)base & (uintptr_t)PAGEMASK);
    segsize = (((uintptr_t)(base + size) + PAGEOFFSET) & PAGEMASK) -
        (uintptr_t)segbase;

    if (!valid_va_range(&segbase, &segsize, segsize, AH_LO))
        return ((struct seg *)NULL);    /* bad virtual addr range */

    if (as != &kas &&
        valid_usr_range(segbase, segsize, 0, as,
        as->a_userlimit) != RANGE_OKAY)
        return ((struct seg *)NULL);    /* bad virtual addr range */

    new = kmem_cache_alloc(seg_cache, KM_SLEEP);
    new->s_ops = NULL;
    new->s_data = NULL;
    new->s_szc = 0;
    new->s_flags = 0;
    mutex_init(&new->s_pmtx, NULL, MUTEX_DEFAULT, NULL);
    new->s_phead.p_lnext = &new->s_phead;
    new->s_phead.p_lprev = &new->s_phead;
    if (seg_attach(as, segbase, segsize, new) < 0) {
        kmem_cache_free(seg_cache, new);
        return ((struct seg *)NULL);
    }
    /* caller must fill in ops, data */
    return (new);
}

/*
 * Attach a segment to the address space.  Used by seg_alloc()
 * and for kernel startup to attach to static segments.
 */
int
seg_attach(struct as *as, caddr_t base, size_t size, struct seg *seg)
{
    seg->s_as = as;
    seg->s_base = base;
    seg->s_size = size;

    /*
     * as_addseg() will add the segment at the appropraite point
     * in the list. It will return -1 if there is overlap with
     * an already existing segment.
     */
    return (as_addseg(as, seg));
}

/*
 * Unmap a segment and free it from its associated address space.
 * This should be called by anybody who's finished with a whole segment's
 * mapping.  Just calls SEGOP_UNMAP() on the whole mapping .  It is the
 * responsibility of the segment driver to unlink the the segment
 * from the address space, and to free public and private data structures
 * associated with the segment.  (This is typically done by a call to
 * seg_free()).
 */
void
seg_unmap(struct seg *seg)
{
#ifdef DEBUG
    int ret;
#endif /* DEBUG */

    ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));

    /* Shouldn't have called seg_unmap if mapping isn't yet established */
    ASSERT(seg->s_data != NULL);

    /* Unmap the whole mapping */
#ifdef DEBUG
    ret = SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
    ASSERT(ret == 0);
#else
    SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
#endif /* DEBUG */
}

/*
 * Free the segment from its associated as. This should only be called
 * if a mapping to the segment has not yet been established (e.g., if
 * an error occurs in the middle of doing an as_map when the segment
 * has already been partially set up) or if it has already been deleted
 * (e.g., from a segment driver unmap routine if the unmap applies to the
 * entire segment). If the mapping is currently set up then seg_unmap() should
 * be called instead.
 */
void
seg_free(struct seg *seg)
{
    register struct as *as = seg->s_as;
    struct seg *tseg = as_removeseg(as, seg);

    ASSERT(tseg == seg);

    /*
     * If the segment private data field is NULL,
     * then segment driver is not attached yet.
     */
    if (seg->s_data != NULL)
        SEGOP_FREE(seg);

    mutex_destroy(&seg->s_pmtx);
    ASSERT(seg->s_phead.p_lnext == &seg->s_phead);
    ASSERT(seg->s_phead.p_lprev == &seg->s_phead);
    kmem_cache_free(seg_cache, seg);
}

/*ARGSUSED*/
static void
seg_p_mem_config_post_add(
    void *arg,
    pgcnt_t delta_pages)
{
    /* Nothing to do. */
}

void
seg_p_enable(void)
{
    mutex_enter(&seg_pcache_mtx);
    ASSERT(seg_pdisabled != 0);
    seg_pdisabled--;
    mutex_exit(&seg_pcache_mtx);
}

/*
 * seg_p_disable - disables seg_pcache, and then attempts to empty the
 * cache.
 * Returns SEGP_SUCCESS if the cache was successfully emptied, or
 * SEGP_FAIL if the cache could not be emptied.
 */
int
seg_p_disable(void)
{
    pgcnt_t old_plocked;
    int stall_count = 0;

    mutex_enter(&seg_pcache_mtx);
    seg_pdisabled++;
    ASSERT(seg_pdisabled != 0);
    mutex_exit(&seg_pcache_mtx);

    /*
     * Attempt to empty the cache. Terminate if seg_plocked does not
     * diminish with SEGP_STALL_THRESHOLD consecutive attempts.
     */
    while (seg_plocked != 0) {
        ASSERT(seg_phashsize_win != 0);
        old_plocked = seg_plocked;
        seg_ppurge_async(1);
        if (seg_plocked == old_plocked) {
            if (stall_count++ > SEGP_STALL_THRESHOLD) {
                return (SEGP_FAIL);
            }
        } else
            stall_count = 0;
        if (seg_plocked != 0)
            delay(hz/SEGP_PREDEL_DELAY_FACTOR);
    }
    return (SEGP_SUCCESS);
}

/*
 * Attempt to purge seg_pcache.  May need to return before this has
 * completed to allow other pre_del callbacks to unlock pages. This is
 * ok because:
 *  1) The seg_pdisabled flag has been set so at least we won't
 *  cache anymore locks and the locks we couldn't purge
 *  will not be held if they do get released by a subsequent
 *  pre-delete callback.
 *
 *  2) The rest of the memory delete thread processing does not
 *  depend on the changes made in this pre-delete callback. No
 *  panics will result, the worst that will happen is that the
 *  DR code will timeout and cancel the delete.
 */
/*ARGSUSED*/
static int
seg_p_mem_config_pre_del(
    void *arg,
    pgcnt_t delta_pages)
{
    if (seg_phashsize_win == 0) {
        return (0);
    }
    if (seg_p_disable() != SEGP_SUCCESS)
        cmn_err(CE_NOTE,
            "!Pre-delete couldn't purge"" pagelock cache - continuing");
    return (0);
}

/*ARGSUSED*/
static void
seg_p_mem_config_post_del(
    void *arg,
    pgcnt_t delta_pages,
    int cancelled)
{
    if (seg_phashsize_win == 0) {
        return;
    }
    seg_p_enable();
}

static kphysm_setup_vector_t seg_p_mem_config_vec = {
    KPHYSM_SETUP_VECTOR_VERSION,
    seg_p_mem_config_post_add,
    seg_p_mem_config_pre_del,
    seg_p_mem_config_post_del,
};

static void
seg_pinit_mem_config(void)
{
    int ret;

    ret = kphysm_setup_func_register(&seg_p_mem_config_vec, (void *)NULL);
    /*
     * Want to catch this in the debug kernel. At run time, if the
     * callbacks don't get run all will be OK as the disable just makes
     * it more likely that the pages can be collected.
     */
    ASSERT(ret == 0);
}

/*
 * Verify that segment is not a shared anonymous segment which reserves
 * swap.  zone.max-swap accounting (zone->zone_max_swap) cannot be transfered
 * from one zone to another if any segments are shared.  This is because the
 * last process to exit will credit the swap reservation.  This could lead
 * to the swap being reserved by one zone, and credited to another.
 */
boolean_t
seg_can_change_zones(struct seg *seg)
{
    struct segvn_data *svd;

    if (seg->s_ops == &segspt_shmops)
        return (B_FALSE);

    if (seg->s_ops == &segvn_ops) {
        svd = (struct segvn_data *)seg->s_data;
        if (svd->type == MAP_SHARED &&
            svd->amp != NULL &&
            svd->amp->swresv > 0)
        return (B_FALSE);
    }
    return (B_TRUE);
}

/*
 * Return swap reserved by a segment backing a private mapping.
 */
size_t
seg_swresv(struct seg *seg)
{
    struct segvn_data *svd;
    size_t swap = 0;

    if (seg->s_ops == &segvn_ops) {
        svd = (struct segvn_data *)seg->s_data;
        if (svd->type == MAP_PRIVATE && svd->swresv > 0)
            swap = svd->swresv;
    }
    return (swap);
}

/*
 * General not supported function for SEGOP_INHERIT
 */
/* ARGSUSED */
int
seg_inherit_notsup(struct seg *seg, caddr_t addr, size_t len, uint_t op)
{
    return (ENOTSUP);
}