vm_page.c revision af4c679f647cf088543c762e33d41a3ac52cfa14
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
/*
* University Copyright- Copyright (c) 1982, 1986, 1988
* The Regents of the University of California
* All Rights Reserved
*
* University Acknowledgment- Portions of this document are derived from
* software developed by the University of California, Berkeley, and its
* contributors.
*/
/*
* VM - physical page management.
*/
#include <sys/tuneable.h>
#include <sys/sysmacros.h>
#include <sys/tnf_probe.h>
#include <sys/condvar_impl.h>
#include <sys/mem_config.h>
#include <sys/mem_cage.h>
#include <vm/seg_kmem.h>
#include <sys/vm_usage.h>
static int nopageage = 0;
/*
* freemem_lock protects all freemem variables:
* availrmem. Also this lock protects the globals which track the
* availrmem changes for accurate kernel footprint calculation.
* See below for an explanation of these
* globals.
*/
/*
* These globals track availrmem changes to get a more accurate
* estimate of tke kernel size. Historically pp_kernel is used for
* kernel size and is based on availrmem. But availrmem is adjusted for
* locked pages in the system not just for kernel locked pages.
* These new counters will track the pages locked through segvn and
* by explicit user locking.
*
* pages_locked : How many pages are locked because of user specified
* locking through mlock or plock.
*
* pages_useclaim,pages_claimed : These two variables track the
* claim adjustments because of the protection changes on a segvn segment.
*
* All these globals are protected by the same lock which protects availrmem.
*/
pgcnt_t pages_locked = 0;
pgcnt_t pages_useclaim = 0;
pgcnt_t pages_claimed = 0;
/*
* new_freemem_lock protects freemem, freemem_wait & freemem_cv.
*/
static kmutex_t new_freemem_lock;
static kcondvar_t freemem_cv;
/*
* The logical page free list is maintained as two lists, the 'free'
* and the 'cache' lists.
* The free list contains those pages that should be reused first.
*
* The implementation of the lists is machine dependent.
* page_get_freelist(), page_get_cachelist(),
* page_list_sub(), and page_list_add()
* form the interface to the machine dependent implementation.
*
* Pages with p_free set are on the cache list.
* Pages with p_free and p_age set are on the free list,
*
* A page may be locked while on either list.
*/
/*
* free list accounting stuff.
*
*
* Spread out the value for the number of pages on the
* page free and page cache lists. If there is just one
* value, then it must be under just one lock.
* The lock contention and cache traffic are a real bother.
*
* When we acquire and then drop a single pcf lock
* we can start in the middle of the array of pcf structures.
* If we acquire more than one pcf lock at a time, we need to
* start at the front to avoid deadlocking.
*
* pcf_count holds the number of pages in each pool.
*
* pcf_block is set when page_create_get_something() has asked the
* PSM page freelist and page cachelist routines without specifying
* a color and nothing came back. This is used to block anything
* else from moving pages from one list to the other while the
* lists are searched again. If a page is freeed while pcf_block is
* set, then pcf_reserve is incremented. pcgs_unblock() takes care
* of clearning pcf_block, doing the wakeups, etc.
*/
#define MAX_PCF_FANOUT NCPU
static uint_t pcf_fanout_mask = 0;
struct pcf {
};
/*
* PCF_INDEX hash needs to be dynamic (every so often the hash changes where
* it will hash the cpu to). This is done to prevent a drain condition
* from happening. This drain condition will occur when pcf_count decrement
* occurs on cpu A and the increment of pcf_count always occurs on cpu B. An
* example of this shows up with device interrupts. The dma buffer is allocated
* by the cpu requesting the IO thus the pcf_count is decremented based on that.
* When the memory is returned by the interrupt thread, the pcf_count will be
* incremented based on the cpu servicing the interrupt.
*/
static int pcf_decrement_bucket(pgcnt_t);
#ifdef VM_STATS
/*
* No locks, but so what, they are only statistics.
*/
static struct page_tcnt {
int pc_free_cache; /* free's into cache list */
int pc_free_dontneed; /* free's with dontneed */
int pc_free_pageout; /* free's from pageout */
int pc_free_free; /* free's into free list */
int pc_free_pages; /* free's into large page free list */
int pc_destroy_pages; /* large page destroy's */
int pc_get_cache; /* get's from cache list */
int pc_get_free; /* get's from free list */
int pc_reclaim; /* reclaim's */
int pc_abortfree; /* abort's of free pages */
int pc_find_hit; /* find's that find page */
int pc_find_miss; /* find's that don't find page */
int pc_destroy_free; /* # of free pages destroyed */
int pc_addclaim_pages;
int pc_subclaim_pages;
int pc_free_replacement_page[2];
int pc_try_demote_pages[6];
int pc_demote_pages[2];
} pagecnt;
/*
* Collects statistics.
*/
\
break; \
} \
pagecnt.pc_find_hit++; \
else \
pagecnt.pc_find_miss++; \
if (mylen > PC_HASH_CNT) \
mylen = PC_HASH_CNT; \
}
#else /* VM_STATS */
/*
* Don't collect statistics
*/
break; \
} \
}
#endif /* VM_STATS */
#ifdef DEBUG
#define MEMSEG_SEARCH_STATS
#endif
#ifdef MEMSEG_SEARCH_STATS
struct memseg_stats {
} memseg_stats;
#define MEMSEG_STAT_INCR(v) \
#else
#define MEMSEG_STAT_INCR(x)
#endif
/*
*
* Setting to LPAP_LOCAL will heavily prefer the local lgroup over remote lgroup
* for large page allocation requests. If a large page is not readily
* avaliable on the local freelists we will go through additional effort
* to create a large page, potentially moving smaller pages around to coalesce
* larger pages in the local lgroup.
* Default value of LPAP_DEFAULT will go to remote freelists if large pages
* are not readily available in the local lgroup.
*/
enum lpap {
LPAP_DEFAULT, /* default large page allocation policy */
LPAP_LOCAL /* local large page allocation policy */
};
static void page_init_mem_config(void);
static void page_do_hashout(page_t *);
static void page_capture_init();
static void page_demote_vp_pages(page_t *);
void
pcf_init(void)
{
if (boot_ncpus != -1) {
} else {
}
#ifdef sun4v
/*
* Force at least 4 buckets if possible for sun4v.
*/
#endif /* sun4v */
/*
* Round up to the nearest power of 2.
*/
if (!ISP2(pcf_fanout)) {
if (pcf_fanout > MAX_PCF_FANOUT) {
}
}
}
/*
* vm subsystem related initialization
*/
void
vm_init(void)
{
boolean_t callb_vm_cpr(void *, int);
}
/*
* This function is called at startup and when memory is added or deleted.
*/
void
{
static pgcnt_t pages_pp_maximum_startup;
static pgcnt_t avrmem_delta;
static int init_done;
if (init_done == 0) {
/* If the user specified a value, save it */
if (pages_pp_maximum != 0) {
user_set = 1;
}
/*
* Setting of pages_pp_maximum is based first time
* on the value of availrmem just after the start-up
* allocations. To preserve this relationship at run
* time, use a delta from availrmem_initial.
*/
/* The allowable floor of pages_pp_maximum */
/* Make sure we don't come through here again. */
init_done = 1;
}
/*
* Determine pages_pp_maximum, the number of currently available
* pages (availrmem) that can't be `locked'. If not set by
* the user, we set it to 4% of the currently available memory
* plus 4MB.
* But we also insist that it be greater than tune.t_minarmem;
* otherwise a process could lock down a lot of memory, get swapped
* out, and never have enough to get swapped back in.
*/
if (user_set)
else
if (pages_pp_maximum <= p_min) {
}
}
void
{
}
static pgcnt_t pending_delete;
/*ARGSUSED*/
static void
void *arg,
{
}
/*ARGSUSED*/
static int
void *arg,
{
return (0);
}
/*ARGSUSED*/
static void
void *arg,
int cancelled)
{
if (!cancelled)
}
static kphysm_setup_vector_t page_mem_config_vec = {
};
static void
page_init_mem_config(void)
{
int ret;
}
/*
* Evenly spread out the PCF counters for large free pages
*/
static void
{
while (npages > 0) {
} else {
npages = 0;
}
p = pcf;
}
}
/*
* Add a physical chunk of memory to the system free lists during startup.
* Platform specific startup() allocates the memory for the page structs.
*
* num - number of page structures
* base - page number (pfn) to be associated with the first page.
*
* Since we are doing this during startup (ie. single threaded), we will
* use shortcut routines to avoid any locking overhead while putting all
* these pages on the freelists.
*
* NOTE: Any changes performed to page_free(), must also be performed to
* add_physmem() since this is how we initialize all page_t's at
* boot time.
*/
void
{
/*
* Arbitrarily limit the max page_get request
* to 1/2 of the page structs we have.
*/
total_pages += num;
/*
* The physical space for the pages array
* representing ram pages has already been
* allocated. Here we initialize each lock
* in the page structure, and put each on
* the free list
*/
/*
* this needs to fill in the page number
* and do any other arch specific initialization
*/
/*
* Initialize the page lock as unlocked, since nobody
* can see or access this page yet.
*/
/*
* Initialize IO lock
*/
/*
* initialize other fields in the page_t
*/
PP_SETFREE(pp);
PP_SETAGED(pp);
/*
* Simple case: System doesn't support large pages.
*/
if (szc == 0) {
continue;
}
/*
* Handle unaligned pages, we collect them up onto
* the root page until we have a full large page.
*/
/*
* If not in a large page,
* just free as small page.
*/
continue;
}
/*
* Link a constituent page into the large page.
*/
/*
* When large page is fully formed, free it.
*/
cnt = 0;
}
continue;
}
/*
* At this point we have a page number which
* is aligned. We assert that we aren't already
* in a different large page.
*/
/*
* If insufficient number of pages left to form
* a large page, just free the small page.
*/
continue;
}
/*
* Otherwise start a new large page.
*/
cnt++;
}
}
/*
* Find a page representing the specified [vp, offset].
* If we find the page but it is intransit coming in,
* it will have an "exclusive" lock and we wait for
* the i/o to complete. A page found on the free list
* is always reclaimed and then locked. On success, the page
* is locked, its data is valid and it isn't on the free
* list, while a NULL is returned if the page doesn't exist.
*/
page_t *
{
}
/*
* Find a page representing the specified [vp, offset].
* We either return the one we found or, if passed in,
* create one with identity of [vp, offset] of the
* pre-allocated page. If we find existing page but it is
* intransit coming in, it will have an "exclusive" lock
* and we wait for the i/o to complete. A page found on
* the free list is always reclaimed and then locked.
* On success, the page is locked, its data is valid and
* it isn't on the free list, while a NULL is returned
* if the page doesn't exist and newpp is NULL;
*/
page_t *
int flags)
{
/*
* Acquire the appropriate page hash lock since
* we have to search the hash list. Pages that
* hash to this list can't change identity while
* this lock is held.
*/
hash_locked = 0;
top:
if (!hash_locked) {
/*
* On a miss, acquire the phm. Then
* next time, page_lock() will be called,
* causing a wait if the page is busy.
* just looping with page_trylock() would
* get pretty boring.
*/
hash_locked = 1;
goto top;
}
} else {
goto top;
}
}
/*
* Since `pp' is locked it can not change identity now.
* Reconfirm we locked the correct page.
*
* Both the p_vnode and p_offset *must* be cast volatile
* to force a reload of their values: The PAGE_HASH_SEARCH
* macro will have stuffed p_vnode and p_offset into
* registers before calling page_trylock(); another thread,
* actually holding the hash lock, could have changed the
* page's identity in memory, but our registers would not
* be changed, fooling the reconfirmation. If the hash
* lock was held during the search, the casting would
* not be needed.
*/
if (hash_locked) {
panic("page_lookup_create: lost page %p",
(void *)pp);
/*NOTREACHED*/
}
hash_locked = 1;
goto top;
}
/*
* If page_trylock() was called, then pp may still be on
* the cachelist (can't be on the free list, it would not
* have been found in the search). If it is on the
* cachelist it must be pulled now. To pull the page from
* the cachelist, it must be exclusively locked.
*
* The other big difference between page_trylock() and
* page_lock(), is that page_lock() will pull the
* page from whatever free list (the cache list in this
* case) the page is on. If page_trylock() was used
* above, then we have to do the reclaim ourselves.
*/
/*
* page_relcaim will insure that we
* have this page exclusively
*/
/*
* Page_reclaim dropped whatever lock
* we held.
*/
hash_locked = 1;
goto top;
}
}
if (hash_locked) {
}
NULL);
if (*nrelocp > 0) {
page_lookup_cnt[11]);
page_lookup_cnt[12]);
} else {
}
}
}
page_lookup_cnt[14]);
page_lookup_cnt[15]);
page_lookup_cnt[16]);
}
} else if (!hash_locked) {
hash_locked = 1;
goto top;
/*
* If we have a preallocated page then
* insert it now and basically behave like
* page_create.
*/
/*
* Since we hold the page hash mutex and
* just searched for this page, page_hashin
* had better not fail. If it does, that
* means some thread did not follow the
* page hash mutex rules. Panic now and
* get it over with. As usual, go down
* holding all the locks.
*/
panic("page_lookup_create: hashin failed %p %p %llx %p",
/*NOTREACHED*/
}
} else {
}
return (pp);
}
/*
* Search the hash list for the page representing the
* specified [vp, offset] and return it locked. Skip
* free pages and pages that cannot be locked as requested.
* Used while attempting to kluster pages.
*/
page_t *
{
locked = 0;
top:
locked = 1;
}
} else {
} else {
/*
* See the comment in page_lookup()
*/
if (locked) {
panic("page_lookup_nowait %p",
(void *)pp);
/*NOTREACHED*/
}
goto top;
}
}
}
}
if (locked) {
}
return (pp);
}
/*
* Search the hash list for a page with the specified [vp, off]
* that is known to exist and is already locked. This routine
* is typically used by segment SOFTUNLOCK routines.
*/
page_t *
{
return (pp);
}
/*
* Determine whether a page with the specified [vp, off]
* currently exists in the system. Obviously this should
* only be considered as a hint since nothing prevents the
* page from disappearing or appearing immediately after
* the return from this routine. Subsequently, we don't
* even bother to lock the list.
*/
page_t *
{
return (pp);
}
/*
* Determine if physically contiguous pages exist for [vp, off] - [vp, off +
* page_size(szc)) range. if they exist and ppa is not NULL fill ppa array
* with these pages locked SHARED. If necessary reclaim pages from
* freelist. Return 1 if contiguous pages exist and 0 otherwise.
*
* If we fail to lock pages still return 1 if pages exist and contiguous.
* But in this case return value is just a hint. ppa array won't be filled.
* Caller should initialize ppa[0] as NULL to distinguish return value.
*
* Returns 0 if pages don't exist or not physically contiguous.
*
* This routine doesn't work for anonymous(swapfs) pages.
*/
int
{
pgcnt_t i;
pgcnt_t j;
int loopcnt = 0;
if (++loopcnt > 3) {
return (0);
}
return (0);
}
return (1);
}
/*
* Also check whether p_pagenum was modified by DR.
*/
goto again;
}
/*
* szc was non zero and vnode and offset matched after we
* locked the page it means it can't become free on us.
*/
return (0);
}
pp++;
pfn++;
pp--;
while (i-- > 0) {
pp--;
}
return (1);
}
pp--;
while (i-- > 0) {
pp--;
}
goto again;
}
/*
* szc the same as for previous already locked pages
* with right identity. Since this page had correct
* szc after we locked it can't get freed or destroyed
* and therefore must have the expected identity.
*/
panic("page_exists_physcontig: "
"large page identity doesn't match");
}
}
return (1);
return (0);
}
return (1);
}
return (0);
}
if (page_numtomemseg_nolock(pfn) !=
return (0);
}
/*
* We loop up 4 times across pages to promote page size.
* We're extra cautious to promote page size atomically with respect
* to everybody else. But we can probably optimize into 1 loop if
* this becomes an issue.
*/
break;
}
/*
* Check whether p_pagenum was modified by DR.
*/
break;
}
break;
}
ASSERT(i == 0);
goto again;
}
}
if (i != pages) {
--pp;
while (i-- > 0) {
--pp;
}
return (0);
}
break;
}
} else {
}
}
if (i < pages) {
/*
* page_reclaim failed because we were out of memory.
* drop the rest of the locks and return because this page
* must be already reallocated anyway.
*/
if (j != i) {
}
}
return (0);
}
}
} else {
page_downgrade(ppa[i]);
}
}
}
return (1);
}
/*
* Determine whether a page with the specified [vp, off]
* currently exists in the system and if so return its
* size code. Obviously this should only be considered as
* a hint since nothing prevents the page from disappearing
* or appearing immediately after the return from this routine.
*/
int
{
int rc = 0;
rc = 1;
}
return (rc);
}
/* wakeup threads waiting for pages in page_create_get_something() */
void
wakeup_pcgs(void)
{
if (!CV_HAS_WAITERS(&pcgs_cv))
return;
}
/*
* 'freemem' is used all over the kernel as an indication of how many
* pages are free (either on the cache list or on the free page list)
* in the system. In very few places is a really accurate 'freemem'
* needed. To avoid contention of the lock protecting a the
* single freemem, it was spread out into NCPU buckets. Set_freemem
* sets freemem to the total of all NCPU buckets. It is called from
* clock() on each TICK.
*/
void
{
struct pcf *p;
ulong_t t;
uint_t i;
t = 0;
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
t += p->pcf_count;
p++;
}
freemem = t;
/*
* Don't worry about grabbing mutex. It's not that
* critical if we miss a tick or two. This is
* where we wakeup possible delayers in
* page_create_get_something().
*/
wakeup_pcgs();
}
{
struct pcf *p;
ulong_t t;
uint_t i;
t = 0;
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
t += p->pcf_count;
p++;
}
/*
* We just calculated it, might as well set it.
*/
freemem = t;
return (t);
}
/*
* Acquire all of the page cache & free (pcf) locks.
*/
void
{
struct pcf *p;
uint_t i;
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
mutex_enter(&p->pcf_lock);
p++;
}
}
/*
* Release all the pcf_locks.
*/
void
{
struct pcf *p;
uint_t i;
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
mutex_exit(&p->pcf_lock);
p++;
}
}
/*
* Inform the VM system that we need some pages freed up.
* Calls must be symmetric, e.g.:
*
* page_needfree(100);
* wait a bit;
* page_needfree(-100);
*/
void
{
}
/*
* Throttle for page_create(): try to prevent freemem from dropping
* below throttlefree. We can't provide a 100% guarantee because
* KM_NOSLEEP allocations, page_reclaim(), and various other things
* nibble away at the freelist. However, we can block all PG_WAIT
* allocations until memory becomes available. The motivation is
* that several things can fall apart when there's no free memory:
*
* (1) If pageout() needs memory to push a page, the system deadlocks.
*
* (2) By (broken) specification, timeout(9F) can neither fail nor
* block, so it has no choice but to panic the system if it
* cannot allocate a callout structure.
*
* (3) Like timeout(), ddi_set_callback() cannot fail and cannot block;
* it panics if it cannot allocate a callback structure.
*
* (4) Untold numbers of third-party drivers have not yet been hardened
* success and panic the system with a data fault on failure.
* (The long-term solution to this particular problem is to ship
* hostile fault-injecting DEBUG kernels with the DDK.)
*
* It is theoretically impossible to guarantee success of non-blocking
* allocations, but in practice, this throttle is very hard to break.
*/
static int
{
uint_t i;
/*
* Never deny pages when:
* - it's a thread that cannot block [NOMEMWAIT()]
* - the allocation cannot block and must not fail
* - the allocation cannot block and is pageout dispensated
*/
if (NOMEMWAIT() ||
return (1);
/*
* If the allocation can't block, we look favorably upon it
* unless we're below pageout_reserve. In that case we fail
* the allocation because we want to make sure there are a few
* pages available for pageout.
*/
/* Calculate the effective throttlefree value */
tf = throttlefree -
for (;;) {
fm = 0;
for (i = 0; i < pcf_fanout; i++) {
}
break;
}
freemem_wait++;
freemem_wait--;
}
return (1);
}
/*
* page_create_wait() is called to either coalesce pages from the
* different pcf buckets or to wait because there simply are not
* enough pages to satisfy the caller's request.
*
* Sadly, this is called from platform/vm/vm_machdep.c
*/
int
{
uint_t i;
struct pcf *p;
/*
* Wait until there are enough free pages to satisfy our
* entire request.
* We set needfree += npages before prodding pageout, to make sure
* it does real work when npages > lotsfree > freemem.
*/
if ((flags & PG_NORELOC) &&
return (0);
if (pcf_decrement_bucket(npages) ||
return (1);
/*
* All of the pcf locks are held, there are not enough pages
* to satisfy the request (npages < total).
* Be sure to acquire the new_freemem_lock before dropping
* the pcf locks. This prevents dropping wakeups in page_free().
* The order is always pcf_lock then new_freemem_lock.
*
* Since we hold all the pcf locks, it is a good time to set freemem.
*
* If the caller does not want to wait, return now.
* Else turn the pageout daemon loose to find something
* and wait till it does.
*
*/
return (0);
}
"page_create_sleep_start: freemem %ld needfree %ld",
/*
* We are going to wait.
* We currently hold all of the pcf_locks,
* get the new_freemem_lock (it protects freemem_wait),
* before dropping the pcf_locks.
*/
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
p->pcf_wait++;
mutex_exit(&p->pcf_lock);
p++;
}
freemem_wait++;
freemem_wait--;
"page_create_sleep_end: freemem %ld needfree %ld",
goto checkagain;
}
/*
* A routine to do the opposite of page_create_wait().
*/
void
{
struct pcf *p;
/*
* When a contiguous lump is broken up, we have to
* deal with lots of pages (min 64) so lets spread
* the wealth around.
*/
mutex_enter(&p->pcf_lock);
if (p->pcf_block) {
which = &p->pcf_reserve;
}
} else {
npages = 0;
}
if (p->pcf_wait) {
/*
* Check to see if some other thread
* is actually waiting. Another bucket
* may have woken it up by now. If there
* are no waiters, then set our pcf_wait
* count to zero to avoid coming in here
* next time.
*/
if (freemem_wait) {
if (npages > 1) {
} else {
}
p->pcf_wait--;
} else {
p->pcf_wait = 0;
}
}
mutex_exit(&p->pcf_lock);
}
}
/*
* A helper routine for page_create_get_something.
* The indenting got to deep down there.
* Unblock the pcf counters. Any pages freed after
* pcf_block got set are moved to pcf_count and
* wakeups (cv_broadcast() or cv_signal()) are done as needed.
*/
static void
pcgs_unblock(void)
{
int i;
struct pcf *p;
/* Update freemem while we're here. */
freemem = 0;
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
mutex_enter(&p->pcf_lock);
p->pcf_count = p->pcf_reserve;
p->pcf_block = 0;
if (p->pcf_wait) {
if (freemem_wait) {
if (p->pcf_reserve > 1) {
p->pcf_wait = 0;
} else {
p->pcf_wait--;
}
} else {
p->pcf_wait = 0;
}
}
p->pcf_reserve = 0;
mutex_exit(&p->pcf_lock);
p++;
}
}
/*
* Called from page_create_va() when both the cache and free lists
* have been checked once.
*
* Either returns a page or panics since the accounting was done
* way before we got here.
*
* We don't come here often, so leave the accounting on permanently.
*/
#define MAX_PCGS 100
#ifdef DEBUG
#define PCGS_TRIES 100
#else /* DEBUG */
#define PCGS_TRIES 10
#endif /* DEBUG */
#ifdef VM_STATS
#endif /* VM_STATS */
static page_t *
{
struct pcf *p;
int cagelocked = 0;
/*
* Tap any reserve freelists: if we fail now, we'll die
* since the page(s) we're looking for have already been
* accounted for.
*/
if ((flags & PG_NORELOC) != 0) {
/*
* Requests for free pages from critical threads
* such as pageout still won't throttle here, but
* we must try again, to give the cageout thread
* another chance to catch up. Since we already
* accounted for the pages, we had better get them
* this time.
*
* N.B. All non-critical threads acquire the pcgs_cagelock
* to serialize access to the freelists. This implements a
* turnstile-type synchornization to avoid starvation of
* critical requests for PG_NORELOC memory by non-critical
* threads: all non-critical threads must acquire a 'ticket'
* before passing through, which entails making sure
* kcage_freemem won't fall below minfree prior to grabbing
* pages from the freelists.
*/
cagelocked = 1;
}
}
/*
* Time to get serious.
* We failed to get a `correctly colored' page from both the
* free and cache lists.
* We escalate in stage.
*
* First try both lists without worring about color.
*
* Then, grab all page accounting locks (ie. pcf[]) and
* steal any pages that they have and set the pcf_block flag to
* stop deletions from the lists. This will help because
* a page can get added to the free list while we are looking
* at the cache list, then another page could be added to the cache
* list allowing the page on the free list to be removed as we
* move from looking at the cache list to the free list. This
* could happen over and over. We would never find the page
* we have accounted for.
*
* Noreloc pages are a subset of the global (relocatable) page pool.
* They are not tracked separately in the pcf bins, so it is
* impossible to know when doing pcf accounting if the available
* page(s) are noreloc pages or not. When looking for a noreloc page
* it is quite easy to end up here even if the global (relocatable)
* page pool has plenty of free pages but the noreloc pool is empty.
*
* When the noreloc pool is empty (or low), additional noreloc pages
* are created by converting pages from the global page pool. This
* process will stall during pcf accounting if the pcf bins are
* already locked. Such is the case when a noreloc allocation is
* looping here in page_create_get_something waiting for more noreloc
* pages to appear.
*
* Short of adding a new field to the pcf bins to accurately track
* the number of free noreloc pages, we instead do not grab the
* pcgs_lock, do not set the pcf blocks and do not timeout when
* allocating a noreloc page. This allows noreloc allocations to
* loop without blocking global page pool allocations.
*
* NOTE: the behaviour of page_create_get_something has not changed
* for the case of global page pool allocations.
*/
flags &= ~PG_MATCH_COLOR;
locked = 0;
#endif
}
/*
* Serialize. Don't fight with other pcgs().
*/
locked = 1;
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
mutex_enter(&p->pcf_lock);
p->pcf_block = 1;
p->pcf_reserve = p->pcf_count;
p->pcf_count = 0;
mutex_exit(&p->pcf_lock);
p++;
}
freemem = 0;
}
if (count) {
/*
* Since page_free() puts pages on
* a list then accounts for it, we
* just have to wait for page_free()
* to unlock any page it was working
* with. The page_lock()-page_reclaim()
* path falls in the same boat.
*
* We don't need to check on the
* PG_WAIT flag, we have already
* accounted for the page we are
* looking for in page_create_va().
*
* We just wait a moment to let any
* locked pages on the lists free up,
* then continue around and try again.
*
* Will be awakened by set_freemem().
*/
}
} else {
#ifdef VM_STATS
if (count >= PCGS_TRIES) {
} else {
}
#endif
if (locked) {
pcgs_unblock();
}
if (cagelocked)
return (pp);
}
}
/*
* we go down holding the pcf locks.
*/
panic("no %spage found %d",
/*NOTREACHED*/
}
/*
* Create enough pages for "bytes" worth of data starting at
* "off" in "vp".
*
* Where flag must be one of:
*
* PG_EXCL: Exclusive create (fail if any page already
* exists in the page cache) which does not
* wait for memory to become available.
*
* PG_WAIT: Non-exclusive create which can wait for
* memory to become available.
*
* PG_PHYSCONTIG: Allocate physically contiguous pages.
* (Not Supported)
*
* A doubly linked list of pages is returned to the caller. Each page
* on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock)
* lock.
*
* Unable to change the parameters to page_create() in a minor release,
* we renamed page_create() to page_create_va(), changed all known calls
* from page_create() to page_create_va(), and created this wrapper.
*
* Upon a major release, we should break compatibility by deleting this
* wrapper, and replacing all the strings "page_create_va", with "page_create".
*
* NOTE: There is a copy of this interface as page_create_io() in
* i86/vm/vm_machdep.c. Any bugs fixed here should be applied
* there.
*/
page_t *
{
#ifdef DEBUG
(void *)caller());
#endif
}
#ifdef DEBUG
#endif
/*
* Used for large page support. It will attempt to allocate
* a large page(s) off the freelist.
*
* Returns non zero on failure.
*/
int
{
int err = 0;
/*
* Check if system heavily prefers local large pages over remote
* on systems with multiple lgroups.
*/
}
VM_STAT_ADD(alloc_pages[0]);
#ifdef DEBUG
return (ENOMEM);
}
#endif
/*
* One must be NULL but not both.
* And one must be non NULL but not both.
*/
while (page_chk_freelist(szc) == 0) {
return (ENOMEM);
}
#endif
0, lgrp);
}
} else {
0, lgrp);
}
} else if (anypgsz) {
szc--;
} else {
return (ENOMEM);
}
}
if (szc == 0) {
}
/*
* Clear the free and age bits. Also if we were passed in a ppa then
* fill it in with all the constituent pages from the large page. But
* if we failed to allocate all the pages just free what we got.
*/
while (npgs != 0) {
if (err == 0) {
PP_CLRFREE(pp);
PP_CLRAGED(pp);
npgs--;
} else {
page_list_add_pages(pp, 0);
}
} else {
PP_CLRFREE(pp);
PP_CLRAGED(pp);
npgs--;
}
}
return (err);
}
/*
* Get a single large page off of the freelists, and set it up for use.
* Number of bytes requested must be a supported page size.
*
* Note that this call may fail even if there is sufficient
* memory available or PG_WAIT is set, so the caller must
* be willing to fallback on page_create_va(), block and retry,
* or fail the requester.
*/
page_t *
{
/* but no others */
/*
* Cage is OFF, or we are single threaded in
* panic, so make everything a RELOC request.
*/
flags &= ~PG_NORELOC;
}
/*
* Make sure there's adequate physical memory available.
* Note: PG_WAIT is ignored here.
*/
return (NULL);
}
/*
* If cage is on, dampen draw from cage when available
* cage space is low.
*/
/*
* The cage is on, the caller wants PG_NORELOC
* pages and available cage memory is very low.
* Call kcage_create_throttle() to attempt to
* control demand on the cage.
*/
return (NULL);
}
}
if (!pcf_decrement_bucket(npages) &&
return (NULL);
}
/*
* This is where this function behaves fundamentally differently
* than page_create_va(); since we're intending to map the page
* with a single TTE, we have to get it as a physically contiguous
* hardware pagesize chunk. If we can't, we fail.
*/
else
return (NULL);
}
/*
* if we got the page with the wrong mtype give it back this is a
* workaround for CR 6249718. When CR 6249718 is fixed we never get
* inside "if" and the workaround becomes just a nop
*/
return (NULL);
}
/*
* If satisfying this request has left us with too little
* memory, start the wheels turning to get some back. The
* first clause of the test prevents waking up the pageout
* daemon in situations where it would decide that there's
* nothing to do.
*/
"pageout_cv_signal:freemem %ld", freemem);
}
while (npages--) {
PP_CLRFREE(pp);
PP_CLRAGED(pp);
panic("page_create_large: hashin failed: page %p",
(void *)pp);
}
return (rootpp);
}
page_t *
{
pgcnt_t found_on_free = 0;
struct pcf *p;
"page_create_start:vp %p off %llx bytes %lu flags %x",
panic("page_create: invalid flags");
/*NOTREACHED*/
}
/* but no others */
/*
* Try to see whether request is too large to *ever* be
* satisfied, in order to prevent deadlock. We arbitrarily
* decide to limit maximum size requests to max_page_get.
*/
if (npages >= max_page_get) {
"page_create_toobig:vp %p off %llx npages "
"%lu max_page_get %lu",
return (NULL);
} else {
"Request for too much kernel memory "
"(%lu bytes), will hang forever", bytes);
for (;;)
delay(1000000000);
}
}
/*
* Cage is OFF, or we are single threaded in
* panic, so make everything a RELOC request.
*/
flags &= ~PG_NORELOC;
}
return (NULL);
/*
* If cage is on, dampen draw from cage when available
* cage space is low.
*/
if ((flags & PG_NORELOC) &&
/*
* The cage is on, the caller wants PG_NORELOC
* pages and available cage memory is very low.
* Call kcage_create_throttle() to attempt to
* control demand on the cage.
*/
return (NULL);
}
if (!pcf_decrement_bucket(npages)) {
/*
* Have to look harder. If npages is greater than
* one, then we might have to coalesce the counters.
*
* Go wait. We come back having accounted
* for the memory.
*/
return (NULL);
}
}
/*
* If satisfying this request has left us with too little
* memory, start the wheels turning to get some back. The
* first clause of the test prevents waking up the pageout
* daemon in situations where it would decide that there's
* nothing to do.
*/
"pageout_cv_signal:freemem %ld", freemem);
}
/*
* Loop around collecting the requested number of pages.
* Most of the time, we have to `create' a new page. With
* this in mind, pull the page off the free list before
* getting the hash lock. This will minimize the hash
* lock hold time, nesting, and the like. If it turns
* out we don't need the page, we put it back at the end.
*/
while (npages--) {
top:
/*
* Try to get a page from the freelist (ie,
* a page with no [vp, off] tag). If that
* fails, use the cachelist.
*
* During the first attempt at both the free
* and cache lists we try for the correct color.
*/
/*
* XXXX-how do we deal with virtual indexed
* caches and and colors?
*/
/*
* Get lgroup to allocate next page of shared memory
* from and use it to specify where to allocate
* the physical memory
*/
flags & ~PG_MATCH_COLOR);
}
/*
* Since this page came from the
* cachelist, we must destroy the
* old vnode association.
*/
}
}
}
/*
* We own this page!
*/
/*
* Here we have a page in our hot little mits and are
* just waiting to stuff it on the appropriate lists.
* Get the mutex and check to see if it really does
* not exist.
*/
/*
* Since we hold the page hash mutex and
* just searched for this page, page_hashin
* had better not fail. If it does, that
* means somethread did not follow the
* page hash mutex rules. Panic now and
* get it over with. As usual, go down
* holding all the locks.
*/
panic("page_create: "
"hashin failed %p %p %llx %p",
/*NOTREACHED*/
}
/*
* Hat layer locking need not be done to set
* the following bits since the page is not hashed
* and was on the free list (i.e., had no mappings).
*
* Set the reference bit to protect
* against immediate pageout
*
* XXXmh modify freelist code to set reference
* bit so we don't have to do it here.
*/
} else {
/*
* Found an existing page, and the caller
* wanted all new pages. Undo all of the work
* we have done.
*/
/* large pages should not end up here */
/*LINTED: constant in conditional ctx*/
}
goto fail;
}
/*
* Start all over again if we blocked trying
* to lock the page.
*/
goto top;
}
PP_CLRFREE(pp);
}
}
/*
* Got a page! It is locked. Acquire the i/o
* lock since we are going to use the p_next and
* p_prev fields to link the requested pages together.
*/
}
fail:
/*
* Did not need this page after all.
* Put it back on the free list.
*/
}
{
if (overshoot) {
mutex_enter(&p->pcf_lock);
if (p->pcf_block) {
p->pcf_reserve += overshoot;
} else {
if (p->pcf_wait) {
if (freemem_wait) {
p->pcf_wait--;
} else {
p->pcf_wait = 0;
}
}
}
mutex_exit(&p->pcf_lock);
/* freemem is approximate, so this test OK */
if (!p->pcf_block)
}
}
return (plist);
}
/*
* One or more constituent pages of this large page has been marked
* toxic. Simply demote the large page to PAGESIZE pages and let
* page_free() handle it. This routine should only be called by
* large page free routines (page_free_pages() and page_destroy_pages().
* All pages are locked SE_EXCL and have already been marked free.
*/
static void
{
}
}
}
/*
* Put page on the "free" list.
* The free list is really two lists maintained by
* the PSM of whatever machine we happen to be on.
*/
void
{
struct pcf *p;
}
panic("page_free: anon or kernel "
"or no vnode large page %p", (void *)pp);
}
}
/*
* The page_struct_lock need not be acquired to examine these
* fields since the page has an "exclusive" lock.
*/
panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d "
/*NOTREACHED*/
}
PP_SETFREE(pp);
/*
* Now we add the page to the head of the free list.
* But if this page is associated with a paged vnode
* then we adjust the head forward so that the page is
* effectively at the end of the list.
*/
/*
* Page has no identity, put it on the free list.
*/
PP_SETAGED(pp);
"page_free_free:pp %p", pp);
} else {
PP_CLRAGED(pp);
/* move it to the tail of the list */
"page_free_cache_tail:pp %p", pp);
} else {
"page_free_cache_head:pp %p", pp);
}
}
/*
* Now do the `freemem' accounting.
*/
mutex_enter(&p->pcf_lock);
if (p->pcf_block) {
p->pcf_reserve += 1;
} else {
p->pcf_count += 1;
if (p->pcf_wait) {
/*
* Check to see if some other thread
* is actually waiting. Another bucket
* may have woken it up by now. If there
* are no waiters, then set our pcf_wait
* count to zero to avoid coming in here
* next time. Also, since only one page
* was put on the free list, just wake
* up one waiter.
*/
if (freemem_wait) {
p->pcf_wait--;
} else {
p->pcf_wait = 0;
}
}
}
mutex_exit(&p->pcf_lock);
/* freemem is approximate, so this test OK */
if (!p->pcf_block)
freemem += 1;
}
/*
* Put page on the "free" list during intial startup.
* This happens during initial single threaded execution.
*/
void
{
struct pcf *p;
/*
* Now do the `freemem' accounting.
*/
p->pcf_count += 1;
/* freemem is approximate, so this is OK */
freemem += 1;
}
void
{
pgcnt_t i;
"page_free_free:pp %p", pp);
/*NOTREACHED*/
}
/*NOTREACHED*/
}
/*NOTREACHED*/
}
}
}
int free_pages = 1;
/*
* This routine attempts to return pages to the cachelist via page_release().
* It does not *have* to be successful in all cases, since the pageout scanner
* will catch any pages it misses. It does need to be fast and not introduce
* too much overhead.
*
* If a page isn't found on the unlocked sweep of the page_hash bucket, we
* don't lock and retry. This is ok, since the page scanner will eventually
* find any page we miss in free_vp_pages().
*/
void
{
if (free_pages == 0)
return;
return;
/*
* find the page using a fast, but inexact search. It'll be OK
* if a few pages slip through the cracks here.
*/
/*
* If we didn't find the page (it may not exist), the page
* is free, looks still in use (shared), or we can't lock it,
* just give up.
*/
page_share_cnt(pp) > 0 ||
continue;
/*
* Once we have locked pp, verify that it's still the
* correct page and not already free
*/
continue;
}
/*
* try to release the page...
*/
}
}
/*
* Reclaim the given page from the free list.
* If pp is part of a large pages, only the given constituent page is reclaimed
* and the large page it belonged to will be demoted. This can only happen
* if the page is not on the cachelist.
*
* Returns 1 on success or 0 on failure.
*
* The page is unlocked if it can't be reclaimed (when freemem == 0).
* If `lock' is non-null, it will be dropped and re-acquired if
* the routine must wait while freemem is 0.
*
* As it turns out, boot_getpages() does this. It picks a page,
* based on where OBP mapped in some address, gets its pfn, searches
* the memsegs, locks the page, then pulls it off the free list!
*/
int
{
struct pcf *p;
int enough;
uint_t i;
/*
* If `freemem' is 0, we cannot reclaim this page from the
* freelist, so release every lock we might hold: the page,
* and the `lock' before blocking.
*
* The only way `freemem' can become 0 while there are pages
* marked free (have their p->p_free bit set) is when the
* system is low on memory and doing a page_create(). In
* order to guarantee that once page_create() starts acquiring
* pages it will be able to get all that it needs since `freemem'
* was decreased by the requested amount. So, we need to release
* this page, and let page_create() have it.
*
* Since `freemem' being zero is not supposed to happen, just
* use the usual hash stuff as a starting point. If that bucket
* is empty, then assume the worst, and start at the beginning
* of the pcf array. If we always start at the beginning
* when acquiring more than one pcf lock, there won't be any
* deadlock problems.
*/
/* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
goto page_reclaim_nomem;
}
if (!enough) {
/*
* Check again. Its possible that some other thread
* could have been right behind us, and added one
* to a list somewhere. Acquire each of the pcf locks
* until we find a page.
*/
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
mutex_enter(&p->pcf_lock);
if (p->pcf_count >= 1) {
p->pcf_count -= 1;
/*
* freemem is not protected by any lock. Thus,
* we cannot have any assertion containing
* freemem here.
*/
freemem -= 1;
enough = 1;
break;
}
p++;
}
if (!enough) {
/*
* We really can't have page `pp'.
* Time for the no-memory dance with
* page_free(). This is just like
* page_create_wait(). Plus the added
* attraction of releasing whatever mutex
* we held when we were called with in `lock'.
* Page_unlock() will wakeup any thread
* waiting around for this page.
*/
if (lock) {
}
/*
* get this before we drop all the pcf locks.
*/
p = pcf;
for (i = 0; i < pcf_fanout; i++) {
p->pcf_wait++;
mutex_exit(&p->pcf_lock);
p++;
}
freemem_wait++;
freemem_wait--;
if (lock) {
}
return (0);
}
/*
* The pcf accounting has been done,
* though none of the pcf_wait flags have been set,
* drop the locks and continue on.
*/
while (p >= pcf) {
mutex_exit(&p->pcf_lock);
p--;
}
}
/*
* page_list_sub will handle the case where pp is a large page.
* It's possible that the page was promoted while on the freelist
*/
"page_reclaim_free:pp %p", pp);
} else {
"page_reclaim_cache:pp %p", pp);
}
/*
* clear the p_free & p_age bits since this page is no longer
* on the free list. Notice that there was a brief time where
* a page is marked as free, but is not on the list.
*
* Set the reference bit to protect against immediate pageout.
*/
PP_CLRFREE(pp);
PP_CLRAGED(pp);
return (1);
}
/*
* Destroy identity of the page and put it back on
* the page free list. Assumes that the caller has
* acquired the "exclusive" lock on the page.
*/
void
{
panic("page_destroy: anon or kernel or no vnode "
"large page %p", (void *)pp);
}
}
/*
* Unload translations, if any, then hash out the
* page to erase its identity.
*/
if (!dontfree) {
/*
* Acquire the "freemem_lock" for availrmem.
* The page_struct_lock need not be acquired for lckcnt
* and cowcnt since the page has an "exclusive" lock.
* We are doing a modified version of page_pp_unlock here.
*/
availrmem++;
pages_locked--;
}
}
}
/*
* Put the page on the "free" list.
*/
}
}
void
{
/*NOTREACHED*/
}
pglcks++;
}
}
if (pglcks != 0) {
}
}
/*
* Similar to page_destroy(), but destroys pages which are
* locked and known to be on the page free list. Since
* the page is known to be free and locked, no one can access
* it.
*
* Also, the number of free pages does not change.
*/
void
{
PP_SETAGED(pp);
if (freemem_wait) {
}
}
/*
* Rename the page "opp" to have an identity specified
* by [vp, off]. If a page already exists with this name
* it is locked and destroyed. Note that the page's
* translations are not unloaded during the rename.
*
* This routine is used by the anon layer to "steal" the
* original page and is not unlike destroying a page and
* creating a new page using the same page frame.
*
* XXX -- Could deadlock if caller 1 tries to rename A to B while
* caller 2 tries to rename B to A.
*/
void
{
int olckcnt = 0;
int ocowcnt = 0;
/*
* CacheFS may call page_rename for a large NFS page
* when both CacheFS and NFS mount points are used
* by applications. Demote this large page before
* renaming it, to ensure that there are no "partial"
* large pages left lying around.
*/
}
/*
* Acquire the appropriate page hash lock, since
* we're going to rename the page.
*/
top:
/*
* Look for an existing page with this name and destroy it if found.
* By holding the page hash lock all the way to the page_hashin()
* call, we are assured that no page can be created with this
* identity. In the case when the phm lock is dropped to undo any
* hat layer mappings, the existing page is held with an "exclusive"
* lock, again preventing another page from being created with
* this identity.
*/
/*
* As it turns out, this is one of only two places where
* page_lock() needs to hold the passed in lock in the
* successful case. In all of the others, the lock could
* be dropped as soon as the attempt is made to lock
* the page. It is tempting to add yet another arguement,
* PL_KEEP or PL_DROP, to let page_lock know what to do.
*/
/*
* Went to sleep because the page could not
* be locked. We were woken up when the page
* was unlocked, or when the page was destroyed.
* In either case, `phm' was dropped while we
* slept. Hence we should not just roar through
* this loop.
*/
goto top;
}
/*
* If an existing page is a large page, then demote
* it to ensure that no "partial" large pages are
* "created" after page_rename. An existing page
* can be a CacheFS page, and can't belong to swapfs.
*/
if (hat_page_is_mapped(pp)) {
/*
* Unload translations. Since we hold the
* exclusive lock on this page, the page
* can not be changed while we drop phm.
* This is also not a lock protocol violation,
* but rather the proper way to do things.
*/
}
}
}
/*
* Hash in the page with the new identity.
*/
/*
* We were holding phm while we searched for [vp, off]
* and only dropped phm if we found and locked a page.
* If we can't create this page now, then some thing
* is really broken.
*/
/*NOTREACHED*/
}
/*
* Now that we have dropped phm, lets get around to finishing up
* with pp.
*/
/* for now large pages should not end up here */
/*
* Save the locks for transfer to the new page and then
* clear them so page_free doesn't think they're important.
* The page_struct_lock need not be acquired for lckcnt and
* cowcnt since the page has an "exclusive" lock.
*/
/*
* Put the page on the "free" list after we drop
* the lock. The less work under the lock the better.
*/
/*LINTED: constant in conditional context*/
}
/*
* Transfer the lock count from the old page (if any).
* The page_struct_lock need not be acquired for lckcnt and
* cowcnt since the page has an "exclusive" lock.
*/
}
/*
* low level routine to add page `pp' to the hash and vp chains for [vp, offset]
*
* Pages are normally inserted at the start of a vnode's v_pages list.
* If the vnode is VMODSORT and the page is modified, it goes at the end.
* This can happen when a modified page is relocated for DR.
*
* Returns 1 on success and 0 on failure.
*/
static int
{
/*
* Be sure to set these up before the page is inserted on the hash
* list. As soon as the page is placed on the list some other
* thread might get confused and wonder how this page could
* possibly hash to this list.
*/
/*
* record if this page is on a swap vnode
*/
PP_SETSWAP(pp);
/*
* If this page is already hashed in, fail this attempt to add it.
*/
return (0);
}
}
/*
* Add the page to the vnode's list of pages
*/
else
return (1);
}
/*
* Add page `pp' to both the hash and vp chains for [vp, offset].
*
* Returns 1 on success and 0 on failure.
* If hold is passed in, it is not dropped.
*/
int
{
int rc;
"page_hashin:pp %p vp %p offset %llx",
else {
}
if (rc == 0)
return (rc);
}
/*
* Remove page ``pp'' from the hash and vp chains and remove vp association.
* All mutexes must be held
*/
static void
{
/*
* First, take pp off of its hash chain.
*/
for (;;) {
break;
panic("page_do_hashout");
/*NOTREACHED*/
}
}
/*
* Now remove it from its associated vnode.
*/
PP_CLRSWAP(pp);
}
/*
* Remove page ``pp'' from the hash and vp chains and remove vp association.
*
* When `phm' is non-NULL it contains the address of the mutex protecting the
* hash list pp is on. It is not dropped.
*/
void
{
/* Kernel probe */
/*
*
*/
}
/*
* grab page vnode mutex and remove it...
*/
/*
* Wake up processes waiting for this page. The page's
* identity has been changed, and is probably not the
* desired page any longer.
*/
}
/*
* Add the page to the front of a linked list of pages
* using the p_next & p_prev pointers for the list.
* The caller is responsible for protecting the list pointers.
*/
void
{
}
/*
* Common code for page_add() and mach_page_add()
*/
void
{
} else {
}
}
/*
* Remove this page from a linked list of pages
* using the p_next & p_prev pointers for the list.
*
* The caller is responsible for protecting the list pointers.
*/
void
{
panic("page_sub: bad arg(s): pp %p, *ppp %p",
/*NOTREACHED*/
}
}
/*
* Common code for page_sub() and mach_page_sub()
*/
void
{
else {
}
}
/*
* Break page list cppp into two lists with npages in the first list.
* The tail is returned in nppp.
*/
void
{
long n = 0;
return;
}
if (npages == 0) {
return;
}
}
/* Fix head and tail of new lists */
/* second list empty */
} else {
}
}
/*
* Concatenate page list nppp onto the end of list ppp.
*/
void
{
return;
}
return;
}
}
/*
* return the next page in the page list
*/
page_t *
{
}
/*
* Add the page to the front of the linked list of pages
*
* The caller is responsible for protecting the lists.
*/
void
{
} else {
}
}
/*
* Remove this page from the linked list of pages
*
* The caller is responsible for protecting the lists.
*/
void
{
panic("page_vpsub: bad arg(s): pp %p, *ppp %p",
/*NOTREACHED*/
}
else {
}
}
/*
* Lock a physical page into memory "long term". Used to support "lock
* in memory" functions. Accepts the page to be locked, and a cow variable
* to indicate whether a the lock will travel to the new page during
* a potential copy-on-write.
*/
int
int cow, /* cow lock */
int kernel) /* must succeed -- ignore checking */
{
int r = 0; /* result -- assume failure */
/*
* Acquire the "freemem_lock" for availrmem.
*/
if (cow) {
if ((availrmem > pages_pp_maximum) &&
availrmem--;
pages_locked++;
r = 1;
"COW lock limit reached on pfn 0x%lx",
page_pptonum(pp));
}
} else
} else {
r = 1;
"reached on pfn 0x%lx",
page_pptonum(pp));
}
}
} else {
if (kernel) {
/* availrmem accounting done by caller */
r = 1;
} else {
if (availrmem > pages_pp_maximum) {
availrmem--;
pages_locked++;
r = 1;
}
}
}
}
return (r);
}
/*
* Decommit a lock on a physical page frame. Account for cow locks if
* appropriate.
*/
void
int cow, /* expect cow lock */
int kernel) /* this was a kernel lock */
{
/*
* Acquire the "freemem_lock" for availrmem.
* If cowcnt or lcknt is already 0 do nothing; i.e., we
* could be called to unlock even if nothing is locked. This could
* happen if locked file pages were truncated (removing the lock)
* and the file was grown again and new pages faulted in; the new
* pages are unlocked but the segment still thinks they're locked.
*/
if (cow) {
availrmem++;
pages_locked--;
}
} else {
if (!kernel) {
availrmem++;
pages_locked--;
}
}
}
}
/*
* This routine reserves availrmem for npages;
* flags: KM_NOSLEEP or KM_SLEEP
* returns 1 on success or 0 on failure
*/
int
{
if (flags & KM_NOSLEEP) {
return (0);
}
kmem_reap();
}
return (1);
}
/*
* This routine unreserves availrmem for npages;
*/
void
{
}
/*
* See Statement at the beginning of segvn_lockop() regarding
* the way we handle cowcnts and lckcnts.
*
* Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage
* that breaks COW has PROT_WRITE.
*
* Note that, we may also break COW in case we are softlocking
* on read access during physio;
* in this softlock case, the vpage may not have PROT_WRITE.
* So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp'
* if the vpage doesn't have PROT_WRITE.
*
* This routine is never called if we are stealing a page
* in anon_private.
*
* The caller subtracted from availrmem for read only mapping.
* if lckcnt is 1 increment availrmem.
*/
void
{
int payback = 0;
/* Don't use claim if nothing is locked (see page_pp_unlock above) */
if (write_perm) {
} else {
/*
* We didn't need availrmem decremented if p_lckcnt on
* original page is 1. Here, we are unlocking
* read-only copy belonging to original page and
* are locking a copy belonging to new page.
*/
payback = 1;
}
}
if (payback) {
availrmem++;
}
}
/*
* Simple claim adjust functions -- used to support changes in
* claims due to changes in access permissions. Used by segvn_setprot().
*/
int
{
int r = 0; /* result */
r = 1;
"COW lock limit reached on pfn 0x%lx",
page_pptonum(pp));
}
}
} else {
if ((availrmem > pages_pp_maximum) &&
--availrmem;
r = 1;
"COW lock limit reached on pfn 0x%lx",
page_pptonum(pp));
}
} else
}
return (r);
}
int
{
int r = 0;
r = 1;
/*
* for availrmem
*/
availrmem++;
"Page lock limit reached on pfn 0x%lx",
page_pptonum(pp));
}
}
} else {
r = 1;
}
return (r);
}
int
{
return (0);
}
lckpgs++;
}
if (lckpgs != 0) {
pages_claimed += lckpgs;
} else {
return (0);
}
}
}
return (1);
}
int
{
return (0);
}
ulckpgs++;
}
if (ulckpgs != 0) {
pages_claimed -= ulckpgs;
}
}
return (1);
}
page_t *
{
}
/*
* Acquire the appropriate lock on the page.
*/
goto retry;
continue;
}
goto retry;
}
return (pp);
}
page_t *
{
}
/*
* Acquire the appropriate lock on the page.
*/
goto retry;
continue;
}
goto retry;
}
return (pp);
}
/*
* This routine is like page_numtopp, but will only return page structs
* for pages which are ok for loading into hardware using the page struct.
*/
page_t *
{
}
/*
* Try to acquire the appropriate lock on the page.
*/
else {
else {
goto retry;
}
}
}
}
return (pp);
}
#define SYNC_PROGRESS_NPAGES 1000
/*
* Returns a count of dirty pages that are in the process
* of being written out. If 'cleanit' is set, try to push the page.
*/
{
int counter = 0;
do {
/*
* Reset the sync timeout. The page list is very long
* on large memory systems.
*/
if (++counter > SYNC_PROGRESS_NPAGES) {
counter = 0;
}
/*
* A page is a candidate for syncing if it is:
*
* (a) On neither the freelist nor the cachelist
* (b) Hashed onto a vnode
* (c) Not a kernel page
* (d) Dirty
* (e) Not part of a swapfile
* (f) a page which belongs to a real vnode; eg has a non-null
* v_vfsp pointer.
* (g) Backed by a filesystem which doesn't have a
* stubbed-out sync operation
*/
nppbusy++;
if (!cleanit)
continue;
continue;
!(hat_pagesync(pp,
continue;
}
}
return (nppbusy);
}
void page_invalidate_pages(void);
/*
* callback handler to vm sub-system
*
* callers make sure no recursive entries to this func.
*/
/*ARGSUSED*/
{
if (code == CB_CODE_CPR_CHKPT)
return (B_TRUE);
}
/*
* Invalidate all pages of the system.
* It shouldn't be called until all user page activities are all stopped.
*/
void
{
int retry = 0;
const int MAXRETRIES = 4;
top:
/*
* Flush dirty pages and destroy the clean ones.
*/
nbusypages = 0;
do {
int mod;
/*
* skip the page if it has no vnode or the page associated
* with the kernel vnode or prom allocated kernel mem.
*/
continue;
/*
* skip the page which is already free invalidated.
*/
continue;
/*
* skip pages that are already locked or can't be "exclusively"
* locked or are already free. After we lock the page, check
* the free and age bits again to be sure it's not destroyed
* yet.
* To achieve max. parallelization, we use page_trylock instead
* of page_lock so that we don't get block on individual pages
* while we have thousands of other pages to process.
*/
nbusypages++;
continue;
} else {
}
continue;
}
/*
* Is this page involved in some I/O? shared?
*
* The page_struct_lock need not be acquired to
* examine these fields since the page has an
* "exclusive" lock.
*/
continue;
}
panic("vp->v_type == VCHR");
/*NOTREACHED*/
}
if (!page_try_demote_pages(pp)) {
continue;
}
/*
* Check the modified bit. Leave the bits alone in hardware
* (they will be modified if we do the putpage).
*/
& P_MOD);
if (mod) {
/*
* Hold the vnode before releasing the page lock
* to prevent it from being freed and re-used by
* some other thread.
*/
/*
* No error return is checked here. Callers such as
* cpr deals with the dirty pages at the dump time
* if this putpage fails.
*/
} else {
/*LINTED: constant in conditional context*/
}
delay(1);
goto top;
}
}
/*
* Replace the page "old" with the page "new" on the page hash and vnode lists
*
* the replacement must be done in place, ie the equivalent sequence:
*
* vp = old->p_vnode;
* off = old->p_offset;
* page_do_hashout(old)
* page_do_hashin(new, vp, off)
*
* doesn't work, since
* 1) if old is the only page on the vnode, the v_pages list has a window
* where it looks empty. This will break file system assumptions.
* and
* 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list.
*/
static void
{
/*
* First find old page on the page hash list
*/
for (;;) {
break;
panic("page_do_hashout");
/*NOTREACHED*/
}
}
/*
* update new and replace old with new on the page hash list
*/
/*
* replace old with new on the vnode's page list
*/
} else {
}
/*
* clear out the old page
*/
/*
* Wake up processes waiting for this page. The page's
* identity has been changed, and is probably not the
* desired page any longer.
*/
}
/*
* This function moves the identity of page "pp_old" to page "pp_new".
* Both pages must be locked on entry. "pp_new" is free, has no identity,
* and need not be hashed out from anywhere.
*/
void
{
/*
* Rehash two pages
*/
/*
* hashout then hashin while holding the mutexes
*/
/* The following comment preserved from page_flip(). */
/*
* The page_struct_lock need not be acquired for lckcnt and
* cowcnt since the page has an "exclusive" lock.
*/
}
/*
* Helper routine used to lock all remaining members of a
* large page. The caller is responsible for passing in a locked
* pp. If pp is a large page, then it succeeds in locking all the
* remaining constituent pages or it returns with only the
* original page locked.
*
* Returns 1 on success, 0 on failure.
*
* If success is returned this routine guarantees p_szc for all constituent
* pages of a large page pp belongs to can't change. To achieve this we
* recheck szc of pp after locking all constituent pages and retry if szc
* changed (it could only decrease). Since hat_page_demote() needs an EXCL
* lock on one of constituent pages it can't be running after all constituent
* pages are locked. hat_page_demote() with a lock on a constituent page
* outside of this large page (i.e. pp belonged to a larger large page) is
* already done with all constituent pages of pp since the root's p_szc is
* changed last. Therefore no need to synchronize with hat_page_demote() that
* locked a constituent page outside of pp's current large page.
*/
#ifdef DEBUG
uint32_t gpg_trylock_mtbf = 0;
#endif
int
{
#ifdef DEBUG
return (0);
}
#endif
return (0);
}
if (pszc == 0) {
return (1);
}
for (j = 1; j < i; j++, tpp++) {
}
return (0);
}
}
}
goto retry;
}
return (1);
}
void
{
}
}
/*
* returns
* 0 : on success and *nrelocp is number of relocated PAGESIZE pages
* ERANGE : this is not a base page
* ENOMEM : failure to obtain replacement pages
* EAGAIN : OBP has not yet completed its boot-time handoff to the kernel
* EIO : An error occurred while trying to copy the page data
*
* Return with all constituent members of target and replacement
* SE_EXCL locked. It is the callers responsibility to drop the
* locks.
*/
int
int grouplock,
{
int repl_contig = 0;
*nrelocp = 0;
#if defined(__sparc)
/*
* We need to wait till OBP has completed
* its boot-time handoff of its resources to the kernel
* before we allow page relocation
*/
if (page_relocate_ready == 0) {
return (EAGAIN);
}
#endif
/*
* If this is not a base page,
* just return with 0x0 pages relocated.
*/
return (ERANGE);
}
return (ERANGE);
}
repl_contig = 1;
}
/*
* We must lock all members of this large page or we cannot
* relocate any part of it.
*/
return (EBUSY);
}
/*
* reread szc it could have been decreased before
* group_page_trylock() was done.
*/
if (!page_create_wait(dofree, 0)) {
if (grouplock != 0) {
}
return (ENOMEM);
}
/*
* seg kmem pages require that the target and replacement
* page be the same pagesize.
*/
if (grouplock != 0) {
}
return (ENOMEM);
}
}
#ifdef DEBUG
else {
}
#endif /* DEBUG */
#if defined(__sparc)
/*
* Let hat_page_relocate() complete the relocation if it's kernel page
*/
*replacement = repl;
if (grouplock != 0) {
}
if (dofree) {
*replacement = NULL;
}
return (EAGAIN);
}
return (0);
}
#else
#if defined(lint)
#endif
#endif
first_repl = repl;
for (i = 0; i < npgs; i++) {
ASSERT(repl_contig == 0 ||
/*
* Copy the page contents and attributes then
* relocate the page in the page hash.
*/
repl = first_repl;
if (grouplock != 0) {
}
if (dofree) {
*replacement = NULL;
}
return (EIO);
}
targ++;
if (repl_contig != 0) {
repl++;
} else {
}
}
repl = first_repl;
for (i = 0; i < npgs; i++) {
/*
* Now clear the props on targ, after the
* page_relocate_hash(), they no longer
* have any meaning.
*/
targ++;
if (repl_contig != 0) {
repl++;
} else {
}
}
/* assert that we have come full circle with repl */
if (*replacement == NULL) {
*replacement = repl;
}
return (0);
}
/*
* On success returns 0 and *nrelocp the number of PAGESIZE pages relocated.
*/
int
int grouplock,
int freetarget,
{
/* do_page_relocate returns 0 on success or errno value */
if (ret != 0 || freetarget == 0) {
return (ret);
}
if (*nrelocp == 1) {
} else {
do {
npgs--;
page_list_add_pages(*target, 0);
}
return (ret);
}
/*
* it is up to the caller to deal with pcf accounting.
*/
void
{
/*
* pp_targ is a linked list.
*/
PP_SETFREE(pp);
PP_SETAGED(pp);
} else {
do {
page_list_add_pages(pp, 0);
}
}
}
/*
* Relocate target to non-relocatable replacement page.
*/
int
{
int result;
do {
}
if (result == 0) {
*replacement = rpp;
panic("page_relocate_cage: partial relocation");
}
return (result);
}
/*
* Release the page lock on a page, place on cachelist
* tail if no longer mapped. Caller can let us know if
* the page is known to be clean.
*/
int
{
int status;
!hat_page_is_mapped(pp)) {
/*
* If page is modified, unlock it
*
* (p_nrm & P_MOD) bit has the latest stuff because:
* (1) We found that this page doesn't have any mappings
* _after_ holding SE_EXCL and
* (2) We didn't drop SE_EXCL lock after the check in (1)
*/
} else {
/*LINTED: constant in conditional context*/
}
} else {
}
return (status);
}
/*
* Given a constituent page, try to demote the large page on the freelist.
*
* Returns nonzero if the page could be demoted successfully. Returns with
* the constituent page still locked.
*/
int
{
/*
* Adjust rootpp and lock it, if `pp' is not the base
* constituent page.
*/
if (npgs == 1) {
return (0);
}
}
return (0);
}
return (0);
}
return (1);
}
/*
* Given a constituent page, try to demote the large page.
*
* Returns nonzero if the page could be demoted successfully. Returns with
* the constituent page still locked.
*/
int
{
return (1);
}
return (1);
}
/*
* Adjust rootpp if passed in is not the base
* constituent page.
*/
}
/*
* We can't demote kernel pages since we can't hat_unload()
* the mappings.
*/
return (0);
/*
* Attempt to lock all constituent pages except the page passed
* in since it's already locked.
*/
break;
}
/*
* If we failed to lock them all then unlock what we have
* locked so far and bail.
*/
if (i < npgs) {
while (i-- > 0) {
tpp++;
}
return (0);
}
}
/*
* Unlock all pages except the page passed in.
*/
}
return (1);
}
/*
* Called by page_free() and page_destroy() to demote the page size code
* (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero
* p_szc on free list, neither can we just clear p_szc of a single page_t
* within a large page since it will break other code that relies on p_szc
* being the same for all page_t's of a large page). Anonymous pages should
* never end up here because anon_map_getpages() cannot deal with p_szc
* changes after a single constituent page is locked. While anonymous or
* kernel large pages are demoted or freed the entire large page at a time
* with all constituent pages locked EXCL for the file system pages we
* have to be able to demote a large page (i.e. decrease all constituent pages
* p_szc) with only just an EXCL lock on one of constituent pages. The reason
* we can easily deal with anonymous page demotion the entire large page at a
* time is that those operation originate at address space level and concern
* the entire large page region with actual demotion only done when pages are
* not shared with any other processes (therefore we can always get EXCL lock
* on all anonymous constituent pages after clearing segment page
* cache). However file system pages can be truncated or invalidated at a
* PAGESIZE level from the file system side and end up in page_free() or
* page_destroy() (we also allow only part of the large page to be SOFTLOCKed
* and therefore pageout should be able to demote a large page by EXCL locking
* any constituent page that is not under SOFTLOCK). In those cases we cannot
* rely on being able to lock EXCL all constituent pages.
*
* To prevent szc changes on file system pages one has to lock all constituent
* pages at least SHARED (or call page_szc_lock()). The only subsystem that
* doesn't rely on locking all constituent pages (or using page_szc_lock()) to
* prevent szc changes is hat layer that uses its own page level mlist
* locks. hat assumes that szc doesn't change after mlist lock for a page is
* taken. Therefore we need to change szc under hat level locks if we only
* have an EXCL lock on a single constituent page and hat still references any
* of constituent pages. (Note we can't "ignore" hat layer by simply
* hat_pageunload() all constituent pages without having EXCL locks on all of
* constituent pages). We use hat_page_demote() call to safely demote szc of
* all constituent pages under hat locks when we only have an EXCL lock on one
* of constituent pages.
*
* This routine calls page_szc_lock() before calling hat_page_demote() to
* allow segvn in one special case not to lock all constituent pages SHARED
* before calling hat_memload_array() that relies on p_szc not changing even
* before hat level mlist lock is taken. In that case segvn uses
* page_szc_lock() to prevent hat_page_demote() changing p_szc values.
*
* Anonymous or kernel page demotion still has to lock all pages exclusively
* and do hat_pageunload() on all constituent pages before demoting the page
* therefore there's no need for anonymous or kernel page demotion to use
* hat_page_demote() mechanism.
*
* hat_page_demote() removes all large mappings that map pp and then decreases
* p_szc starting from the last constituent page of the large page. By working
* from the tail of a large page in pfn decreasing order allows one looking at
* the root page to know that hat_page_demote() is done for root's szc area.
* e.g. if a root page has szc 1 one knows it only has to lock all constituent
* pages within szc 1 area to prevent szc changes because hat_page_demote()
* that started on this page when it had szc > 1 is done for this szc 1 area.
*
* We are guaranteed that all constituent pages of pp's large page belong to
* the same vnode with the consecutive offsets increasing in the direction of
* the pfn i.e. the identity of constituent pages can't change until their
* p_szc is decreased. Therefore it's safe for hat_page_demote() to remove
* large mappings to pp even though we don't lock any constituent page except
* pp (i.e. we won't unload e.g. kernel locked page).
*/
static void
{
}
}
/*
* Mark any existing pages for migration in the given range
*/
void
{
pgcnt_t i;
/*
* Don't do anything if don't need to do lgroup optimizations
* on this system
*/
if (!lgrp_optimizations())
return;
/*
* Align address and length to (potentially large) page boundary
*/
if (rflag)
/*
* Allocate page array to accommodate largest page size
*/
/*
* Do one (large) page at a time
*/
/*
* Lookup (root) page for vnode and offset corresponding to
* this virtual address
* Try anonmap first since there may be copy-on-write
* pages, but initialize vnode pointer and offset using
* vnode arguments just in case there isn't an amp.
*/
if (amp) {
if (ap)
}
if (curvp)
/*
* If there isn't a page at this virtual address,
* skip to next page
*/
continue;
}
/*
* Figure out which lgroup this page is in for kstats
*/
/*
* Get page size, and round up and skip to next page boundary
* if unaligned address
*/
continue;
}
/*
* Upgrade to exclusive lock on page
*/
if (!page_tryupgrade(pp)) {
continue;
}
/*
* Remember pages locked exclusively and how many
*/
nlocked = 1;
/*
* Lock constituent pages if this is large page
*/
if (pages > 1) {
/*
* Lock all constituents except root page, since it
* should be locked already.
*/
for (i = 1; i < pages; i++) {
pp++;
break;
}
/*
* hat_page_demote() raced in with us.
*/
break;
}
nlocked++;
}
}
/*
* If all constituent pages couldn't be locked,
* unlock pages locked so far and skip to next page.
*/
for (i = 0; i < nlocked; i++)
page_unlock(ppa[i]);
continue;
}
/*
* hat_page_demote() can no longer happen
* since last cons page had the right p_szc after
* all cons pages were locked. all cons pages
* should now have the same p_szc.
*/
/*
* All constituent pages locked successfully, so mark
* large page for migration and unload the mappings of
* constituent pages, so a fault will occur on any part of the
* large page
*/
PP_SETMIGRATE(ppa[0]);
for (i = 0; i < nlocked; i++) {
}
}
}
/*
* Migrate any pages that have been marked for migration in the given range
*/
void
{
spgcnt_t i;
while (npages > 0) {
/*
* Check to see whether this page is marked for migration
*
* Assume that root page of large page is marked for
* migration and none of the other constituent pages
* are marked. This really simplifies clearing the
* migrate bit by not having to clear it from each
* constituent page.
*
* note we don't want to relocate an entire large page if
* someone is only using one subpage.
*/
break;
/*
* Is it marked for migration?
*/
if (!PP_ISMIGRATE(pp))
goto next;
/*
* Determine lgroups that page is being migrated between
*/
break;
}
/*
* Need to get exclusive lock's to migrate
*/
for (i = 0; i < page_cnt; i++) {
break;
}
if (!page_tryupgrade(ppa[i])) {
page_cnt);
break;
}
/*
* Check to see whether we are trying to migrate
* page to lgroup where it is allocated already.
* If so, clear the migrate bit and skip to next
* page.
*/
PP_CLRMIGRATE(ppa[0]);
page_downgrade(ppa[0]);
goto next;
}
}
/*
* If all constituent pages couldn't be locked,
* unlock pages locked so far and skip to next page.
*/
if (i != page_cnt) {
while (--i != -1) {
page_downgrade(ppa[i]);
}
goto next;
}
for (i = 0; i < page_cnt; i++) {
page_downgrade(ppa[i]);
}
page_cnt);
goto next;
}
/*
* Clear migrate bit and relocate page
*/
panic("page_migrate: page_relocate failed");
}
/*
* Keep stats for number of pages migrated from and to
* each lgroup
*/
/*
* update the page_t array we were passed in and
* unlink constituent pages of a large page.
*/
}
next:
}
}
ulong_t mem_waiters = 0;
#define MAX_DELAY 0x1ff
/*
* Check if enough memory is available to proceed.
* Depending on system configuration and how much memory is
* reserved for swap we need to check against two variables.
* e.g. on systems with little physical swap availrmem can be
* more reliable indicator of how much memory is available.
* On systems with large phys swap freemem can be better indicator.
* If freemem drops below threshold level don't return an error
* immediately but wake up pageout to free memory and block.
* This is done number of times. If pageout is not able to free
* memory within certain time return an error.
* The same applies for availrmem but kmem_reap is used to
* free memory.
*/
int
{
#if defined(__i386)
return (1);
#else
return (1);
#endif
return (0);
}
}
if (count == 0) {
return (0);
}
kmem_reap();
return (0);
}
}
if (count == 0)
return (0);
#if defined(__i386)
return (0);
#endif
return (1);
}
/*
* If there is not enough memory start reaping seg, kmem caches.
* Start pageout scanner (via page_needfree()).
* Exit after ~ MAX_CNT s regardless of how much memory has been released.
* Note: There is no guarantee that any availrmem will be freed as
* this memory typically is locked (kernel heap) or reserved for swap.
* Also due to memory fragmentation kmem allocator may not be able
* to free any memory (single user allocated buffer will prevent
* freeing slab or a page).
*/
int
{
int i = 0;
int ret = 0;
kmem_reap();
}
ret = 1;
}
return (ret);
}
/*
* Search the memory segments to locate the desired page. Within a
* segment, pages increase linearly with one page structure per
* physical page frame (size PAGESIZE). The search begins
* with the segment that was accessed last, to take advantage of locality.
* If the hint misses, we start from the beginning of the sorted memseg list
*/
/*
* Some data structures for pfn to pp lookup.
*/
page_t *
{
/*
* We need to disable kernel preemption while referencing the
* cpu_vm_data field in order to prevent us from being switched to
* another cpu and trying to reference it after it has been freed.
* This will keep us on cpu and prevent it from being removed while
* we are still on it.
*
* We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
* which is being resued by DR who will flush those references
* before modifying the reused memseg. See memseg_cpu_vm_flush().
*/
/* Try last winner first */
}
}
/* Else Try hash */
}
}
/* Else Brute force */
}
}
}
}
struct memseg *
{
/*
* We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
* which is being resued by DR who will flush those references
* before modifying the reused memseg. See memseg_cpu_vm_flush().
*/
/* Try hash */
return (seg);
}
}
/* Else Brute force */
return (seg);
}
}
}
}
/*
* Given a page and a count return the page struct that is
* n structs away from the current one in the global page
* list.
*
* This function wraps to the first page upon
* reaching the end of the memseg list.
*/
page_t *
{
/*
* We need to disable kernel preemption while referencing the
* cpu_vm_data field in order to prevent us from being switched to
* another cpu and trying to reference it after it has been freed.
* This will keep us on cpu and prevent it from being removed while
* we are still on it.
*
* We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
* which is being resued by DR who will flush those references
* before modifying the reused memseg. See memseg_cpu_vm_flush().
*/
break;
}
/* Memory delete got in, return something valid. */
/* TODO: fix me. */
}
}
/* check for wraparound - possible if n is large */
}
return (ppn);
}
/*
* Initialize for a loop using page_next_scan_large().
*/
page_t *
page_next_scan_init(void **cookie)
{
}
/*
* Return the next page in a scan of page_t's, assuming we want
* to skip over sub-pages within larger page sizes.
*
* The cookie is used to keep track of the current memseg.
*/
page_t *
ulong_t *n,
void **cookie)
{
/*
* get the count of page_t's to skip based on the page size
*/
cnt = 1;
} else {
}
*n += cnt;
/*
* Catch if we went past the end of the current memory segment. If so,
* just move to the next segment with pages.
*/
do {
}
return (new_pp);
}
/*
* Returns next page in list. Note: this function wraps
* to the first page in the list upon reaching the end
* of the list. Callers should be aware of this fact.
*/
/* We should change this be a #define */
page_t *
{
}
page_t *
{
}
/*
* This routine is called at boot with the initial memory configuration
* and when memory is added or removed.
*/
void
{
int i;
/*
* Clear memseg_hash array.
* with normal operation, the hash rebuild must be able to run
* concurrently with page_numtopp_nolock(). To support this
* functionality, assignments to memseg_hash array members must
* be done atomically.
*
* NOTE: bzero() does not currently guarantee this for kernel
* threads, and cannot be used here.
*/
for (i = 0; i < N_MEM_SLOTS; i++)
memseg_hash[i] = NULL;
/*
* Physmax is the last valid pfn.
*/
do {
if (index >= N_MEM_SLOTS)
}
cur += mhash_per_slot;
index++;
}
}
/*
* Return the pagenum for the pp
*/
{
}
/*
* interface to the referenced and modified etc bits
* in the PSM part of the page struct
* when no locking is desired.
*/
void
{
}
void
{
}
/*
* Clear p_lckcnt and p_cowcnt, adjusting freemem if required.
*/
int
{
int f_amount;
/*
* The page_struct_lock need not be acquired here since
* we require the caller hold the page exclusively locked.
*/
f_amount = 0;
f_amount = 1;
}
}
}
return (f_amount);
}
/*
* The following functions is called from free_vp_pages()
* for an inexact estimate of a newly free'd page...
*/
{
return (hat_page_getshare(pp));
}
int
{
}
int
{
}
int
{
}
int
{
}
/*
* The following code all currently relates to the page capture logic:
*
* This logic is used for cases where there is a desire to claim a certain
* physical page in the system for the caller. As it may not be possible
* to capture the page immediately, the p_toxic bits are used in the page
* structure to indicate that someone wants to capture this page. When the
* page gets unlocked, the toxic flag will be noted and an attempt to capture
* the page will be made. If it is successful, the original callers callback
* will be called with the page to do with it what they please.
*
* There is also an async thread which wakes up to attempt to capture
* pages occasionally which have the capture bit set. All of the pages which
* need to be captured asynchronously have been inserted into the
* page_capture_hash and thus this thread walks that hash list. Items in the
* hash have an expiration time so this thread handles that as well by removing
* the item from the hash if it has expired.
*
* Some important things to note are:
* - if the PR_CAPTURE bit is set on a page, then the page is in the
* page_capture_hash. The page_capture_hash_head.pchh_mutex is needed
* to set and clear this bit, and while the lock is held is the only time
* you can add or remove an entry from the hash.
* - the PR_CAPTURE bit can only be set and cleared while holding the
* page_capture_hash_head.pchh_mutex
* - the t_flag field of the thread struct is used with the T_CAPTURING
* flag to prevent recursion while dealing with large pages.
* - pages which need to be retired never expire on the page_capture_hash.
*/
static void page_capture_thread(void);
static kthread_t *pc_thread_id;
static kmutex_t pc_thread_mutex;
static clock_t pc_thread_shortwait;
static clock_t pc_thread_longwait;
static int pc_thread_retry;
/* Note that this is a circular linked list */
typedef struct page_capture_hash_bucket {
void *datap; /* Cached data passed in for callback */
struct page_capture_hash_bucket *next;
struct page_capture_hash_bucket *prev;
/*
* Each hash bucket will have it's own mutex and two lists which are:
* active (0): represents requests which have not been processed by
* the page_capture async thread yet.
* walked (1): represents requests which have been processed by the
* page_capture async thread within it's given walk of this bucket.
*
* These are all needed so that we can synchronize all async page_capture
* events. When the async thread moves to a new bucket, it will append the
* walked list to the active list and walk each item one at a time, moving it
* from the active list to the walked list. Thus if there is an async request
* outstanding for a given page, it will always be in one of the two lists.
* New requests will always be added to the active list.
* If we were not able to capture a page before the request expired, we'd free
* up the request structure which would indicate to page_capture that there is
* no longer a need for the given page, and clear the PR_CAPTURE flag if
* possible.
*/
typedef struct page_capture_hash_head {
#ifdef DEBUG
#define NUM_PAGE_CAPTURE_BUCKETS 4
#else
#define NUM_PAGE_CAPTURE_BUCKETS 64
#endif
/* for now use a very simple hash based upon the size of a page struct */
#define PAGE_CAPTURE_HASH(pp) \
extern pgcnt_t swapfs_minfree;
/*
* a callback function is required for page capture requests.
*/
void
{
}
void
{
int i, j;
struct page_capture_hash_bucket *bp1;
struct page_capture_hash_bucket *bp2;
/*
* Just move all the entries to a private list which we can walk
* through without the need to hold any locks.
* No more requests can get added to the hash lists for this consumer
* as the cb_active field for the callback has been cleared.
*/
for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
for (j = 0; j < 2; j++) {
/* walk through all but first (sentinel) element */
/*
* Clear the PR_CAPTURE bit as we
* hold appropriate locks here.
*/
page_capture_hash[i].num_pages--;
continue;
}
}
}
}
}
}
/*
* Find pp in the active list and move it to the walked list if it
* exists.
* Note that most often pp should be at the front of the active list
* as it is currently used and thus there is no other sort of optimization
* being done here as this is a linked list data structure.
* Returns 1 on successful move or 0 if page could not be found.
*/
static int
{
int index;
/* Remove from old list */
/* Add to new list */
return (1);
}
}
return (0);
}
/*
* Add a new entry to the page capture hash. The only case where a new
* entry is not added is when the page capture consumer is no longer registered.
* In this case, we'll silently not add the page to the hash. We know that
* page retire will always be registered for the case where we are currently
* unretiring a page and thus there are no conflicts.
*/
static void
{
int index;
int cb_index;
int i;
#ifdef DEBUG
int l;
#endif
break;
}
}
} else {
}
} else {
/* There's no callback registered so don't add to the hash */
return;
}
/*
* Only allow capture flag to be modified under this mutex.
* Prevents multiple entries for same page getting added.
*/
/*
* if not already on the hash, set capture bit and add to the hash
*/
#ifdef DEBUG
/* Check for duplicate entries */
for (l = 0; l < 2; l++) {
panic("page pp 0x%p already on hash "
"at 0x%p\n",
}
}
}
#endif
if (flags & CAPTURE_RETIRE) {
}
return;
}
/*
* A page retire request will replace any other request.
* A second physmem request which is for a different process than
* the currently registered one will be dropped as there is
* no way to hold the private data for both calls.
* In the future, once there are more callers, this will have to
* be worked out better as there needs to be private storage for
* at least each type of caller (maybe have datap be an array of
* *void's so that we can index based upon callers index).
*/
/* walk hash list to update expire time */
for (i = 0; i < 2; i++) {
if (flags & CAPTURE_RETIRE) {
datap);
}
} else {
}
}
return;
}
}
}
/*
* the PR_CAPTURE flag is protected by the page_capture_hash mutexes
* and thus it either has to be set or not set and can't change
* while holding the mutex above.
*/
panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n",
(void *)pp);
}
/*
* We have a page in our hands, lets try and make it ours by turning
* it into a clean page like it had just come off the freelists.
*
* Returns 0 on success, with the page still EXCL locked.
* On failure, the page will be unlocked, and returns EAGAIN
*/
static int
{
int skip_unlock = 0;
int ret = 0;
int extra;
skip_unlock = 1;
goto cleanup;
}
/*
* Since this page came from the
* cachelist, we must destroy the
* old vnode association.
*/
}
goto cleanup;
}
/*
* If we know page_relocate will fail, skip it
* It could still fail due to a UE on another page but we
* can't do anything about that.
*/
goto skip_relocate;
}
/*
* It's possible that pages can not have a vnode as fsflush comes
* through and cleans up these pages. It's ugly but that's how it is.
*/
goto skip_relocate;
}
/*
* Page was not free, so lets try to relocate it.
* page_relocate only works with root pages, so if this is not a root
* page, we need to demote it to try and relocate it.
* Unfortunately this is the best we can do right now.
*/
if (page_try_demote_pages(pp) == 0) {
goto cleanup;
}
}
if (ret == 0) {
/* unlock the new page(s) */
while (count-- > 0) {
}
/*
* Check to see if the page we have is too large.
* If so, demote it freeing up the extra pages.
*/
/* For now demote extra pages to szc == 0 */
while (extra > 0) {
extra--;
}
/* Make sure to set our page to szc 0 as well */
}
goto cleanup;
goto cleanup;
} else {
/*
* Need to reset return type as we failed to relocate the page
* but that does not mean that some of the next steps will not
* work.
*/
ret = 0;
}
if (page_try_demote_pages(pp) == 0) {
goto cleanup;
}
}
goto cleanup;
}
goto cleanup;
}
goto cleanup;
}
/*
* This is a semi-odd case as the page is now modified but not
* mapped as we just unloaded the mappings above.
*/
goto cleanup;
}
}
/*
* At this point, the page should be in a clean state and
* we can do whatever we want with it.
*/
if (ret != 0) {
if (!skip_unlock) {
}
} else {
}
return (ret);
}
/*
* Various callers of page_trycapture() can have different restrictions upon
* what memory they have access to.
* Returns 0 on success, with the following error codes on failure:
* EPERM - The requested page is long term locked, and thus repeated
* requests to capture this page will likely fail.
* ENOMEM - There was not enough free memory in the system to safely
* map the requested page.
* ENOENT - The requested page was inside the kernel cage, and the
* PHYSMEM_CAGE flag was not set.
*/
int
{
#if defined(__sparc)
return (EPERM);
}
(flags & CAPTURE_PHYSMEM)) {
return (ENOENT);
}
if (PP_ISNORELOCKERNEL(pp)) {
return (EPERM);
}
#else
return (EPERM);
}
#endif /* __sparc */
/* only physmem currently has the restrictions checked below */
if (!(flags & CAPTURE_PHYSMEM)) {
return (0);
}
if (availrmem < swapfs_minfree) {
/*
* We won't try to capture this page as we are
* running low on memory.
*/
return (ENOMEM);
}
return (0);
}
/*
* Once we have a page in our mits, go ahead and complete the capture
* operation.
* Returns 1 on failure where page is no longer needed
* Returns 0 on success
* Returns -1 if there was a transient failure.
* Failure cases must release the SE_EXCL lock on pp (usually via page_free).
*/
int
{
int cb_index;
int ret = 0;
int index;
int found = 0;
int i;
break;
}
}
/*
* Remove the entry from the page_capture hash, but don't free it yet
* as we may need to put it back.
* Since we own the page at this point in time, we should find it
* in the hash if this is an ASYNC call. If we don't it's likely
* that the page_capture_async() thread decided that this request
* had expired, in which case we just continue on.
*/
if (flags & CAPTURE_ASYNC) {
for (i = 0; i < 2 && !found; i++) {
found = 1;
break;
}
}
}
}
/* Synchronize with the unregister func. */
if (found) {
}
return (1);
}
/*
* We need to remove the entry from the page capture hash and turn off
* the PR_CAPTURE bit before calling the callback. We'll need to cache
* the entry here, and then based upon the return value, cleanup
* appropriately or re-add it to the hash, making sure that someone else
* hasn't already done so.
* It should be rare for the callback to fail and thus it's ok for
* the failure path to be a bit complicated as the success path is
* cleaner and the locking rules are easier to follow.
*/
/*
* If this was an ASYNC request, we need to cleanup the hash if the
* callback was successful or if the request was no longer valid.
* For non-ASYNC requests, we return failure to map and the caller
* will take care of adding the request to the hash.
* Note also that the callback itself is responsible for the page
* at this point in time in terms of locking ... The most common
* case for the failure path should just be a page_free.
*/
if (ret >= 0) {
if (found) {
}
}
return (ret);
}
if (!found) {
return (ret);
}
/*
* Check for expiration time first as we can just free it up if it's
* expired.
*/
return (ret);
}
/*
* The callback failed and there used to be an entry in the hash for
* this page, so we need to add it back to the hash.
*/
/* just add bp1 back to head of walked list */
return (ret);
}
/*
* Otherwise there was a new capture request added to list
* Need to make sure that our original data is represented if
* appropriate.
*/
for (i = 0; i < 2; i++) {
}
} else {
}
}
return (ret);
}
}
}
/*NOTREACHED*/
}
/*
* Try to capture the given page for the caller specified in the flags
* parameter. The page will either be captured and handed over to the
* appropriate callback, or will be queued up in the page capture hash
* to be captured asynchronously.
* If the current request is due to an async capture, the page must be
* exclusively locked before calling this function.
* Currently szc must be 0 but in the future this should be expandable to
* other page sizes.
* Returns 0 on success, with the following error codes on failure:
* EPERM - The requested page is long term locked, and thus repeated
* requests to capture this page will likely fail.
* ENOMEM - There was not enough free memory in the system to safely
* map the requested page.
* ENOENT - The requested page was inside the kernel cage, and the
* CAPTURE_GET_CAGE flag was not set.
* EAGAIN - The requested page could not be capturead at this point in
* time but future requests will likely work.
* EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag
* was not set.
*/
int
{
int ret;
int cb_index;
if (flags & CAPTURE_ASYNC) {
goto async;
}
/* Make sure there's enough availrmem ... */
if (ret != 0) {
return (ret);
}
break;
}
}
/* Special case for retired pages */
if (PP_RETIRED(pp)) {
if (flags & CAPTURE_GET_RETIRED) {
/*
* Need to set capture bit and add to
* hash so that the page will be
* retired when freed.
*/
ret = 0;
goto own_page;
}
} else {
return (EBUSY);
}
}
return (ret);
}
/* Need to check for physmem async requests that availrmem is sane */
(CAPTURE_ASYNC | CAPTURE_PHYSMEM) &&
(availrmem < swapfs_minfree)) {
return (ENOMEM);
}
if (ret != 0) {
/* We failed to get the page, so lets add it to the hash */
if (!(flags & CAPTURE_ASYNC)) {
}
return (ret);
}
/* Call the callback */
if (ret == 0) {
return (0);
}
/*
* Note that in the failure cases from page_capture_take_action, the
* EXCL lock will have already been dropped.
*/
}
return (EAGAIN);
}
int
{
int ret;
return (ret);
}
/*
* When unlocking a page which has the PR_CAPTURE bit set, this routine
* gets called to try and capture the page.
*/
void
{
int index;
int i;
void *datap;
extern vnode_t retired_pages;
/*
* We need to protect against a possible deadlock here where we own
* the vnode page hash mutex and want to acquire it again as there
* are locations in the code, where we unlock a page while holding
* the mutex which can lead to the page being captured and eventually
* end up here. As we may be hashing out the old page and hashing into
* the retire vnode, we need to make sure we don't own them.
* Other callbacks who do hash operations also need to make sure that
* before they hashin to a vnode that they do not currently own the
* vphm mutex otherwise there will be a panic.
*/
return;
}
return;
}
for (i = 0; i < 2; i++) {
mutex_exit(mp);
return;
}
}
}
/* Failed to find page in hash so clear flags and unlock it. */
mutex_exit(mp);
}
void
{
int i;
for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
&page_capture_hash[i].lists[0];
&page_capture_hash[i].lists[0];
}
pc_thread_retry = 3;
}
/*
* It is necessary to scrub any failing pages prior to reboot in order to
* prevent a latent error trap from occurring on the next boot.
*/
void
{
int i, j;
/* walk lists looking for pages to scrub */
for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
if (page_capture_hash[i].num_pages == 0)
continue;
for (j = 0; j < 2; j++) {
PP_CLRFREE(pp);
}
}
}
}
}
}
/*
* Walk the page_capture_hash trying to capture pages and also cleanup old
* entries which have expired.
*/
void
{
int i;
int ret;
void *datap;
/* If there are outstanding pages to be captured, get to work */
for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
if (page_capture_hash[i].num_pages == 0)
continue;
/* Append list 1 to list 0 and then walk through list 0 */
}
/* list[1] will be empty now */
/* Check expiration time */
&page_capture_hash[i].lists[0];
page_capture_hash[i].num_pages--;
/*
* We can safely remove the PR_CAPTURE bit
* without holding the EXCL lock on the page
* as the PR_CAPTURE bit requres that the
* page_capture_hash[].pchh_mutex be held
* to modify it.
*/
continue;
}
} else {
}
if (ret != 0) {
/* Move to walked hash */
(void) page_capture_move_to_walked(pp);
}
}
}
}
/*
* This function is called by the page_capture_thread, and is needed in
* in order to initiate aio cleanup, so that pages used in aio
* will be unlocked and subsequently retired by page_capture_thread.
*/
static int
do_aio_cleanup(void)
{
int (*aio_cleanup_dr_delete_memory)(proc_t *);
int cleaned = 0;
return (0);
}
/*
* We use the aio_cleanup_dr_delete_memory function to
* initiate the actual clean up; this function will wake
* up the per-process aio_cleanup_thread.
*/
aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
if (aio_cleanup_dr_delete_memory == NULL) {
"aio_cleanup_dr_delete_memory not found in kaio");
return (0);
}
/* cleanup proc's outstanding kaio */
}
}
return (cleaned);
}
/*
* helper function for page_capture_thread
*/
static void
{
int ntry;
/* Reap pages before attempting capture pages */
kmem_reap();
if ((page_retire_pend_count() > page_retire_pend_kas_count()) &&
hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
/*
* Note: Purging only for platforms that support
* platforms ISM pages SE_SHARED locked until destroyed.
*/
/* disable and purge seg_pcache */
(void) seg_p_disable();
if (!page_retire_pend_count())
break;
if (do_aio_cleanup()) {
/*
* allow the apps cleanup threads
* to run
*/
}
}
/* reenable seg_pcache */
seg_p_enable();
/* completed what can be done. break out */
return;
}
/*
* and then attempt to capture.
*/
seg_preap();
}
/*
* The page_capture_thread loops forever, looking to see if there are
* pages still waiting to be captured.
*/
static void
page_capture_thread(void)
{
callb_cpr_t c;
int outstanding;
int i;
for (;;) {
outstanding = 0;
for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++)
if (outstanding) {
CALLB_CPR_SAFE_BEGIN(&c);
} else {
CALLB_CPR_SAFE_BEGIN(&c);
}
}
/*NOTREACHED*/
}
/*
* Attempt to locate a bucket that has enough pages to satisfy the request.
* The initial check is done without the lock to avoid unneeded contention.
* The function returns 1 if enough pages were found, else 0 if it could not
* find enough pages in a bucket.
*/
static int
{
struct pcf *p;
struct pcf *q;
int i;
q = &pcf[pcf_fanout];
for (i = 0; i < pcf_fanout; i++) {
/*
* a good one to try.
*/
mutex_enter(&p->pcf_lock);
/*
* freemem is not protected by any lock.
* Thus, we cannot have any assertion
* containing freemem here.
*/
mutex_exit(&p->pcf_lock);
return (1);
}
mutex_exit(&p->pcf_lock);
}
p++;
if (p >= q) {
p = pcf;
}
}
return (0);
}
/*
* Arguments:
* pcftotal_ret: If the value is not NULL and we have walked all the
* buckets but did not find enough pages then it will
* be set to the total number of pages in all the pcf
* buckets.
* npages: Is the number of pages we have been requested to
* find.
* unlock: If set to 0 we will leave the buckets locked if the
* requested number of pages are not found.
*
* Go and try to satisfy the page request from any number of buckets.
* This can be a very expensive operation as we have to lock the buckets
* we are checking (and keep them locked), starting at bucket 0.
*
* The function returns 1 if enough pages were found, else 0 if it could not
* find enough pages in the buckets.
*
*/
static int
{
struct pcf *p;
int i;
p = pcf;
/* try to collect pages from several pcf bins */
for (pcftotal = 0, i = 0; i < pcf_fanout; i++) {
mutex_enter(&p->pcf_lock);
/*
* Wow! There are enough pages laying around
* to satisfy the request. Do the accounting,
* drop the locks we acquired, and go back.
*
* freemem is not protected by any lock. So,
* we cannot have any assertion containing
* freemem.
*/
while (p >= pcf) {
p->pcf_count = 0;
} else {
npages = 0;
}
mutex_exit(&p->pcf_lock);
p--;
}
return (1);
}
p++;
}
if (unlock) {
/* failed to collect pages - release the locks */
while (--p >= pcf) {
mutex_exit(&p->pcf_lock);
}
}
if (pcftotal_ret != NULL)
*pcftotal_ret = pcftotal;
return (0);
}