vm_page.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
/*
* University Copyright- Copyright (c) 1982, 1986, 1988
* The Regents of the University of California
* All Rights Reserved
*
* University Acknowledgment- Portions of this document are derived from
* software developed by the University of California, Berkeley, and its
* contributors.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* VM - physical page management.
*/
#include <sys/tuneable.h>
#include <sys/sysmacros.h>
#include <sys/tnf_probe.h>
#include <sys/condvar_impl.h>
#include <sys/mem_config.h>
#include <sys/mem_cage.h>
#include <vm/seg_kmem.h>
static int nopageage = 0;
/*
* vnode for all pages which are retired from the VM system;
* such as pages with Uncorrectable Errors.
*/
struct vnode retired_ppages;
static void page_retired_init(void);
void page_unretire_pages(void);
/*
* The maximum number of pages that will be unretired in one iteration.
* This number is totally arbitrary.
*/
#define UNRETIRE_PAGES 256
/*
* We limit the number of pages that may be retired to
* a percentage of the total physical memory. Note that
* the percentage values are stored as 'basis points',
* ie, 100 basis points is 1%.
*/
static int pages_retired_limit_exceeded(void);
/*
* operations vector for vnode with retired pages. Only VOP_DISPOSE
* and VOP_INACTIVE are intercepted.
*/
struct vnodeops retired_vnodeops = {
"retired_vnodeops",
fs_nosys, /* open */
fs_nosys, /* close */
fs_nosys, /* read */
fs_nosys, /* write */
fs_nosys, /* ioctl */
fs_nosys, /* setfl */
fs_nosys, /* getattr */
fs_nosys, /* setattr */
fs_nosys, /* access */
fs_nosys, /* lookup */
fs_nosys, /* create */
fs_nosys, /* remove */
fs_nosys, /* link */
fs_nosys, /* rename */
fs_nosys, /* mkdir */
fs_nosys, /* rmdir */
fs_nosys, /* readdir */
fs_nosys, /* symlink */
fs_nosys, /* readlink */
fs_nosys, /* fsync */
fs_nosys, /* fid */
fs_rwlock, /* rwlock */
fs_rwunlock, /* rwunlock */
fs_nosys, /* seek */
fs_nosys, /* cmp */
fs_nosys, /* frlock */
fs_nosys, /* space */
fs_nosys, /* realvp */
fs_nosys, /* getpage */
fs_nosys, /* putpage */
fs_nosys, /* delmap */
fs_nosys, /* dump */
fs_nosys, /* l_pathconf */
fs_nosys, /* pageio */
fs_nosys, /* dumpctl */
fs_nosys, /* setsecattr */
fs_nosys, /* getsecatt */
fs_nosys, /* shrlock */
fs_vnevent_nosupport /* vnevent */
};
/*
* freemem_lock protects all freemem variables:
* availrmem. Also this lock protects the globals which track the
* availrmem changes for accurate kernel footprint calculation.
* See below for an explanation of these
* globals.
*/
/*
* These globals track availrmem changes to get a more accurate
* estimate of tke kernel size. Historically pp_kernel is used for
* kernel size and is based on availrmem. But availrmem is adjusted for
* locked pages in the system not just for kernel locked pages.
* These new counters will track the pages locked through segvn and
* by explicit user locking.
*
* segvn_pages_locked : This keeps track on a global basis how many pages
* are currently locked because of I/O.
*
* pages_locked : How many pages are locked becuase of user specified
* locking through mlock or plock.
*
* pages_useclaim,pages_claimed : These two variables track the
* cliam adjustments because of the protection changes on a segvn segment.
*
* All these globals are protected by the same lock which protects availrmem.
*/
/*
* new_freemem_lock protects freemem, freemem_wait & freemem_cv.
*/
static kmutex_t new_freemem_lock;
static kcondvar_t freemem_cv;
/*
* The logical page free list is maintained as two lists, the 'free'
* and the 'cache' lists.
* The free list contains those pages that should be reused first.
*
* The implementation of the lists is machine dependent.
* page_get_freelist(), page_get_cachelist(),
* page_list_sub(), and page_list_add()
* form the interface to the machine dependent implementation.
*
* Pages with p_free set are on the cache list.
* Pages with p_free and p_age set are on the free list,
*
* A page may be locked while on either list.
*/
/*
* free list accounting stuff.
*
*
* Spread out the value for the number of pages on the
* page free and page cache lists. If there is just one
* value, then it must be under just one lock.
* The lock contention and cache traffic are a real bother.
*
* When we acquire and then drop a single pcf lock
* we can start in the middle of the array of pcf structures.
* If we acquire more than one pcf lock at a time, we need to
* start at the front to avoid deadlocking.
*
* pcf_count holds the number of pages in each pool.
*
* pcf_block is set when page_create_get_something() has asked the
* PSM page freelist and page cachelist routines without specifying
* a color and nothing came back. This is used to block anything
* else from moving pages from one list to the other while the
* lists are searched again. If a page is freeed while pcf_block is
* set, then pcf_reserve is incremented. pcgs_unblock() takes care
* of clearning pcf_block, doing the wakeups, etc.
*/
#if NCPU <= 4
#define PAD 1
#define PCF_FANOUT 4
#else
#define PAD 9
#ifdef sun4v
#define PCF_FANOUT 32
#else
#define PCF_FANOUT 128
#endif
#endif
struct pcf {
};
#define PAGE_LOCK_MAXIMUM \
/*
* Control over the verbosity of page retirement. When set to zero, no messages
* will be printed. A value of one will trigger messages for retirement
* operations, and is intended for processors which don't yet support FMA
* (spitfire). Two will cause verbose messages to be printed when retirements
* complete, and is intended only for debugging purposes.
*/
int page_retire_messages = 0;
#ifdef VM_STATS
/*
* No locks, but so what, they are only statistics.
*/
static struct page_tcnt {
int pc_free_cache; /* free's into cache list */
int pc_free_dontneed; /* free's with dontneed */
int pc_free_pageout; /* free's from pageout */
int pc_free_free; /* free's into free list */
int pc_free_pages; /* free's into large page free list */
int pc_destroy_pages; /* large page destroy's */
int pc_get_cache; /* get's from cache list */
int pc_get_free; /* get's from free list */
int pc_reclaim; /* reclaim's */
int pc_abortfree; /* abort's of free pages */
int pc_find_hit; /* find's that find page */
int pc_find_miss; /* find's that don't find page */
int pc_destroy_free; /* # of free pages destroyed */
int pc_addclaim_pages;
int pc_subclaim_pages;
int pc_free_replacement_page[2];
int pc_try_demote_pages[6];
int pc_demote_pages[2];
} pagecnt;
/*
* Collects statistics.
*/
\
break; \
} \
pagecnt.pc_find_hit++; \
else \
pagecnt.pc_find_miss++; \
if (mylen > PC_HASH_CNT) \
mylen = PC_HASH_CNT; \
}
#else /* VM_STATS */
/*
* Don't collect statistics
*/
break; \
} \
}
#endif /* VM_STATS */
#ifdef DEBUG
#define MEMSEG_SEARCH_STATS
#endif
#ifdef MEMSEG_SEARCH_STATS
struct memseg_stats {
} memseg_stats;
#define MEMSEG_STAT_INCR(v) \
#else
#define MEMSEG_STAT_INCR(x)
#endif
static void page_init_mem_config(void);
static void page_do_hashout(page_t *);
static void page_demote_vp_pages(page_t *);
/*
* vm subsystem related initialization
*/
void
vm_init(void)
{
boolean_t callb_vm_cpr(void *, int);
/*
* initialise the vnode for retired pages
*/
}
/*
* This function is called at startup and when memory is added or deleted.
*/
void
{
static pgcnt_t pages_pp_maximum_startup;
static pgcnt_t avrmem_delta;
static int init_done;
if (init_done == 0) {
/* If the user specified a value, save it */
if (pages_pp_maximum != 0) {
user_set = 1;
}
/*
* Setting of pages_pp_maximum is based first time
* on the value of availrmem just after the start-up
* allocations. To preserve this relationship at run
* time, use a delta from availrmem_initial.
*/
/* The allowable floor of pages_pp_maximum */
/* Make sure we don't come through here again. */
init_done = 1;
}
/*
* Determine pages_pp_maximum, the number of currently available
* pages (availrmem) that can't be `locked'. If not set by
* the user, we set it to 4% of the currently available memory
* plus 4MB.
* But we also insist that it be greater than tune.t_minarmem;
* otherwise a process could lock down a lot of memory, get swapped
* out, and never have enough to get swapped back in.
*/
if (user_set)
else
if (pages_pp_maximum <= p_min) {
}
}
void
{
}
static pgcnt_t pending_delete;
/*ARGSUSED*/
static void
void *arg,
{
}
/*ARGSUSED*/
static int
void *arg,
{
return (0);
}
/*ARGSUSED*/
static void
void *arg,
int cancelled)
{
if (!cancelled)
}
static kphysm_setup_vector_t page_mem_config_vec = {
};
static void
page_init_mem_config(void)
{
int ret;
}
/*
* Evenly spread out the PCF counters for large free pages
*/
static void
{
while (npages > 0) {
} else {
npages = 0;
}
p = pcf;
}
}
/*
* Add a physical chunk of memory to the system freee lists during startup.
* Platform specific startup() allocates the memory for the page structs.
*
* num - number of page structures
* base - page number (pfn) to be associated with the first page.
*
* Since we are doing this during startup (ie. single threaded), we will
* use shortcut routines to avoid any locking overhead while putting all
* these pages on the freelists.
*
* NOTE: Any changes performed to page_free(), must also be performed to
* add_physmem() since this is how we initialize all page_t's at
* boot time.
*/
void
{
/*
* Arbitrarily limit the max page_get request
* to 1/2 of the page structs we have.
*/
total_pages += num;
/*
* The physical space for the pages array
* representing ram pages has already been
* allocated. Here we initialize each lock
* in the page structure, and put each on
* the free list
*/
/*
* this needs to fill in the page number
* and do any other arch specific initialization
*/
/*
* Initialize the page lock as unlocked, since nobody
* can see or access this page yet.
*/
/*
* Initialize IO lock
*/
/*
* initialize other fields in the page_t
*/
PP_SETFREE(pp);
PP_SETAGED(pp);
/*
* Simple case: System doesn't support large pages.
*/
if (szc == 0) {
continue;
}
/*
* Handle unaligned pages, we collect them up onto
* the root page until we have a full large page.
*/
/*
* If not in a large page,
* just free as small page.
*/
continue;
}
/*
* Link a constituent page into the large page.
*/
/*
* When large page is fully formed, free it.
*/
cnt = 0;
}
continue;
}
/*
* At this point we have a page number which
* is aligned. We assert that we aren't already
* in a different large page.
*/
/*
* If insufficient number of pages left to form
* a large page, just free the small page.
*/
continue;
}
/*
* Otherwise start a new large page.
*/
cnt++;
}
}
/*
* Find a page representing the specified [vp, offset].
* If we find the page but it is intransit coming in,
* it will have an "exclusive" lock and we wait for
* the i/o to complete. A page found on the free list
* is always reclaimed and then locked. On success, the page
* is locked, its data is valid and it isn't on the free
* list, while a NULL is returned if the page doesn't exist.
*/
page_t *
{
}
/*
* Find a page representing the specified [vp, offset].
* We either return the one we found or, if passed in,
* create one with identity of [vp, offset] of the
* pre-allocated page. If we find exsisting page but it is
* intransit coming in, it will have an "exclusive" lock
* and we wait for the i/o to complete. A page found on
* the free list is always reclaimed and then locked.
* On success, the page is locked, its data is valid and
* it isn't on the free list, while a NULL is returned
* if the page doesn't exist and newpp is NULL;
*/
page_t *
int flags)
{
/*
* Acquire the appropriate page hash lock since
* we have to search the hash list. Pages that
* hash to this list can't change identity while
* this lock is held.
*/
hash_locked = 0;
top:
if (!hash_locked) {
/*
* On a miss, acquire the phm. Then
* next time, page_lock() will be called,
* causing a wait if the page is busy.
* just looping with page_trylock() would
* get pretty boring.
*/
hash_locked = 1;
goto top;
}
} else {
goto top;
}
}
/*
* Since `pp' is locked it can not change identity now.
* Reconfirm we locked the correct page.
*
* Both the p_vnode and p_offset *must* be cast volatile
* to force a reload of their values: The PAGE_HASH_SEARCH
* macro will have stuffed p_vnode and p_offset into
* registers before calling page_trylock(); another thread,
* actually holding the hash lock, could have changed the
* page's identity in memory, but our registers would not
* be changed, fooling the reconfirmation. If the hash
* lock was held during the search, the casting would
* not be needed.
*/
if (hash_locked) {
panic("page_lookup_create: lost page %p",
(void *)pp);
/*NOTREACHED*/
}
hash_locked = 1;
goto top;
}
/*
* If page_trylock() was called, then pp may still be on
* the cachelist (can't be on the free list, it would not
* have been found in the search). If it is on the
* cachelist it must be pulled now. To pull the page from
* the cachelist, it must be exclusively locked.
*
* The other big difference between page_trylock() and
* page_lock(), is that page_lock() will pull the
* page from whatever free list (the cache list in this
* case) the page is on. If page_trylock() was used
* above, then we have to do the reclaim ourselves.
*/
/*
* page_relcaim will insure that we
* have this page exclusively
*/
/*
* Page_reclaim dropped whatever lock
* we held.
*/
hash_locked = 1;
goto top;
}
}
if (hash_locked) {
}
NULL);
if (*nrelocp > 0) {
page_lookup_cnt[11]);
page_lookup_cnt[12]);
} else {
}
}
}
page_lookup_cnt[14]);
page_lookup_cnt[15]);
page_lookup_cnt[16]);
}
} else if (!hash_locked) {
hash_locked = 1;
goto top;
/*
* If we have a preallocated page then
* insert it now and basically behave like
* page_create.
*/
/*
* Since we hold the page hash mutex and
* just searched for this page, page_hashin
* had better not fail. If it does, that
* means some thread did not follow the
* page hash mutex rules. Panic now and
* get it over with. As usual, go down
* holding all the locks.
*/
panic("page_lookup_create: hashin failed %p %p %llx %p",
/*NOTREACHED*/
}
} else {
}
return (pp);
}
/*
* Search the hash list for the page representing the
* specified [vp, offset] and return it locked. Skip
* free pages and pages that cannot be locked as requested.
* Used while attempting to kluster pages.
*/
page_t *
{
locked = 0;
top:
locked = 1;
}
} else {
} else {
/*
* See the comment in page_lookup()
*/
if (locked) {
panic("page_lookup_nowait %p",
(void *)pp);
/*NOTREACHED*/
}
goto top;
}
}
}
}
if (locked) {
}
return (pp);
}
/*
* Search the hash list for a page with the specified [vp, off]
* that is known to exist and is already locked. This routine
* is typically used by segment SOFTUNLOCK routines.
*/
page_t *
{
return (pp);
}
/*
* Determine whether a page with the specified [vp, off]
* currently exists in the system. Obviously this should
* only be considered as a hint since nothing prevents the
* page from disappearing or appearing immediately after
* the return from this routine. Subsequently, we don't
* even bother to lock the list.
*/
page_t *
{
return (pp);
}
/*
* Determine if physically contiguous pages exist for [vp, off] - [vp, off +
* page_size(szc)) range. if they exist and ppa is not NULL fill ppa array
* with these pages locked SHARED. If necessary reclaim pages from
* freelist. Return 1 if contiguous pages exist and 0 otherwise.
*
* If we fail to lock pages still return 1 if pages exist and contiguous.
* But in this case return value is just a hint. ppa array won't be filled.
* Caller should initialize ppa[0] as NULL to distinguish return value.
*
* Returns 0 if pages don't exist or not physically contiguous.
*
* This routine doesn't work for anonymous(swapfs) pages.
*/
int
{
pgcnt_t i;
pgcnt_t j;
int loopcnt = 0;
if (++loopcnt > 3) {
return (0);
}
return (0);
}
return (1);
}
goto again;
}
/*
* szc was non zero and vnode and offset matched after we
* locked the page it means it can't become free on us.
*/
return (0);
}
pp++;
pfn++;
pp--;
while (i-- > 0) {
pp--;
}
return (1);
}
pp--;
while (i-- > 0) {
pp--;
}
goto again;
}
/*
* szc the same as for previous already locked pages
* with right identity. Since this page had correct
* szc after we locked it can't get freed or destroyed
* and therefore must have the expected identity.
*/
panic("page_exists_physcontig: "
"large page identity doesn't match");
}
}
return (1);
return (0);
}
return (1);
}
return (0);
}
if (page_numtomemseg_nolock(pfn) !=
return (0);
}
/*
* We loop up 4 times across pages to promote page size.
* We're extra cautious to promote page size atomically with respect
* to everybody else. But we can probably optimize into 1 loop if
* this becomes an issue.
*/
break;
}
break;
}
ASSERT(i == 0);
goto again;
}
}
if (i != pages) {
--pp;
while (i-- > 0) {
--pp;
}
return (0);
}
break;
}
} else {
}
}
if (i < pages) {
/*
* page_reclaim failed because we were out of memory.
* drop the rest of the locks and return because this page
* must be already reallocated anyway.
*/
if (j != i) {
}
}
return (0);
}
}
} else {
page_downgrade(ppa[i]);
}
}
}
return (1);
}
/*
* Determine whether a page with the specified [vp, off]
* currently exists in the system and if so return its
* size code. Obviously this should only be considered as
* a hint since nothing prevents the page from disappearing
* or appearing immediately after the return from this routine.
*/
int
{
int rc = 0;
rc = 1;
}
return (rc);
}
/* wakeup threads waiting for pages in page_create_get_something() */
void
wakeup_pcgs(void)
{
if (!CV_HAS_WAITERS(&pcgs_cv))
return;
}
/*
* 'freemem' is used all over the kernel as an indication of how many
* pages are free (either on the cache list or on the free page list)
* in the system. In very few places is a really accurate 'freemem'
* needed. To avoid contention of the lock protecting a the
* single freemem, it was spread out into NCPU buckets. Set_freemem
* sets freemem to the total of all NCPU buckets. It is called from
* clock() on each TICK.
*/
void
{
struct pcf *p;
ulong_t t;
uint_t i;
t = 0;
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
t += p->pcf_count;
p++;
}
freemem = t;
/*
* Don't worry about grabbing mutex. It's not that
* critical if we miss a tick or two. This is
* where we wakeup possible delayers in
* page_create_get_something().
*/
wakeup_pcgs();
}
{
struct pcf *p;
ulong_t t;
uint_t i;
t = 0;
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
t += p->pcf_count;
p++;
}
/*
* We just calculated it, might as well set it.
*/
freemem = t;
return (t);
}
/*
* Acquire all of the page cache & free (pcf) locks.
*/
void
{
struct pcf *p;
uint_t i;
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
p->pcf_touch = 1;
mutex_enter(&p->pcf_lock);
p++;
}
}
/*
* Release all the pcf_locks.
*/
void
{
struct pcf *p;
uint_t i;
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
mutex_exit(&p->pcf_lock);
p++;
}
}
/*
* Inform the VM system that we need some pages freed up.
* Calls must be symmetric, e.g.:
*
* page_needfree(100);
* wait a bit;
* page_needfree(-100);
*/
void
{
}
/*
* Throttle for page_create(): try to prevent freemem from dropping
* below throttlefree. We can't provide a 100% guarantee because
* KM_NOSLEEP allocations, page_reclaim(), and various other things
* nibble away at the freelist. However, we can block all PG_WAIT
* allocations until memory becomes available. The motivation is
* that several things can fall apart when there's no free memory:
*
* (1) If pageout() needs memory to push a page, the system deadlocks.
*
* (2) By (broken) specification, timeout(9F) can neither fail nor
* block, so it has no choice but to panic the system if it
* cannot allocate a callout structure.
*
* (3) Like timeout(), ddi_set_callback() cannot fail and cannot block;
* it panics if it cannot allocate a callback structure.
*
* (4) Untold numbers of third-party drivers have not yet been hardened
* success and panic the system with a data fault on failure.
* (The long-term solution to this particular problem is to ship
* hostile fault-injecting DEBUG kernels with the DDK.)
*
* It is theoretically impossible to guarantee success of non-blocking
* allocations, but in practice, this throttle is very hard to break.
*/
static int
{
uint_t i;
/*
* Never deny pages when:
* - it's a thread that cannot block [NOMEMWAIT()]
* - the allocation cannot block and must not fail
* - the allocation cannot block and is pageout dispensated
*/
if (NOMEMWAIT() ||
return (1);
/*
* If the allocation can't block, we look favorably upon it
* unless we're below pageout_reserve. In that case we fail
* the allocation because we want to make sure there are a few
* pages available for pageout.
*/
/* Calculate the effective throttlefree value */
tf = throttlefree -
fm = 0;
for (i = 0; i < PCF_FANOUT; i++) {
}
freemem_wait++;
freemem_wait--;
}
return (1);
}
/*
* page_create_wait() is called to either coalecse pages from the
* different pcf buckets or to wait because there simply are not
* enough pages to satisfy the caller's request.
*
* Sadly, this is called from platform/vm/vm_machdep.c
*/
int
{
uint_t i;
struct pcf *p;
/*
* Wait until there are enough free pages to satisfy our
* entire request.
* We set needfree += npages before prodding pageout, to make sure
* it does real work when npages > lotsfree > freemem.
*/
if ((flags & PG_NORELOC) &&
return (0);
/*
* Since page_create_va() looked at every
* bucket, assume we are going to have to wait.
* Get all of the pcf locks.
*/
total = 0;
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
p->pcf_touch = 1;
mutex_enter(&p->pcf_lock);
/*
* Wow! There are enough pages laying around
* to satisfy the request. Do the accounting,
* drop the locks we acquired, and go back.
*
* freemem is not protected by any lock. So,
* we cannot have any assertion containing
* freemem.
*/
while (p >= pcf) {
p->pcf_count = 0;
} else {
npages = 0;
}
mutex_exit(&p->pcf_lock);
p--;
}
return (1);
}
p++;
}
/*
* All of the pcf locks are held, there are not enough pages
* to satisfy the request (npages < total).
* Be sure to acquire the new_freemem_lock before dropping
* the pcf locks. This prevents dropping wakeups in page_free().
* The order is always pcf_lock then new_freemem_lock.
*
* Since we hold all the pcf locks, it is a good time to set freemem.
*
* If the caller does not want to wait, return now.
* Else turn the pageout daemon loose to find something
* and wait till it does.
*
*/
return (0);
}
"page_create_sleep_start: freemem %ld needfree %ld",
/*
* We are going to wait.
* We currently hold all of the pcf_locks,
* get the new_freemem_lock (it protects freemem_wait),
* before dropping the pcf_locks.
*/
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
p->pcf_wait++;
mutex_exit(&p->pcf_lock);
p++;
}
freemem_wait++;
freemem_wait--;
"page_create_sleep_end: freemem %ld needfree %ld",
goto checkagain;
}
/*
* A routine to do the opposite of page_create_wait().
*/
void
{
struct pcf *p;
/*
* When a contiguous lump is broken up, we have to
* deal with lots of pages (min 64) so lets spread
* the wealth around.
*/
mutex_enter(&p->pcf_lock);
if (p->pcf_block) {
which = &p->pcf_reserve;
}
} else {
npages = 0;
}
if (p->pcf_wait) {
/*
* Check to see if some other thread
* is actually waiting. Another bucket
* may have woken it up by now. If there
* are no waiters, then set our pcf_wait
* count to zero to avoid coming in here
* next time.
*/
if (freemem_wait) {
if (npages > 1) {
} else {
}
p->pcf_wait--;
} else {
p->pcf_wait = 0;
}
}
mutex_exit(&p->pcf_lock);
}
}
/*
* A helper routine for page_create_get_something.
* The indenting got to deep down there.
* Unblock the pcf counters. Any pages freed after
* pcf_block got set are moved to pcf_count and
* wakeups (cv_broadcast() or cv_signal()) are done as needed.
*/
static void
pcgs_unblock(void)
{
int i;
struct pcf *p;
/* Update freemem while we're here. */
freemem = 0;
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
mutex_enter(&p->pcf_lock);
p->pcf_count = p->pcf_reserve;
p->pcf_block = 0;
if (p->pcf_wait) {
if (freemem_wait) {
if (p->pcf_reserve > 1) {
p->pcf_wait = 0;
} else {
p->pcf_wait--;
}
} else {
p->pcf_wait = 0;
}
}
p->pcf_reserve = 0;
mutex_exit(&p->pcf_lock);
p++;
}
}
/*
* Called from page_create_va() when both the cache and free lists
* have been checked once.
*
* Either returns a page or panics since the accounting was done
* way before we got here.
*
* We don't come here often, so leave the accounting on permanently.
*/
#define MAX_PCGS 100
#ifdef DEBUG
#define PCGS_TRIES 100
#else /* DEBUG */
#define PCGS_TRIES 10
#endif /* DEBUG */
#ifdef VM_STATS
#endif /* VM_STATS */
static page_t *
{
struct pcf *p;
int cagelocked = 0;
/*
* Tap any reserve freelists: if we fail now, we'll die
* since the page(s) we're looking for have already been
* accounted for.
*/
if ((flags & PG_NORELOC) != 0) {
/*
* Requests for free pages from critical threads
* such as pageout still won't throttle here, but
* we must try again, to give the cageout thread
* another chance to catch up. Since we already
* accounted for the pages, we had better get them
* this time.
*
* N.B. All non-critical threads acquire the pcgs_cagelock
* to serialize access to the freelists. This implements a
* turnstile-type synchornization to avoid starvation of
* critical requests for PG_NORELOC memory by non-critical
* threads: all non-critical threads must acquire a 'ticket'
* before passing through, which entails making sure
* kcage_freemem won't fall below minfree prior to grabbing
* pages from the freelists.
*/
cagelocked = 1;
}
}
/*
* Time to get serious.
* We failed to get a `correctly colored' page from both the
* free and cache lists.
* We escalate in stage.
*
* First try both lists without worring about color.
*
* Then, grab all page accounting locks (ie. pcf[]) and
* steal any pages that they have and set the pcf_block flag to
* stop deletions from the lists. This will help because
* a page can get added to the free list while we are looking
* at the cache list, then another page could be added to the cache
* list allowing the page on the free list to be removed as we
* move from looking at the cache list to the free list. This
* could happen over and over. We would never find the page
* we have accounted for.
*
* Noreloc pages are a subset of the global (relocatable) page pool.
* They are not tracked separately in the pcf bins, so it is
* impossible to know when doing pcf accounting if the available
* page(s) are noreloc pages or not. When looking for a noreloc page
* it is quite easy to end up here even if the global (relocatable)
* page pool has plenty of free pages but the noreloc pool is empty.
*
* When the noreloc pool is empty (or low), additional noreloc pages
* are created by converting pages from the global page pool. This
* process will stall during pcf accounting if the pcf bins are
* already locked. Such is the case when a noreloc allocation is
* looping here in page_create_get_something waiting for more noreloc
* pages to appear.
*
* Short of adding a new field to the pcf bins to accurately track
* the number of free noreloc pages, we instead do not grab the
* pcgs_lock, do not set the pcf blocks and do not timeout when
* allocating a noreloc page. This allows noreloc allocations to
* loop without blocking global page pool allocations.
*
* NOTE: the behaviour of page_create_get_something has not changed
* for the case of global page pool allocations.
*/
flags &= ~PG_MATCH_COLOR;
locked = 0;
#ifndef __sparc
/*
* page_create_get_something may be called because 4g memory may be
* depleted. Set flags to allow for relocation of base page below
* 4g if necessary.
*/
if (physmax4g)
#endif
}
/*
* Serialize. Don't fight with other pcgs().
*/
locked = 1;
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
mutex_enter(&p->pcf_lock);
p->pcf_block = 1;
p->pcf_reserve = p->pcf_count;
p->pcf_count = 0;
mutex_exit(&p->pcf_lock);
p++;
}
freemem = 0;
}
if (count) {
/*
* Since page_free() puts pages on
* a list then accounts for it, we
* just have to wait for page_free()
* to unlock any page it was working
* with. The page_lock()-page_reclaim()
* path falls in the same boat.
*
* We don't need to check on the
* PG_WAIT flag, we have already
* accounted for the page we are
* looking for in page_create_va().
*
* We just wait a moment to let any
* locked pages on the lists free up,
* then continue around and try again.
*
* Will be awakened by set_freemem().
*/
}
} else {
#ifdef VM_STATS
if (count >= PCGS_TRIES) {
} else {
}
#endif
if (locked) {
pcgs_unblock();
}
if (cagelocked)
return (pp);
}
}
/*
* we go down holding the pcf locks.
*/
panic("no %spage found %d",
/*NOTREACHED*/
}
/*
* Create enough pages for "bytes" worth of data starting at
* "off" in "vp".
*
* Where flag must be one of:
*
* PG_EXCL: Exclusive create (fail if any page already
* exists in the page cache) which does not
* wait for memory to become available.
*
* PG_WAIT: Non-exclusive create which can wait for
* memory to become available.
*
* PG_PHYSCONTIG: Allocate physically contiguous pages.
* (Not Supported)
*
* A doubly linked list of pages is returned to the caller. Each page
* on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock)
* lock.
*
* Unable to change the parameters to page_create() in a minor release,
* we renamed page_create() to page_create_va(), changed all known calls
* from page_create() to page_create_va(), and created this wrapper.
*
* Upon a major release, we should break compatibility by deleting this
* wrapper, and replacing all the strings "page_create_va", with "page_create".
*
* NOTE: There is a copy of this interface as page_create_io() in
* i86/vm/vm_machdep.c. Any bugs fixed here should be applied
* there.
*/
page_t *
{
#ifdef DEBUG
(void *)caller());
#endif
}
#ifdef DEBUG
#endif
/*
* Used for large page support. It will attempt to allocate
* a large page(s) off the freelist.
*
* Returns non zero on failure.
*/
int
{
int err = 0;
VM_STAT_ADD(alloc_pages[0]);
#ifdef DEBUG
return (ENOMEM);
}
#endif
/*
* One must be NULL but not both.
* And one must be non NULL but not both.
*/
} else if (anypgsz) {
szc--;
} else {
return (ENOMEM);
}
}
if (szc == 0) {
}
/*
* Clear the free and age bits. Also if we were passed in a ppa then
* fill it in with all the constituent pages from the large page. But
* if we failed to allocate all the pages just free what we got.
*/
while (npgs != 0) {
if (err == 0) {
PP_CLRFREE(pp);
PP_CLRAGED(pp);
npgs--;
} else {
page_list_add_pages(pp, 0);
}
} else {
PP_CLRFREE(pp);
PP_CLRAGED(pp);
npgs--;
}
}
return (err);
}
/*
* Get a single large page off of the freelists, and set it up for use.
* Number of bytes requested must be a supported page size.
*
* Note that this call may fail even if there is sufficient
* memory available or PG_WAIT is set, so the caller must
* be willing to fallback on page_create_va(), block and retry,
* or fail the requester.
*/
page_t *
{
uint_t i;
struct pcf *p;
struct pcf *q;
/* but no others */
/*
* Cage is OFF, or we are single threaded in
* panic, so make everything a RELOC request.
*/
flags &= ~PG_NORELOC;
}
/*
* Make sure there's adequate physical memory available.
* Note: PG_WAIT is ignored here.
*/
return (NULL);
}
/*
* If cage is on, dampen draw from cage when available
* cage space is low.
*/
/*
* The cage is on, the caller wants PG_NORELOC
* pages and available cage memory is very low.
* Call kcage_create_throttle() to attempt to
* control demand on the cage.
*/
return (NULL);
}
}
enough = 0;
p->pcf_touch = 1;
q = &pcf[PCF_FANOUT];
for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) {
/*
* a good one to try.
*/
mutex_enter(&p->pcf_lock);
/*
* freemem is not protected by any lock.
* Thus, we cannot have any assertion
* containing freemem here.
*/
enough = 1;
mutex_exit(&p->pcf_lock);
break;
}
mutex_exit(&p->pcf_lock);
}
p++;
if (p >= q) {
p = pcf;
}
p->pcf_touch = 1;
}
if (!enough) {
/* If there isn't enough memory available, give up. */
return (NULL);
}
/* try to collect pages from several pcf bins */
p->pcf_touch = 1;
mutex_enter(&p->pcf_lock);
/*
* Wow! There are enough pages laying around
* to satisfy the request. Do the accounting,
* drop the locks we acquired, and go back.
*
* freemem is not protected by any lock. So,
* we cannot have any assertion containing
* freemem.
*/
while (p >= pcf) {
p->pcf_count = 0;
} else {
tpages = 0;
}
mutex_exit(&p->pcf_lock);
p--;
}
break;
}
p++;
}
if (i == PCF_FANOUT) {
/* failed to collect pages - release the locks */
while (--p >= pcf) {
mutex_exit(&p->pcf_lock);
}
return (NULL);
}
}
/*
* This is where this function behaves fundamentally differently
* than page_create_va(); since we're intending to map the page
* with a single TTE, we have to get it as a physically contiguous
* hardware pagesize chunk. If we can't, we fail.
*/
else
return (NULL);
}
/*
* if we got the page with the wrong mtype give it back this is a
* workaround for CR 6249718. When CR 6249718 is fixed we never get
* inside "if" and the workaround becomes just a nop
*/
return (NULL);
}
/*
* If satisfying this request has left us with too little
* memory, start the wheels turning to get some back. The
* first clause of the test prevents waking up the pageout
* daemon in situations where it would decide that there's
* nothing to do.
*/
"pageout_cv_signal:freemem %ld", freemem);
}
while (npages--) {
PP_CLRFREE(pp);
PP_CLRAGED(pp);
panic("page_create_large: hashin failed: page %p",
(void *)pp);
}
return (rootpp);
}
page_t *
{
pgcnt_t found_on_free = 0;
uint_t i;
struct pcf *p;
struct pcf *q;
"page_create_start:vp %p off %llx bytes %lu flags %x",
panic("page_create: invalid flags");
/*NOTREACHED*/
}
/* but no others */
/*
* Try to see whether request is too large to *ever* be
* satisfied, in order to prevent deadlock. We arbitrarily
* decide to limit maximum size requests to max_page_get.
*/
if (npages >= max_page_get) {
"page_create_toobig:vp %p off %llx npages "
"%lu max_page_get %lu",
return (NULL);
} else {
"Request for too much kernel memory "
"(%lu bytes), will hang forever", bytes);
for (;;)
delay(1000000000);
}
}
/*
* Cage is OFF, or we are single threaded in
* panic, so make everything a RELOC request.
*/
flags &= ~PG_NORELOC;
}
return (NULL);
/*
* If cage is on, dampen draw from cage when available
* cage space is low.
*/
if ((flags & PG_NORELOC) &&
/*
* The cage is on, the caller wants PG_NORELOC
* pages and available cage memory is very low.
* Call kcage_create_throttle() to attempt to
* control demand on the cage.
*/
return (NULL);
}
enough = 0;
p->pcf_touch = 1;
q = &pcf[PCF_FANOUT];
for (i = 0; i < PCF_FANOUT; i++) {
/*
* a good one to try.
*/
mutex_enter(&p->pcf_lock);
/*
* freemem is not protected by any lock.
* Thus, we cannot have any assertion
* containing freemem here.
*/
enough = 1;
mutex_exit(&p->pcf_lock);
break;
}
mutex_exit(&p->pcf_lock);
}
p++;
if (p >= q) {
p = pcf;
}
p->pcf_touch = 1;
}
if (!enough) {
/*
* Have to look harder. If npages is greater than
* one, then we might have to coalecse the counters.
*
* Go wait. We come back having accounted
* for the memory.
*/
return (NULL);
}
}
/*
* If satisfying this request has left us with too little
* memory, start the wheels turning to get some back. The
* first clause of the test prevents waking up the pageout
* daemon in situations where it would decide that there's
* nothing to do.
*/
"pageout_cv_signal:freemem %ld", freemem);
}
/*
* Loop around collecting the requested number of pages.
* Most of the time, we have to `create' a new page. With
* this in mind, pull the page off the free list before
* getting the hash lock. This will minimize the hash
* lock hold time, nesting, and the like. If it turns
* out we don't need the page, we put it back at the end.
*/
while (npages--) {
top:
/*
* Try to get a page from the freelist (ie,
* a page with no [vp, off] tag). If that
* fails, use the cachelist.
*
* During the first attempt at both the free
* and cache lists we try for the correct color.
*/
/*
* XXXX-how do we deal with virtual indexed
* caches and and colors?
*/
/*
* Get lgroup to allocate next page of shared memory
* from and use it to specify where to allocate
* the physical memory
*/
flags & ~PG_MATCH_COLOR);
}
/*
* Since this page came from the
* cachelist, we must destroy the
* old vnode association.
*/
}
}
}
/*
* We own this page!
*/
/*
* Here we have a page in our hot little mits and are
* just waiting to stuff it on the appropriate lists.
* Get the mutex and check to see if it really does
* not exist.
*/
/*
* Since we hold the page hash mutex and
* just searched for this page, page_hashin
* had better not fail. If it does, that
* means somethread did not follow the
* page hash mutex rules. Panic now and
* get it over with. As usual, go down
* holding all the locks.
*/
panic("page_create: "
"hashin failed %p %p %llx %p",
/*NOTREACHED*/
}
/*
* Hat layer locking need not be done to set
* the following bits since the page is not hashed
* and was on the free list (i.e., had no mappings).
*
* Set the reference bit to protect
* against immediate pageout
*
* XXXmh modify freelist code to set reference
* bit so we don't have to do it here.
*/
} else {
/*
* Found an existing page, and the caller
* wanted all new pages. Undo all of the work
* we have done.
*/
/* large pages should not end up here */
/*LINTED: constant in conditional ctx*/
}
goto fail;
}
/*
* Start all over again if we blocked trying
* to lock the page.
*/
goto top;
}
PP_CLRFREE(pp);
}
}
/*
* Got a page! It is locked. Acquire the i/o
* lock since we are going to use the p_next and
* p_prev fields to link the requested pages together.
*/
}
fail:
/*
* Did not need this page after all.
* Put it back on the free list.
*/
}
{
if (overshoot) {
p->pcf_touch = 1;
mutex_enter(&p->pcf_lock);
if (p->pcf_block) {
p->pcf_reserve += overshoot;
} else {
if (p->pcf_wait) {
if (freemem_wait) {
p->pcf_wait--;
} else {
p->pcf_wait = 0;
}
}
}
mutex_exit(&p->pcf_lock);
/* freemem is approximate, so this test OK */
if (!p->pcf_block)
}
}
return (plist);
}
/*
* One or more constituent pages of this large page has been marked
* toxic. Simply demote the large page to PAGESIZE pages and let
* page_free() handle it. This routine should only be called by
* large page free routines (page_free_pages() and page_destroy_pages().
* All pages are locked SE_EXCL and have already been marked free.
*/
static void
{
}
}
}
/*
* Put page on the "free" list.
* The free list is really two lists maintained by
* the PSM of whatever machine we happen to be on.
*/
void
{
struct pcf *p;
if (page_deteriorating(pp)) {
volatile int i = 0;
char *kaddr;
volatile int ue = 0;
/*
* Let page_destroy() do its bean counting and
* hash out the page; it will then call back
* into page_free() with pp->p_vnode == NULL.
*/
page_destroy(pp, 0);
return;
}
if (page_isfailing(pp)) {
/*
* If we have already exceeded the limit for
* pages retired, we will treat this page as
* 'toxic' rather than failing. That will ensure
* that the page is at least cleaned, and if
* a UE is detected, the page will be retired
* anyway.
*/
if (pages_retired_limit_exceeded()) {
/*
* clear the flag and reset to toxic
*/
} else {
if (page_retire_messages) {
"removed from service",
}
goto page_failed;
}
}
/*
* We want to determine whether the error that occurred on
* this page is transient or persistent, so we get a mapping
* to the page and try every possible bit pattern to compare
* what we write with what we read back. A smaller number
* of bit patterns might suffice, but there's no point in
* getting fancy. If this is the hot path on your system,
* you've got bigger problems.
*/
if (page_retire_messages) {
"occurred at PA 0x%08x.%08x while "
"attempting to clear previously "
"reported error; page removed from "
}
ue++;
break;
}
/*
* Write out the bit pattern, flush it to memory, and
* read it back while under on_trap() protection.
*/
for (i = 0; i < PAGESIZE; i++)
for (i = 0; i < PAGESIZE; i++) {
goto out;
}
}
}
out:
no_trap();
if (page_retire_messages) {
"0x%08x.%08x [ 0x%x != 0x%x ] while "
"attempting to clear previously reported "
"error; page removed from service",
}
}
/*
* DR operations change the association between a page_t
* and the physical page it represents. Check if the
* page is still bad. If it is, then retire it.
*/
/*
* In the future, it might be useful to have a platform
* callback here to tell the hardware to fence off this
* page during the next reboot.
*
* We move the page to the retired_vnode here
*/
availrmem--;
/*
* If DR raced with the above page retirement code,
* we might have retired a good page. If so, unretire
* the page.
*/
if (!page_isfaulty(pp))
return;
}
if (page_retire_messages) {
}
}
}
panic("page_free: anon or kernel "
"or no vnode large page %p", (void *)pp);
}
}
/*
* The page_struct_lock need not be acquired to examine these
* fields since the page has an "exclusive" lock.
*/
panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d",
/*NOTREACHED*/
}
PP_SETFREE(pp);
/*
* Now we add the page to the head of the free list.
* But if this page is associated with a paged vnode
* then we adjust the head forward so that the page is
* effectively at the end of the list.
*/
/*
* Page has no identity, put it on the free list.
*/
PP_SETAGED(pp);
"page_free_free:pp %p", pp);
} else {
PP_CLRAGED(pp);
/* move it to the tail of the list */
"page_free_cache_tail:pp %p", pp);
} else {
"page_free_cache_head:pp %p", pp);
}
}
/*
* Now do the `freemem' accounting.
*/
p->pcf_touch = 1;
mutex_enter(&p->pcf_lock);
if (p->pcf_block) {
p->pcf_reserve += 1;
} else {
p->pcf_count += 1;
if (p->pcf_wait) {
/*
* Check to see if some other thread
* is actually waiting. Another bucket
* may have woken it up by now. If there
* are no waiters, then set our pcf_wait
* count to zero to avoid coming in here
* next time. Also, since only one page
* was put on the free list, just wake
* up one waiter.
*/
if (freemem_wait) {
p->pcf_wait--;
} else {
p->pcf_wait = 0;
}
}
}
mutex_exit(&p->pcf_lock);
/* freemem is approximate, so this test OK */
if (!p->pcf_block)
freemem += 1;
}
/*
* Put page on the "free" list during intial startup.
* This happens during initial single threaded execution.
*/
void
{
struct pcf *p;
/*
* Now do the `freemem' accounting.
*/
p->pcf_touch = 1;
p->pcf_count += 1;
/* freemem is approximate, so this is OK */
freemem += 1;
}
void
{
pgcnt_t i;
int toxic = 0;
"page_free_free:pp %p", pp);
/*NOTREACHED*/
}
/*NOTREACHED*/
}
/*NOTREACHED*/
}
if (page_deteriorating(tpp))
toxic = 1;
}
if (toxic) {
return;
}
}
int free_pages = 1;
/*
* This routine attempts to return pages to the cachelist via page_release().
* It does not *have* to be successful in all cases, since the pageout scanner
* will catch any pages it misses. It does need to be fast and not introduce
* too much overhead.
*
* If a page isn't found on the unlocked sweep of the page_hash bucket, we
* don't lock and retry. This is ok, since the page scanner will eventually
* find any page we miss in free_vp_pages().
*/
void
{
if (free_pages == 0)
return;
return;
/*
* find the page using a fast, but inexact search. It'll be OK
* if a few pages slip through the cracks here.
*/
/*
* If we didn't find the page (it may not exist), the page
* is free, looks still in use (shared), or we can't lock it,
* just give up.
*/
page_share_cnt(pp) > 0 ||
continue;
/*
* Once we have locked pp, verify that it's still the
* correct page and not already free
*/
continue;
}
/*
* try to release the page...
*/
}
}
/*
* Reclaim the given page from the free list.
* Returns 1 on success or 0 on failure.
*
* The page is unlocked if it can't be reclaimed (when freemem == 0).
* If `lock' is non-null, it will be dropped and re-acquired if
* the routine must wait while freemem is 0.
*
* As it turns out, boot_getpages() does this. It picks a page,
* based on where OBP mapped in some address, gets its pfn, searches
* the memsegs, locks the page, then pulls it off the free list!
*/
int
{
struct pcf *p;
int enough;
uint_t i;
/*
* If `freemem' is 0, we cannot reclaim this page from the
* freelist, so release every lock we might hold: the page,
* and the `lock' before blocking.
*
* The only way `freemem' can become 0 while there are pages
* marked free (have their p->p_free bit set) is when the
* system is low on memory and doing a page_create(). In
* order to guarantee that once page_create() starts acquiring
* pages it will be able to get all that it needs since `freemem'
* was decreased by the requested amount. So, we need to release
* this page, and let page_create() have it.
*
* Since `freemem' being zero is not supposed to happen, just
* use the usual hash stuff as a starting point. If that bucket
* is empty, then assume the worst, and start at the beginning
* of the pcf array. If we always start at the beginning
* when acquiring more than one pcf lock, there won't be any
* deadlock problems.
*/
/* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
goto page_reclaim_nomem;
}
enough = 0;
p->pcf_touch = 1;
mutex_enter(&p->pcf_lock);
if (p->pcf_count >= 1) {
enough = 1;
p->pcf_count--;
}
mutex_exit(&p->pcf_lock);
if (!enough) {
/*
* Check again. Its possible that some other thread
* could have been right behind us, and added one
* to a list somewhere. Acquire each of the pcf locks
* until we find a page.
*/
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
p->pcf_touch = 1;
mutex_enter(&p->pcf_lock);
if (p->pcf_count >= 1) {
p->pcf_count -= 1;
enough = 1;
break;
}
p++;
}
if (!enough) {
/*
* We really can't have page `pp'.
* Time for the no-memory dance with
* page_free(). This is just like
* page_create_wait(). Plus the added
* attraction of releasing whatever mutex
* we held when we were called with in `lock'.
* Page_unlock() will wakeup any thread
* waiting around for this page.
*/
if (lock) {
}
/*
* get this before we drop all the pcf locks.
*/
p = pcf;
for (i = 0; i < PCF_FANOUT; i++) {
p->pcf_wait++;
mutex_exit(&p->pcf_lock);
p++;
}
freemem_wait++;
freemem_wait--;
if (lock) {
}
return (0);
}
/*
* There was a page to be found.
* The pcf accounting has been done,
* though none of the pcf_wait flags have been set,
* drop the locks and continue on.
*/
while (p >= pcf) {
mutex_exit(&p->pcf_lock);
p--;
}
}
/*
* freemem is not protected by any lock. Thus, we cannot
* have any assertion containing freemem here.
*/
freemem -= 1;
"page_reclaim_free:pp %p", pp);
} else {
"page_reclaim_cache:pp %p", pp);
}
/*
* clear the p_free & p_age bits since this page is no longer
* on the free list. Notice that there was a brief time where
* a page is marked as free, but is not on the list.
*
* Set the reference bit to protect against immediate pageout.
*/
PP_CLRFREE(pp);
PP_CLRAGED(pp);
return (1);
}
/*
* Destroy identity of the page and put it back on
* the page free list. Assumes that the caller has
* acquired the "exclusive" lock on the page.
*/
void
{
panic("page_destroy: anon or kernel or no vnode "
"large page %p", (void *)pp);
}
}
/*
* Unload translations, if any, then hash out the
* page to erase its identity.
*/
if (!dontfree) {
/*
* Acquire the "freemem_lock" for availrmem.
* The page_struct_lock need not be acquired for lckcnt
* and cowcnt since the page has an "exclusive" lock.
*/
availrmem++;
}
}
}
/*
* Put the page on the "free" list.
*/
}
}
void
{
int toxic = 0;
/*NOTREACHED*/
}
pglcks++;
}
if (page_deteriorating(tpp))
toxic = 1;
}
if (pglcks != 0) {
}
if (toxic) {
return;
}
}
/*
* Similar to page_destroy(), but destroys pages which are
* locked and known to be on the page free list. Since
* the page is known to be free and locked, no one can access
* it.
*
* Also, the number of free pages does not change.
*/
void
{
PP_SETAGED(pp);
if (freemem_wait) {
}
}
/*
* Rename the page "opp" to have an identity specified
* by [vp, off]. If a page already exists with this name
* it is locked and destroyed. Note that the page's
* translations are not unloaded during the rename.
*
* This routine is used by the anon layer to "steal" the
* original page and is not unlike destroying a page and
* creating a new page using the same page frame.
*
* XXX -- Could deadlock if caller 1 tries to rename A to B while
* caller 2 tries to rename B to A.
*/
void
{
int olckcnt = 0;
int ocowcnt = 0;
/*
* Acquire the appropriate page hash lock, since
* we're going to rename the page.
*/
top:
/*
* Look for an existing page with this name and destroy it if found.
* By holding the page hash lock all the way to the page_hashin()
* call, we are assured that no page can be created with this
* identity. In the case when the phm lock is dropped to undo any
* hat layer mappings, the existing page is held with an "exclusive"
* lock, again preventing another page from being created with
* this identity.
*/
/*
* As it turns out, this is one of only two places where
* page_lock() needs to hold the passed in lock in the
* successful case. In all of the others, the lock could
* be dropped as soon as the attempt is made to lock
* the page. It is tempting to add yet another arguement,
* PL_KEEP or PL_DROP, to let page_lock know what to do.
*/
/*
* Went to sleep because the page could not
* be locked. We were woken up when the page
* was unlocked, or when the page was destroyed.
* In either case, `phm' was dropped while we
* slept. Hence we should not just roar through
* this loop.
*/
goto top;
}
if (hat_page_is_mapped(pp)) {
/*
* Unload translations. Since we hold the
* exclusive lock on this page, the page
* can not be changed while we drop phm.
* This is also not a lock protocol violation,
* but rather the proper way to do things.
*/
}
}
/*
* Hash in the page with the new identity.
*/
/*
* We were holding phm while we searched for [vp, off]
* and only dropped phm if we found and locked a page.
* If we can't create this page now, then some thing
* is really broken.
*/
/*NOTREACHED*/
}
/*
* Now that we have dropped phm, lets get around to finishing up
* with pp.
*/
/* for now large pages should not end up here */
/*
* Save the locks for transfer to the new page and then
* clear them so page_free doesn't think they're important.
* The page_struct_lock need not be acquired for lckcnt and
* cowcnt since the page has an "exclusive" lock.
*/
/*
* Put the page on the "free" list after we drop
* the lock. The less work under the lock the better.
*/
/*LINTED: constant in conditional context*/
}
/*
* Transfer the lock count from the old page (if any).
* The page_struct_lock need not be acquired for lckcnt and
* cowcnt since the page has an "exclusive" lock.
*/
}
/*
* low level routine to add page `pp' to the hash and vp chains for [vp, offset]
*
* Pages are normally inserted at the start of a vnode's v_pages list.
* If the vnode is VMODSORT and the page is modified, it goes at the end.
* This can happen when a modified page is relocated for DR.
*
* Returns 1 on success and 0 on failure.
*/
static int
{
/*
* Be sure to set these up before the page is inserted on the hash
* list. As soon as the page is placed on the list some other
* thread might get confused and wonder how this page could
* possibly hash to this list.
*/
/*
* record if this page is on a swap vnode
*/
PP_SETSWAP(pp);
/*
* If this page is already hashed in, fail this attempt to add it.
*/
return (0);
}
}
/*
* Add the page to the vnode's list of pages
*/
else
return (1);
}
/*
* Add page `pp' to both the hash and vp chains for [vp, offset].
*
* Returns 1 on success and 0 on failure.
* If hold is passed in, it is not dropped.
*/
int
{
int rc;
"page_hashin:pp %p vp %p offset %llx",
else {
}
if (rc == 0)
return (rc);
}
/*
* Remove page ``pp'' from the hash and vp chains and remove vp association.
* All mutexes must be held
*/
static void
{
/*
* First, take pp off of its hash chain.
*/
for (;;) {
break;
panic("page_do_hashout");
/*NOTREACHED*/
}
}
/*
* Now remove it from its associated vnode.
*/
PP_CLRSWAP(pp);
}
/*
* Remove page ``pp'' from the hash and vp chains and remove vp association.
*
* When `phm' is non-NULL it contains the address of the mutex protecting the
* hash list pp is on. It is not dropped.
*/
void
{
/* Kernel probe */
/*
*
*/
}
/*
* grab page vnode mutex and remove it...
*/
/*
* If the page was retired, update the pages_retired
* total and clear the page flag
*/
if (page_isretired(pp)) {
}
/*
* Wake up processes waiting for this page. The page's
* identity has been changed, and is probably not the
* desired page any longer.
*/
}
/*
* Add the page to the front of a linked list of pages
* using the p_next & p_prev pointers for the list.
* The caller is responsible for protecting the list pointers.
*/
void
{
}
/*
* Common code for page_add() and mach_page_add()
*/
void
{
} else {
}
}
/*
* Remove this page from a linked list of pages
* using the p_next & p_prev pointers for the list.
*
* The caller is responsible for protecting the list pointers.
*/
void
{
panic("page_sub: bad arg(s): pp %p, *ppp %p",
/*NOTREACHED*/
}
}
/*
* Common code for page_sub() and mach_page_sub()
*/
void
{
else {
}
}
/*
* Break page list cppp into two lists with npages in the first list.
* The tail is returned in nppp.
*/
void
{
long n = 0;
return;
}
if (npages == 0) {
return;
}
}
/* Fix head and tail of new lists */
/* second list empty */
} else {
}
}
/*
* Concatenate page list nppp onto the end of list ppp.
*/
void
{
return;
}
return;
}
}
/*
* return the next page in the page list
*/
page_t *
{
}
/*
* Add the page to the front of the linked list of pages
*
* The caller is responsible for protecting the lists.
*/
void
{
} else {
}
}
/*
* Remove this page from the linked list of pages
*
* The caller is responsible for protecting the lists.
*/
void
{
panic("page_vpsub: bad arg(s): pp %p, *ppp %p",
/*NOTREACHED*/
}
else {
}
}
/*
* Lock a physical page into memory "long term". Used to support "lock
* in memory" functions. Accepts the page to be locked, and a cow variable
* to indicate whether a the lock will travel to the new page during
* a potential copy-on-write.
*/
int
int cow, /* cow lock */
int kernel) /* must succeed -- ignore checking */
{
int r = 0; /* result -- assume failure */
/*
* Acquire the "freemem_lock" for availrmem.
*/
if (cow) {
if ((availrmem > pages_pp_maximum) &&
availrmem--;
pages_locked++;
r = 1;
"COW lock limit reached on pfn 0x%lx",
page_pptonum(pp));
}
} else
} else {
r = 1;
"reached on pfn 0x%lx",
page_pptonum(pp));
}
}
} else {
if (kernel) {
/* availrmem accounting done by caller */
r = 1;
} else {
if (availrmem > pages_pp_maximum) {
availrmem--;
pages_locked++;
r = 1;
}
}
}
}
return (r);
}
/*
* Decommit a lock on a physical page frame. Account for cow locks if
* appropriate.
*/
void
int cow, /* expect cow lock */
int kernel) /* this was a kernel lock */
{
/*
* Acquire the "freemem_lock" for availrmem.
* If cowcnt or lcknt is already 0 do nothing; i.e., we
* could be called to unlock even if nothing is locked. This could
* happen if locked file pages were truncated (removing the lock)
* and the file was grown again and new pages faulted in; the new
* pages are unlocked but the segment still thinks they're locked.
*/
if (cow) {
availrmem++;
pages_locked--;
}
} else {
if (!kernel) {
availrmem++;
pages_locked--;
}
}
}
}
/*
* This routine reserves availrmem for npages;
* flags: KM_NOSLEEP or KM_SLEEP
* returns 1 on success or 0 on failure
*/
int
{
if (flags & KM_NOSLEEP) {
return (0);
}
kmem_reap();
}
return (1);
}
/*
* This routine unreserves availrmem for npages;
*/
void
{
}
/*
* See Statement at the beginning of segvn_lockop() regarding
* the way we handle cowcnts and lckcnts.
*
* Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage
* that breaks COW has PROT_WRITE.
*
* Note that, we may also break COW in case we are softlocking
* on read access during physio;
* in this softlock case, the vpage may not have PROT_WRITE.
* So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp'
* if the vpage doesn't have PROT_WRITE.
*
* This routine is never called if we are stealing a page
* in anon_private.
*
* The caller subtracted from availrmem for read only mapping.
* if lckcnt is 1 increment availrmem.
*/
void
{
int payback = 0;
/* Don't use claim if nothing is locked (see page_pp_unlock above) */
if (write_perm) {
} else {
/*
* We didn't need availrmem decremented if p_lckcnt on
* original page is 1. Here, we are unlocking
* read-only copy belonging to original page and
* are locking a copy belonging to new page.
*/
payback = 1;
}
}
if (payback) {
availrmem++;
}
}
/*
* Simple claim adjust functions -- used to support changes in
* claims due to changes in access permissions. Used by segvn_setprot().
*/
int
{
int r = 0; /* result */
r = 1;
"COW lock limit reached on pfn 0x%lx",
page_pptonum(pp));
}
}
} else {
if ((availrmem > pages_pp_maximum) &&
--availrmem;
r = 1;
"COW lock limit reached on pfn 0x%lx",
page_pptonum(pp));
}
} else
}
return (r);
}
int
{
int r = 0;
r = 1;
/*
* for availrmem
*/
availrmem++;
"Page lock limit reached on pfn 0x%lx",
page_pptonum(pp));
}
}
} else {
r = 1;
}
return (r);
}
int
{
return (0);
}
lckpgs++;
}
if (lckpgs != 0) {
pages_claimed += lckpgs;
} else {
return (0);
}
}
}
return (1);
}
int
{
return (0);
}
ulckpgs++;
}
if (ulckpgs != 0) {
pages_claimed -= ulckpgs;
}
}
return (1);
}
page_t *
{
}
/*
* Acquire the appropriate lock on the page.
*/
goto retry;
continue;
}
goto retry;
}
return (pp);
}
page_t *
{
}
/*
* Acquire the appropriate lock on the page.
*/
goto retry;
continue;
}
goto retry;
}
return (pp);
}
/*
* This routine is like page_numtopp, but will only return page structs
* for pages which are ok for loading into hardware using the page struct.
*/
page_t *
{
}
/*
* Try to acquire the appropriate lock on the page.
*/
else {
else {
goto retry;
}
}
}
}
return (pp);
}
/*
* Returns a count of dirty pages that are in the process
* of being written out. If 'cleanit' is set, try to push the page.
*/
{
do {
/*
* A page is a candidate for syncing if it is:
*
* (a) On neither the freelist nor the cachelist
* (b) Hashed onto a vnode
* (c) Not a kernel page
* (d) Dirty
* (e) Not part of a swapfile
* (f) a page which belongs to a real vnode; eg has a non-null
* v_vfsp pointer.
* (g) Backed by a filesystem which doesn't have a
* stubbed-out sync operation
*/
nppbusy++;
if (!cleanit)
continue;
continue;
!(hat_pagesync(pp,
continue;
}
}
return (nppbusy);
}
void page_invalidate_pages(void);
/*
* callback handler to vm sub-system
*
* callers make sure no recursive entries to this func.
*/
/*ARGSUSED*/
{
if (code == CB_CODE_CPR_CHKPT)
return (B_TRUE);
}
/*
* Invalidate all pages of the system.
* It shouldn't be called until all user page activities are all stopped.
*/
void
{
int retry = 0;
const int MAXRETRIES = 4;
#if defined(__sparc)
extern struct vnode prom_ppages;
#endif /* __sparc */
top:
/*
* Flush dirty pages and destory the clean ones.
*/
nbusypages = 0;
do {
int mod;
/*
* skip the page if it has no vnode or the page associated
* with the kernel vnode or prom allocated kernel mem.
*/
#if defined(__sparc)
vp == &prom_ppages)
#else /* x86 doesn't have prom or prom_ppage */
#endif /* __sparc */
continue;
/*
* skip the page which is already free invalidated.
*/
continue;
/*
* skip pages that are already locked or can't be "exclusively"
* locked or are already free. After we lock the page, check
* the free and age bits again to be sure it's not destroied
* yet.
* To achieve max. parallelization, we use page_trylock instead
* of page_lock so that we don't get block on individual pages
* while we have thousands of other pages to process.
*/
nbusypages++;
continue;
} else {
}
continue;
}
/*
* Is this page involved in some I/O? shared?
*
* The page_struct_lock need not be acquired to
* examine these fields since the page has an
* "exclusive" lock.
*/
continue;
}
panic("vp->v_type == VCHR");
/*NOTREACHED*/
}
if (!page_try_demote_pages(pp)) {
continue;
}
/*
* Check the modified bit. Leave the bits alone in hardware
* (they will be modified if we do the putpage).
*/
& P_MOD);
if (mod) {
/*
* Hold the vnode before releasing the page lock
* to prevent it from being freed and re-used by
* some other thread.
*/
/*
* No error return is checked here. Callers such as
* cpr deals with the dirty pages at the dump time
* if this putpage fails.
*/
kcred);
} else {
page_destroy(pp, 0);
}
delay(1);
goto top;
}
}
/*
* Replace the page "old" with the page "new" on the page hash and vnode lists
*
* the replacemnt must be done in place, ie the equivalent sequence:
*
* vp = old->p_vnode;
* off = old->p_offset;
* page_do_hashout(old)
* page_do_hashin(new, vp, off)
*
* doesn't work, since
* 1) if old is the only page on the vnode, the v_pages list has a window
* where it looks empty. This will break file system assumptions.
* and
* 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list.
*/
static void
{
/*
* First find old page on the page hash list
*/
for (;;) {
break;
panic("page_do_hashout");
/*NOTREACHED*/
}
}
/*
* update new and replace old with new on the page hash list
*/
/*
* replace old with new on the vnode's page list
*/
} else {
}
/*
* clear out the old page
*/
/*
* Wake up processes waiting for this page. The page's
* identity has been changed, and is probably not the
* desired page any longer.
*/
}
/*
* This function moves the identity of page "pp_old" to page "pp_new".
* Both pages must be locked on entry. "pp_new" is free, has no identity,
* and need not be hashed out from anywhere.
*/
void
{
/*
* Rehash two pages
*/
/*
* hashout then hashin while holding the mutexes
*/
/*
* The page_struct_lock need not be acquired for lckcnt and
* cowcnt since the page has an "exclusive" lock.
*/
/* The following comment preserved from page_flip(). */
/* XXX - Do we need to protect fsdata? */
}
/*
* Helper routine used to lock all remaining members of a
* large page. The caller is responsible for passing in a locked
* pp. If pp is a large page, then it succeeds in locking all the
* remaining constituent pages or it returns with only the
* original page locked.
*
* Returns 1 on success, 0 on failure.
*
* If success is returned this routine gurantees p_szc for all constituent
* pages of a large page pp belongs to can't change. To achieve this we
* recheck szc of pp after locking all constituent pages and retry if szc
* changed (it could only decrease). Since hat_page_demote() needs an EXCL
* lock on one of constituent pages it can't be running after all constituent
* pages are locked. hat_page_demote() with a lock on a constituent page
* outside of this large page (i.e. pp belonged to a larger large page) is
* already done with all constituent pages of pp since the root's p_szc is
* changed last. Thefore no need to synchronize with hat_page_demote() that
* locked a constituent page outside of pp's current large page.
*/
#ifdef DEBUG
uint32_t gpg_trylock_mtbf = 0;
#endif
int
{
#ifdef DEBUG
return (0);
}
#endif
return (0);
}
if (pszc == 0) {
return (1);
}
for (j = 1; j < i; j++, tpp++) {
}
return (0);
}
}
}
goto retry;
}
return (1);
}
void
{
}
}
/*
* returns
* 0 : on success and *nrelocp is number of relocated PAGESIZE pages
* ERANGE : this is not a base page
* ENOMEM : failure to obtain replacement pages
* EAGAIN : OBP has not yet completed its boot-time handoff to the kernel
*
* Return with all constituent members of target and replacement
* SE_EXCL locked. It is the callers responsibility to drop the
* locks.
*/
int
int grouplock,
{
#ifdef DEBUG
#endif /* DEBUG */
int repl_contig = 0;
*nrelocp = 0;
#if defined(__sparc)
/*
* We need to wait till OBP has completed
* its boot-time handoff of its resources to the kernel
* before we allow page relocation
*/
if (page_relocate_ready == 0) {
return (EAGAIN);
}
#endif
/*
* If this is not a base page,
* just return with 0x0 pages relocated.
*/
return (ERANGE);
}
return (ERANGE);
}
repl_contig = 1;
}
/*
* We must lock all members of this large page or we cannot
* relocate any part of it.
*/
return (EBUSY);
}
/*
* reread szc it could have been decreased before
* group_page_trylock() was done.
*/
if (!page_create_wait(dofree, 0)) {
if (grouplock != 0) {
}
return (ENOMEM);
}
/*
* seg kmem pages require that the target and replacement
* page be the same pagesize.
*/
if (grouplock != 0) {
}
return (ENOMEM);
}
}
#ifdef DEBUG
else {
}
#endif /* DEBUG */
#if defined(__sparc)
/*
* Let hat_page_relocate() complete the relocation if it's kernel page
*/
*replacement = repl;
if (grouplock != 0) {
}
if (dofree) {
*replacement = NULL;
}
return (EAGAIN);
}
return (0);
}
#else
#if defined(lint)
#endif
#endif
#ifdef DEBUG
first_repl = repl;
#endif /* DEBUG */
for (i = 0; i < npgs; i++) {
ASSERT(repl_contig == 0 ||
/*
* Copy the page contents and attributes then
* relocate the page in the page hash.
*/
/*
* Now clear the props on targ, after the
* page_relocate_hash(), they no longer
* have any meaning.
*/
targ++;
if (repl_contig != 0) {
repl++;
} else {
}
}
/* assert that we have come full circle with repl */
if (*replacement == NULL) {
*replacement = repl;
}
return (0);
}
/*
* On success returns 0 and *nrelocp the number of PAGESIZE pages relocated.
*/
int
int grouplock,
int freetarget,
{
/* do_page_relocate returns 0 on success or errno value */
if (ret != 0 || freetarget == 0) {
return (ret);
}
if (*nrelocp == 1) {
} else {
do {
npgs--;
page_list_add_pages(*target, 0);
}
return (ret);
}
/*
* it is up to the caller to deal with pcf accounting.
*/
void
{
/*
* pp_targ is a linked list.
*/
PP_SETFREE(pp);
PP_SETAGED(pp);
} else {
do {
page_list_add_pages(pp, 0);
}
}
}
/*
* Relocate target to non-relocatable replacement page.
*/
int
{
int result;
do {
}
if (result == 0) {
*replacement = rpp;
panic("page_relocate_cage: partial relocation");
}
return (result);
}
/*
* Release the page lock on a page, place on cachelist
* tail if no longer mapped. Caller can let us know if
* the page is known to be clean.
*/
int
{
int status;
!hat_page_is_mapped(pp)) {
/*
* If page is modified, unlock it
*
* (p_nrm & P_MOD) bit has the latest stuff because:
* (1) We found that this page doesn't have any mappings
* _after_ holding SE_EXCL and
* (2) We didn't drop SE_EXCL lock after the check in (1)
*/
} else {
/*LINTED: constant in conditional context*/
}
} else {
}
return (status);
}
int
{
return (1);
}
return (1);
}
/*
* Adjust rootpp if passed in is not the base
* constituent page.
*/
}
/*
* We can't demote kernel pages since we can't hat_unload()
* the mappings.
*/
return (0);
/*
* Attempt to lock all constituent pages except the page passed
* in since it's already locked.
*/
break;
}
/*
* If we failed to lock them all then unlock what we have locked
* so far and bail.
*/
if (i < npgs) {
while (i-- > 0) {
}
return (0);
}
/*
* XXX probably p_szc clearing and page unlocking can be done within
* one loop but since this is rare code we can play very safe.
*/
}
/*
* Unlock all pages except the page passed in.
*/
}
return (1);
}
/*
* Called by page_free() and page_destroy() to demote the page size code
* (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero
* p_szc on free list, neither can we just clear p_szc of a single page_t
* within a large page since it will break other code that relies on p_szc
* being the same for all page_t's of a large page). Anonymous pages should
* never end up here because anon_map_getpages() cannot deal with p_szc
* changes after a single constituent page is locked. While anonymous or
* kernel large pages are demoted or freed the entire large page at a time
* with all constituent pages locked EXCL for the file system pages we
* have to be able to demote a large page (i.e. decrease all constituent pages
* p_szc) with only just an EXCL lock on one of constituent pages. The reason
* we can easily deal with anonymous page demotion the entire large page at a
* time is that those operation originate at address space level and concern
* the entire large page region with actual demotion only done when pages are
* not shared with any other processes (therefore we can always get EXCL lock
* on all anonymous constituent pages after clearing segment page
* cache). However file system pages can be truncated or invalidated at a
* PAGESIZE level from the file system side and end up in page_free() or
* page_destroy() (we also allow only part of the large page to be SOFTLOCKed
* and therfore pageout should be able to demote a large page by EXCL locking
* any constituent page that is not under SOFTLOCK). In those cases we cannot
* rely on being able to lock EXCL all constituent pages.
*
* To prevent szc changes on file system pages one has to lock all constituent
* pages at least SHARED (or call page_szc_lock()). The only subsystem that
* doesn't rely on locking all constituent pages (or using page_szc_lock()) to
* prevent szc changes is hat layer that uses its own page level mlist
* locks. hat assumes that szc doesn't change after mlist lock for a page is
* taken. Therefore we need to change szc under hat level locks if we only
* have an EXCL lock on a single constituent page and hat still references any
* of constituent pages. (Note we can't "ignore" hat layer by simply
* hat_pageunload() all constituent pages without having EXCL locks on all of
* constituent pages). We use hat_page_demote() call to safely demote szc of
* all constituent pages under hat locks when we only have an EXCL lock on one
* of constituent pages.
*
* This routine calls page_szc_lock() before calling hat_page_demote() to
* allow segvn in one special case not to lock all constituent pages SHARED
* before calling hat_memload_array() that relies on p_szc not changeing even
* before hat level mlist lock is taken. In that case segvn uses
* page_szc_lock() to prevent hat_page_demote() changeing p_szc values.
*
* Anonymous or kernel page demotion still has to lock all pages exclusively
* and do hat_pageunload() on all constituent pages before demoting the page
* therefore there's no need for anonymous or kernel page demotion to use
* hat_page_demote() mechanism.
*
* hat_page_demote() removes all large mappings that map pp and then decreases
* p_szc starting from the last constituent page of the large page. By working
* from the tail of a large page in pfn decreasing order allows one looking at
* the root page to know that hat_page_demote() is done for root's szc area.
* e.g. if a root page has szc 1 one knows it only has to lock all constituent
* pages within szc 1 area to prevent szc changes because hat_page_demote()
* that started on this page when it had szc > 1 is done for this szc 1 area.
*
* We are guranteed that all constituent pages of pp's large page belong to
* the same vnode with the consecutive offsets increasing in the direction of
* the pfn i.e. the identity of constituent pages can't change until their
* p_szc is decreased. Therefore it's safe for hat_page_demote() to remove
* large mappings to pp even though we don't lock any constituent page except
* pp (i.e. we won't unload e.g. kernel locked page).
*/
static void
{
}
}
/*
* Page retire operation.
*
* page_retire()
* Attempt to retire (throw away) page pp. We cannot do this if
* the page is dirty; if the page is clean, we can try. We return 0 on
* success, -1 on failure. This routine should be invoked by the platform's
* memory error detection code.
*
* pages_retired_limit_exceeded()
* We set a limit on the number of pages which may be retired. This
* is set to a percentage of total physical memory. This limit is
* enforced here.
*/
static pgcnt_t retired_pgcnt = 0;
/*
* routines to update the count of retired pages
*/
static void
{
}
static void
{
ASSERT(retired_pgcnt > 0);
}
static int
{
/*
* If the percentage is zero or is not set correctly,
* return TRUE so that pages are not retired.
*/
if (max_pages_retired_bps <= 0 ||
max_pages_retired_bps >= 10000)
return (1);
/*
* Calculate the maximum number of pages allowed to
* be retired as a percentage of total physical memory
* (Remember that we are using basis points, hence the 10000.)
*/
/*
* return 'TRUE' if we have already retired more
* than the legal limit
*/
return (retired_pgcnt >= retired_max);
}
#define PAGE_RETIRE_SELOCK 0
#define PAGE_RETIRE_NORECLAIM 1
#define PAGE_RETIRE_LOCKED 2
#define PAGE_RETIRE_COW 3
#define PAGE_RETIRE_DIRTY 4
#define PAGE_RETIRE_LPAGE 5
#define PAGE_RETIRE_SUCCESS 6
#define PAGE_RETIRE_LIMIT 7
#define PAGE_RETIRE_NCODES 8
typedef struct page_retire_op {
int pr_count;
short pr_unlock;
short pr_retval;
char *pr_message;
{ 0, 0, -1, "cannot lock page" },
{ 0, 0, -1, "cannot reclaim cached page" },
{ 0, 1, -1, "page is locked" },
{ 0, 1, -1, "copy-on-write page" },
{ 0, 1, -1, "page is dirty" },
{ 0, 1, -1, "cannot demote large page" },
{ 0, 0, 0, "page successfully retired" },
{ 0, 0, -1, "excess pages retired already" },
};
static int
{
if (page_retire_messages > 1) {
printf("page_retire(%p) pfn 0x%lx %s: %s\n",
prop->pr_message);
}
}
int
{
/*
* DR operations change the association between a page_t
* and the physical page it represents. Check if the
* page is still bad.
*/
if (!page_isfaulty(pp)) {
}
/*
* We set the flag here so that even if we fail due
* to exceeding the limit for retired pages, the
* page will still be checked and either cleared
* or retired in page_free().
*/
if (flag == PAGE_IS_TOXIC) {
if (page_retire_messages) {
" page 0x%08x.%08x",
}
} else { /* PAGE_IS_FAILING */
if (pages_retired_limit_exceeded()) {
/*
* Return as we have already exceeded the
* maximum number of pages allowed to be
* retired
*/
}
if (page_retire_messages) {
"page 0x%08x.%08x",
}
}
/*
* If this is a large page we first try and demote it
* to PAGESIZE pages and then dispose of the toxic page.
* code handle it later since this is a mapped page.
* Note that free large pages can always be demoted.
*
*/
(void) page_demote_free_pages(pp);
else
(void) page_try_demote_pages(pp);
}
/*LINTED: constant in conditional context*/
}
/*
* Unload all translations to this page. No new translations
* can be created while we hold the exclusive lock on the page.
*/
/*LINTED: constant in conditional context*/
}
/*
* Mark any existing pages for migration in the given range
*/
void
{
pgcnt_t i;
/*
* Don't do anything if don't need to do lgroup optimizations
* on this system
*/
if (!lgrp_optimizations())
return;
/*
* Align address and length to (potentially large) page boundary
*/
if (rflag)
/*
* Allocate page array to accomodate largest page size
*/
/*
* Do one (large) page at a time
*/
/*
* Lookup (root) page for vnode and offset corresponding to
* this virtual address
* Try anonmap first since there may be copy-on-write
* pages, but initialize vnode pointer and offset using
* vnode arguments just in case there isn't an amp.
*/
if (amp) {
if (ap)
}
if (curvp)
/*
* If there isn't a page at this virtual address,
* skip to next page
*/
continue;
}
/*
* Figure out which lgroup this page is in for kstats
*/
/*
* Get page size, and round up and skip to next page boundary
* if unaligned address
*/
continue;
}
/*
* Upgrade to exclusive lock on page
*/
if (!page_tryupgrade(pp)) {
continue;
}
/*
* Remember pages locked exclusively and how many
*/
nlocked = 1;
/*
* Lock constituent pages if this is large page
*/
if (pages > 1) {
/*
* Lock all constituents except root page, since it
* should be locked already.
*/
for (i = 1; i < pages; i++) {
break;
}
/*
* hat_page_demote() raced in with us.
*/
break;
}
nlocked++;
}
}
/*
* If all constituent pages couldn't be locked,
* unlock pages locked so far and skip to next page.
*/
for (i = 0; i < nlocked; i++)
page_unlock(ppa[i]);
continue;
}
/*
* hat_page_demote() can no longer happen
* since last cons page had the right p_szc after
* all cons pages were locked. all cons pages
* should now have the same p_szc.
*/
/*
* All constituent pages locked successfully, so mark
* large page for migration and unload the mappings of
* constituent pages, so a fault will occur on any part of the
* large page
*/
PP_SETMIGRATE(ppa[0]);
for (i = 0; i < nlocked; i++) {
}
}
}
/*
* Migrate any pages that have been marked for migration in the given range
*/
void
{
spgcnt_t i;
while (npages > 0) {
/*
* Check to see whether this page is marked for migration
*
* Assume that root page of large page is marked for
* migration and none of the other constituent pages
* are marked. This really simplifies clearing the
* migrate bit by not having to clear it from each
* constituent page.
*
* note we don't want to relocate an entire large page if
* someone is only using one subpage.
*/
break;
/*
* Is it marked for migration?
*/
if (!PP_ISMIGRATE(pp))
goto next;
/*
* Determine lgroups that page is being migrated between
*/
break;
}
/*
* Check to see whether we are trying to migrate page to lgroup
* where it is allocated already
*/
goto next;
}
/*
* Need to get exclusive lock's to migrate
*/
for (i = 0; i < page_cnt; i++) {
break;
}
if (!page_tryupgrade(ppa[i])) {
page_cnt);
break;
}
}
if (i != page_cnt) {
while (--i != -1) {
page_downgrade(ppa[i]);
}
goto next;
}
for (i = 0; i < page_cnt; i++) {
page_downgrade(ppa[i]);
}
page_cnt);
goto next;
}
/*
* Clear migrate bit and relocate page
*/
panic("page_migrate: page_relocate failed");
}
/*
* Keep stats for number of pages migrated from and to
* each lgroup
*/
/*
* update the page_t array we were passed in and
* unlink constituent pages of a large page.
*/
}
next:
}
}
/*
* initialize the vnode for retired pages
*/
static void
page_retired_init(void)
{
}
/* ARGSUSED */
static void
{
panic("retired_dispose invoked");
}
/* ARGSUSED */
static void
{}
void
page_unretire_pages(void)
{
rmem = 0;
for (;;) {
/*
* We do this in 2 steps:
*
* 1. We walk the retired pages list and collect a list of
* pages that have the toxic field cleared.
*
* 2. We iterate through the page list and unretire each one.
*
* We have to do it in two steps on account of the mutexes that
* we need to acquire.
*/
vp = &retired_ppages;
break;
}
i = 0;
do {
/*
* DR operations change the association between a page_t
* and the physical page it represents. Check if the
* page is still bad. If not, unretire it.
*/
if (!page_isfaulty(pp))
npages = i;
for (i = 0; i < npages; i++) {
/*
* Need to upgrade the shared lock to an exclusive
* lock in order to hash out the page.
*
* The page could have been retired but the page lock
* may not have been downgraded yet. If so, skip this
* page. page_free() will call this function after the
* lock is downgraded.
*/
continue;
/*
* Both page_free() and DR call this function. They
* can potentially call this function at the same
* time and race with each other.
*/
continue;
}
"unretiring retired page 0x%08x.%08x",
/*
* When a page is removed from the retired pages vnode,
* its toxic field is also cleared. So, we do not have
* to do that seperately here.
*/
/*
* This is a good page. So, free it.
*/
rmem++;
}
/*
* If the rpages array was filled up, then there could be more
* retired pages that are not faulty. We need to iterate
* again and unretire them. Otherwise, we are done.
*/
if (npages < UNRETIRE_PAGES)
break;
}
}
ulong_t mem_waiters = 0;
#define MAX_DELAY 0x1ff
/*
* Check if enough memory is available to proceed.
* Depending on system configuration and how much memory is
* reserved for swap we need to check against two variables.
* e.g. on systems with little physical swap availrmem can be
* more reliable indicator of how much memory is available.
* On systems with large phys swap freemem can be better indicator.
* If freemem drops below threshold level don't return an error
* immediately but wake up pageout to free memory and block.
* This is done number of times. If pageout is not able to free
* memory within certain time return an error.
* The same applies for availrmem but kmem_reap is used to
* free memory.
*/
int
{
#if defined(__i386)
return (1);
#else
return (1);
#endif
return (0);
}
}
if (count == 0) {
return (0);
}
kmem_reap();
return (0);
}
}
if (count == 0)
return (0);
#if defined(__i386)
return (0);
#endif
return (1);
}
/*
* Search the memory segments to locate the desired page. Within a
* segment, pages increase linearly with one page structure per
* physical page frame (size PAGESIZE). The search begins
* with the segment that was accessed last, to take advantage of locality.
* If the hint misses, we start from the beginning of the sorted memseg list
*/
/*
* Some data structures for pfn to pp lookup.
*/
page_t *
{
/*
* XXX - Since page_numtopp_nolock is called in many places where
* the search fails more than it succeeds. It maybe worthwhile
* to put a check for pf_is_memory or a pfnum <= max_pfn (set at
* boot time).
*
* if (!pf_is_memory(pfnum) || (pfnum > max_pfn))
* return (NULL);
*/
/* Try last winner first */
}
/* Else Try hash */
}
/* Else Brute force */
}
}
}
struct memseg *
{
/* Try hash */
return (seg);
}
/* Else Brute force */
return (seg);
}
}
}
/*
* Given a page and a count return the page struct that is
* n structs away from the current one in the global page
* list.
*
* This function wraps to the first page upon
* reaching the end of the memseg list.
*/
page_t *
{
break;
}
/* Memory delete got in, return something valid. */
/* TODO: fix me. */
}
}
/* check for wraparound - possible if n is large */
}
return (ppn);
}
/*
* Initialize for a loop using page_next_scan_large().
*/
page_t *
page_next_scan_init(void **cookie)
{
}
/*
* Return the next page in a scan of page_t's, assuming we want
* to skip over sub-pages within larger page sizes.
*
* The cookie is used to keep track of the current memseg.
*/
page_t *
ulong_t *n,
void **cookie)
{
/*
* get the count of page_t's to skip based on the page size
*/
cnt = 1;
} else {
}
*n += cnt;
/*
* Catch if we went past the end of the current memory segment. If so,
* just move to the next segment with pages.
*/
do {
}
return (new_pp);
}
/*
* Returns next page in list. Note: this function wraps
* to the first page in the list upon reaching the end
* of the list. Callers should be aware of this fact.
*/
/* We should change this be a #define */
page_t *
{
}
/*
* Special for routines processing an array of page_t.
*/
page_t *
{
return (pp+n);
}
page_t *
{
}
/*
* This routine is called at boot with the initial memory configuration
* and when memory is added or removed.
*/
void
{
int i;
/*
* Clear memseg_hash array.
* with normal operation, the hash rebuild must be able to run
* concurrently with page_numtopp_nolock(). To support this
* functionality, assignments to memseg_hash array members must
* be done atomically.
*
* NOTE: bzero() does not currently guarantee this for kernel
* threads, and cannot be used here.
*/
for (i = 0; i < N_MEM_SLOTS; i++)
memseg_hash[i] = NULL;
/*
* Physmax is the last valid pfn.
*/
do {
if (index >= N_MEM_SLOTS)
}
cur += mhash_per_slot;
index++;
}
}
/*
* Return the pagenum for the pp
*/
{
}
/*
* interface to the referenced and modified etc bits
* in the PSM part of the page struct
* when no locking is desired.
*/
void
{
}
void
{
}
/*
* The following functions is called from free_vp_pages()
* for an inexact estimate of a newly free'd page...
*/
{
return (hat_page_getshare(pp));
}
/*
* The following functions are used in handling memory
* errors.
*/
int
{
}
int
{
}
int
{
}
int
{
}
void
{
}
}
void
{
/*
* We don't need to worry about atomicity on the
* p_toxic flag here as this is only called from
* page_free() while holding an exclusive lock on
* the page
*/
}
void
{
}
}
int
{
}
/*
* The following four functions are called from /proc code
* for the /proc/<pid>/xmap interface.
*/
int
{
}
int
{
}
int
{
}
int
{
}