vm_pagelist.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
/*
* Portions of this source code were derived from Berkeley 4.3 BSD
* under license from the Regents of the University of California.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* This file contains common functions to access and manage the page lists.
* Many of these routines originated from platform dependent modules
* a platform independent manner.
*
*/
#include <sys/sysmacros.h>
#include <vm/seg_kmem.h>
#include <sys/mem_config.h>
#include <sys/mem_cage.h>
extern uint_t vac_colors;
/*
* number of page colors equivalent to reqested color in page_get routines.
* If set, keeps large pages intact longer and keeps MPO allocation
* from the local mnode in favor of acquiring the 'correct' page color from
* a demoted large page or from a remote mnode.
*/
int colorequiv;
/*
* if set, specifies the percentage of large pages that are free from within
* a large page region before attempting to lock those pages for
* page_get_contig_pages processing.
*
* Should be turned on when kpr is available when page_trylock_contig_pages
* can be more selective.
*/
int ptcpthreshold;
/*
* Limit page get contig page search based on failure cnts in pgcpfailcnt[].
* use slot 0 (base page size unused) to enable or disable limiting search.
* Enabled by default.
*/
int pgcpfailcnt[MMU_PAGE_SIZES];
int pgcplimitsearch = 1;
#ifdef VM_STATS
struct vmm_vmstats_str vmm_vmstats;
#endif /* VM_STATS */
#if defined(__sparc)
#define LPGCREATE 0
#else
/* enable page_get_contig_pages */
#define LPGCREATE 1
#endif
int pg_contig_disable;
int pg_lpgcreate_nocage = LPGCREATE;
/*
* page_freelist_fill pfn flag to signify no hi pfn requirement.
*/
#define PFNNULL 0
/* Flags involved in promotion and demotion routines */
/*
* Flag for page_demote to be used with PC_FREE to denote that we don't care
* what the color is as the color parameter to the function is ignored.
*/
#define PC_NO_COLOR (-1)
/*
* page counters candidates info
* See page_ctrs_cands comment below for more details.
* fields are as follows:
* pcc_pages_free: # pages which freelist coalesce can create
* pcc_color_free_len: number of elements in pcc_color_free array
* pcc_color_free: pointer to page free counts per color
*/
typedef struct pcc_info {
int pcc_color_free_len;
} pcc_info_t;
/*
* On big machines it can take a long time to check page_counters
* arrays. page_ctrs_cands is a summary array whose elements are a dynamically
* updated sum of all elements of the corresponding page_counters arrays.
* page_freelist_coalesce() searches page_counters only if an appropriate
* element of page_ctrs_cands array is greater than 0.
*
* An extra dimension is used for page_ctrs_cands to spread the elements
* over a few e$ cache lines to avoid serialization during the array
* updates.
*/
/*
* Return in val the total number of free pages which can be created
* for the given mnode (m) and region size (r)
*/
#define PGCTRS_CANDS_GETVALUE(m, r, val) { \
int i; \
val = 0; \
for (i = 0; i < NPC_MUTEX; i++) { \
} \
}
/*
* Return in val the total number of free pages which can be created
* for the given mnode (m), region size (r), and color (c)
*/
#define PGCTRS_CANDS_GETVALUECOLOR(m, r, c, val) { \
int i; \
val = 0; \
for (i = 0; i < NPC_MUTEX; i++) { \
} \
}
/*
* We can only allow a single thread to update a counter within the physical
* range of the largest supported page size. That is the finest granularity
* possible since the counter values are dependent on each other
* as you move accross region sizes. PP_CTR_LOCK_INDX is used to determine the
* ctr_mutex lock index for a particular physical range.
*/
#define PP_CTR_LOCK_INDX(pp) \
/*
* Local functions prototypes.
*/
void page_ctr_add(page_t *, int);
void page_ctr_add_internal(int, page_t *, int);
void page_ctr_sub(page_t *, int);
void page_freelist_lock(int);
void page_freelist_unlock(int);
#define PNUM_SHIFT(szc) \
/*
* The page_counters array below is used to keep track of free contiguous
* physical memory. A hw_page_map_t will be allocated per mnode per szc.
* This contains an array of counters, the size of the array, a shift value
* used to convert a pagenum into a counter array index or vice versa, as
* well as a cache of the last successful index to be promoted to a larger
* page size. As an optimization, we keep track of the last successful index
* to be promoted per page color for the given size region, and this is
* allocated dynamically based upon the number of colors for a given
* region size.
*
* Conceptually, the page counters are represented as:
*
* page_counters[region_size][mnode]
*
* region_size: size code of a candidate larger page made up
* of contiguous free smaller pages.
*
* page_counters[region_size][mnode].hpm_counters[index]:
* represents how many (region_size - 1) pages either
* exist or can be created within the given index range.
*
* Let's look at a sparc example:
* If we want to create a free 512k page, we look at region_size 2
* for the mnode we want. We calculate the index and look at a specific
* hpm_counters location. If we see 8 (FULL_REGION_CNT on sparc) at
* this location, it means that 8 64k pages either exist or can be created
* from 8K pages in order to make a single free 512k page at the given
* index. Note that when a region is full, it will contribute to the
* counts in the region above it. Thus we will not know what page
* size the free pages will be which can be promoted to this new free
* page unless we look at all regions below the current region.
*/
/*
* Note: hpmctr_t is defined in platform vm_dep.h
* hw_page_map_t contains all the information needed for the page_counters
* logic. The fields are as follows:
*
* hpm_counters: dynamically allocated array to hold counter data
* hpm_entries: entries in hpm_counters
* hpm_base: PFN mapped to counter index 0
* hpm_color_current_len: # of elements in hpm_color_current "array" below
* hpm_color_current: last index in counter array for this color at
* which we successfully created a large page
*/
typedef struct hw_page_map {
int hpm_shift;
/*
* Element zero is not used, but is allocated for convenience.
*/
/*
* The following macros are convenient ways to get access to the individual
* elements of the page_counters arrays. They can be used on both
* the left side and right side of equations.
*/
/*
* Protects the hpm_counters and hpm_color_current memory from changing while
* looking at page counters information.
* Grab the write lock to modify what these fields point at.
* Grab the read lock to prevent any pointers from changing.
* The write lock can not be held during memory allocation due to a possible
* recursion deadlock with trying to grab the read lock while the
* write lock is already held.
*/
/*
* page size to page size code
*/
int
{
int i = 0;
while (hw_page_array[i].hp_size) {
return (i);
i++;
}
return (-1);
}
/*
* page size to page size code for user supported page sizes
*/
int
{
if (szc != -1)
return (SZC_2_USERSZC(szc));
return (-1);
}
/*
* Return how many page sizes are available for the user to use. This is
* what the hardware supports and not based upon how the OS implements the
* support of different page sizes.
*/
page_num_user_pagesizes(void)
{
return (mmu_exported_page_sizes);
}
page_num_pagesizes(void)
{
return (mmu_page_sizes);
}
/*
* returns the count of the number of base pagesize pages associated with szc
*/
{
if (szc >= mmu_page_sizes)
}
{
if (szc >= mmu_page_sizes)
}
/*
* Return the size of a page based upon the index passed in. An index of
* zero refers to the smallest page size in the system, and as index increases
* it refers to the next larger supported page size in the system.
* Note that szc and userszc may not be the same due to unsupported szc's on
* some systems.
*/
{
if (szc >= mmu_page_sizes)
}
{
if (szc >= mmu_page_sizes)
}
{
ASSERT(page_colors != 0);
}
/*
* Called by startup().
* Size up the per page size free list counters based on physmax
* of each node and max_mem_nodes.
*/
page_ctrs_sz(void)
{
int r; /* region size */
int mnode;
int i;
/*
* We need to determine how many page colors there are for each
* page size in order to allocate memory for any color specific
* arrays.
*/
colors_per_szc[0] = page_colors;
for (i = 1; i < mmu_page_sizes; i++) {
colors_per_szc[i] =
}
continue;
/*
* determine size needed for page counter arrays with
* base aligned to large page size.
*/
for (r = 1; r < mmu_page_sizes; r++) {
/* add in space for hpm_counters */
r_align = page_get_pagecnt(r);
/*
* Round up to always allocate on pointer sized
* boundaries.
*/
sizeof (hpmctr_t *));
/* add in space for hpm_color_current */
ctrs_sz += (colors_per_szc[r] *
sizeof (size_t));
}
}
for (r = 1; r < mmu_page_sizes; r++) {
/* add in space for page_ctrs_cands */
sizeof (pgcnt_t);
}
/* ctr_mutex */
/* size for page list counts */
/*
* add some slop for roundups. page_ctrs_alloc will roundup the start
* address of the counters to ecache_alignsize boundary for every
* memory node.
*/
}
{
int mnode;
int r; /* region size */
int i;
/*
* We need to determine how many page colors there are for each
* page size in order to allocate memory for any color specific
* arrays.
*/
colors_per_szc[0] = page_colors;
for (i = 1; i < mmu_page_sizes; i++) {
colors_per_szc[i] =
}
for (r = 1; r < mmu_page_sizes; r++) {
}
/* page_ctrs_cands */
for (r = 1; r < mmu_page_sizes; r++) {
for (i = 0; i < NPC_MUTEX; i++) {
}
}
/* page_ctrs_cands pcc_color_free array */
for (r = 1; r < mmu_page_sizes; r++) {
for (i = 0; i < NPC_MUTEX; i++) {
= colors_per_szc[r];
(pgcnt_t *)alloc_base;
alloc_base += colors_per_szc[r] *
sizeof (pgcnt_t);
}
}
}
/* ctr_mutex */
for (i = 0; i < NPC_MUTEX; i++) {
}
/* initialize page list counts */
int r_shift;
continue;
for (r = 1; r < mmu_page_sizes; r++) {
/*
* the page_counters base has to be aligned to the
* page count of page size code r otherwise the counts
* will cross large page boundaries.
*/
r_align = page_get_pagecnt(r);
/* base needs to be aligned - lower to aligned value */
r_shift = PAGE_BSZS_SHIFT(r);
colors_per_szc[r];
(size_t *)alloc_base;
for (i = 0; i < colors_per_szc[r]; i++) {
PAGE_COUNTERS_CURRENT_COLOR(mnode, r, i) = i;
}
PAGE_COUNTERS_COUNTERS(mnode, r) =
(hpmctr_t *)alloc_base;
/*
* Round up to make alloc_base always be aligned on
* a pointer boundary.
*/
sizeof (hpmctr_t *));
/*
* Verify that PNUM_TO_IDX and IDX_TO_PNUM
* satisfy the identity requirement.
* We should be able to go from one to the other
* and get consistent values.
*/
(IDX_TO_PNUM(mnode, r, 0))) == 0);
}
/*
* Roundup the start address of the page_counters to
* cache aligned boundary for every memory node.
* page_ctrs_sz() has added some slop for these roundups.
*/
}
/* Initialize other page counter specific data structures. */
}
return (alloc_base);
}
/*
* Functions to adjust region counters for each size free list.
* Caller is responsible to acquire the ctr_mutex lock if necessary and
* thus can be called during startup without locks.
*/
/* ARGSUSED */
void
{
ssize_t r; /* region size */
int lckidx;
/* no counter update needed for largest page size */
return;
}
/*
* Increment the count of free pages for the current
* region. Continue looping up in region size incrementing
* count if the preceeding region is full.
*/
while (r < mmu_page_sizes) {
break;
r++;
}
}
void
{
}
void
{
int lckidx;
ssize_t r; /* region size */
/* no counter update needed for largest page size */
return;
}
/*
* Decrement the count of free pages for the current
* region. Continue looping up in region size decrementing
* count if the preceeding region was full.
*/
while (r < mmu_page_sizes) {
break;
}
r++;
}
}
/*
* Adjust page counters following a memory attach, since typically the
* size of the array needs to change, and the PFN to counter index
* mapping needs to change.
*/
page_ctrs_adjust(int mnode)
{
int r; /* region size */
int i;
PC_BASE_ALIGN) - newbase;
/*
* We need to determine how many page colors there are for each
* page size in order to allocate memory for any color specific
* arrays.
*/
colors_per_szc[0] = page_colors;
for (r = 1; r < mmu_page_sizes; r++) {
colors_per_szc[r] =
}
/*
* Preallocate all of the new hpm_counters arrays as we can't
* hold the page_ctrs_rwlock as a writer and allocate memory.
* If we can't allocate all of the arrays, undo our work so far
* and return failure.
*/
for (r = 1; r < mmu_page_sizes; r++) {
sizeof (hpmctr_t), KM_NOSLEEP);
while (--r >= 1) {
size_cache[r] * sizeof (hpmctr_t));
}
return (ENOMEM);
}
size_cache[r] = pcsz;
}
/*
* Preallocate all of the new color current arrays as we can't
* hold the page_ctrs_rwlock as a writer and allocate memory.
* If we can't allocate all of the arrays, undo our work so far
* and return failure.
*/
for (r = 1; r < mmu_page_sizes; r++) {
colors_per_szc[r], KM_NOSLEEP);
if (color_cache[r] == NULL) {
while (--r >= 1) {
kmem_free(color_cache[r],
colors_per_szc[r] * sizeof (size_t));
}
for (r = 1; r < mmu_page_sizes; r++) {
size_cache[r] * sizeof (hpmctr_t));
}
return (ENOMEM);
}
}
/*
* Grab the write lock to prevent others from walking these arrays
* while we are modifying them.
*/
for (r = 1; r < mmu_page_sizes; r++) {
/*
* Map the intersection of the old and new
* counters into the new array.
*/
PAGE_COUNTERS_SHIFT(mnode, r);
sizeof (hpmctr_t));
} else {
PAGE_COUNTERS_SHIFT(mnode, r);
sizeof (hpmctr_t));
}
}
color_cache[r] = NULL;
/*
* for now, just reset on these events as it's probably
* not worthwhile to try and optimize this.
*/
for (i = 0; i < colors_per_szc[r]; i++) {
PAGE_COUNTERS_CURRENT_COLOR(mnode, r, i) = i;
}
/* cache info for freeing out of the critical path */
size_cache[r] = old_csz;
}
color_cache[r] = old_color_array;
}
/*
* Verify that PNUM_TO_IDX and IDX_TO_PNUM
* satisfy the identity requirement.
* We should be able to go from one to the other
* and get consistent values.
*/
(IDX_TO_PNUM(mnode, r, 0))) == 0);
}
/*
* Now that we have dropped the write lock, it is safe to free all
* of the memory we have cached above.
*/
for (r = 1; r < mmu_page_sizes; r++) {
size_cache[r] * sizeof (hpmctr_t));
}
if (color_cache[r] != NULL) {
kmem_free(color_cache[r],
colors_per_szc[r] * sizeof (size_t));
}
}
return (0);
}
/*
* color contains a valid color index or bin for cur_szc
*/
{
}
return (color);
}
#ifdef DEBUG
/*
* confirm pp is a large page corresponding to szc
*/
void
{
if (npgs == 1) {
return;
}
/*
* Check list of pages.
*/
while (npgs--) {
if (npgs != 0) {
}
}
}
#endif /* DEBUG */
void
page_freelist_lock(int mnode)
{
int i;
for (i = 0; i < NPC_MUTEX; i++) {
}
}
void
{
int i;
for (i = 0; i < NPC_MUTEX; i++) {
}
}
/*
* add pp to the specified page list. Defaults to head of the page list
* unless PG_LIST_TAIL is specified.
*/
void
{
int mnode;
/*
* Large pages should be freed via page_list_add_pages().
*/
/*
* Don't need to lock the freelist first here
* because the page isn't on the freelist yet.
* This means p_szc can't change on us.
*/
if (flags & PG_LIST_ISINIT) {
/*
* PG_LIST_ISINIT is set during system startup (ie. single
* threaded), add a page to the free list and add to the
* the free region counters w/o any locking
*/
/* inline version of page_add() */
} else
} else {
if (flags & PG_FREE_LIST) {
} else {
}
if (flags & PG_LIST_TAIL)
/*
* Add counters before releasing pcm mutex to avoid a race with
* page_freelist_coalesce and page_freelist_fill.
*/
}
#if defined(__sparc)
if (PP_ISNORELOC(pp)) {
}
#endif
/*
* It is up to the caller to unlock the page!
*/
}
#ifdef __sparc
/*
* This routine is only used by kcage_init during system startup.
* It performs the function of page_list_sub/PP_SETNORELOC/page_list_add
* without the overhead of taking locks and updating counters.
*/
void
{
int mnode;
int mtype;
int flags = PG_LIST_ISCAGE;
/*
* If this is a large page on the freelist then
* break it up into smaller pages.
*/
/*
* Get list page is currently on.
*/
flags |= PG_FREE_LIST;
} else {
flags |= PG_CACHE_LIST;
}
/*
* Delete page from current list.
*/
} else {
}
/* LINTED */
/*
* Set no reloc for cage initted pages.
*/
/*
* Get new list for page.
*/
} else {
}
/*
* Insert page on new list.
*/
} else {
}
/* LINTED */
/*
* Update cage freemem counter
*/
}
#else /* __sparc */
/* ARGSUSED */
void
{
panic("page_list_noreloc_startup: should be here only for sparc");
}
#endif
void
{
int mnode;
if (flags & PG_LIST_ISINIT) {
} else {
#if defined(__sparc)
if (PP_ISNORELOC(pp))
#endif
}
}
/*
* During boot, need to demote a large page to base
* pagesize pages for seg_kmem for use in boot_alloc()
*/
void
{
PC_FREE);
}
/*
* Take a particular page off of whatever freelist the page
* is claimed to be on.
*
* NOTE: Only used for PAGESIZE pages.
*/
void
{
int bin;
int mnode;
/*
* The p_szc field can only be changed by page_promote()
* and page_demote(). Only free pages can be promoted and
* demoted and the free list MUST be locked during these
* operations. So to prevent a race in page_list_sub()
* between computing which bin of the freelist lock to
* grab and actually grabing the lock we check again that
* the bin we locked is still the correct one. Notice that
* the p_szc field could have actually changed on us but
* if the bin happens to still be the same we are safe.
*/
goto try_again;
}
if (flags & PG_FREE_LIST) {
} else {
}
/*
* Common PAGESIZE case.
*
* Note that we locked the freelist. This prevents
* the p_szc will not change until we drop pcm mutex.
*/
/*
* Subtract counters before releasing pcm mutex
* to avoid race with page_freelist_coalesce.
*/
#if defined(__sparc)
if (PP_ISNORELOC(pp)) {
}
#endif
return;
}
/*
* Large pages on the cache list are not supported.
*/
if (flags & PG_CACHE_LIST)
panic("page_list_sub: large page on cachelist");
/*
* Slow but rare.
*
* Somebody wants this particular page which is part
* of a large page. In this case we just demote the page
* if it's on the freelist.
*
* We have to drop pcm before locking the entire freelist.
* Once we have re-locked the freelist check to make sure
* the page hasn't already been demoted or completely
* freed.
*/
/*
* Large page is on freelist.
*/
}
/*
* Subtract counters before releasing pcm mutex
* to avoid race with page_freelist_coalesce.
*/
#if defined(__sparc)
if (PP_ISNORELOC(pp)) {
}
#endif
}
void
{
int mnode;
/*
* See comment in page_list_sub().
*/
goto try_again;
}
/*
* If we're called with a page larger than szc or it got
* promoted above szc before we locked the freelist then
* drop pcm and re-lock entire freelist. If page still larger
* than szc then demote it.
*/
(void) page_demote(mnode,
}
}
} else {
}
} else {
}
#if defined(__sparc)
if (PP_ISNORELOC(pp)) {
}
#endif
}
/*
* Add the page to the front of a linked list of pages
* using the p_next & p_prev pointers for the list.
* The caller is responsible for protecting the list pointers.
*/
void
{
} else {
}
}
/*
* Remove this page from a linked list of pages
* using the p_next & p_prev pointers for the list.
*
* The caller is responsible for protecting the list pointers.
*/
void
{
panic("mach_page_sub");
else {
}
}
/*
* Routine fsflush uses to gradually coalesce the free list into larger pages.
*/
void
{
int mnode;
int idx;
}
static uint_t page_promote_err;
static uint_t page_promote_noreloc_err;
/*
* Create a single larger page (of szc new_szc) from smaller contiguous pages
* for the given mnode starting at pfnum. Pages involved are on the freelist
* before the call and may be returned to the caller if requested, otherwise
* they will be placed back on the freelist.
* If flags is PC_ALLOC, then the large page will be returned to the user in
* a state which is consistent with a page being taken off the freelist. If
* we failed to lock the new large page, then we will return NULL to the
* caller and put the large page on the freelist instead.
* If flags is PC_FREE, then the large page will be placed on the freelist,
* and NULL will be returned.
* The caller is responsible for locking the freelist as well as any other
* accounting which needs to be done for a returned page.
*
* RFE: For performance pass in pp instead of pfnum so
* we can avoid excessive calls to page_numtopp_nolock().
* This would depend on an assumption that all contiguous
* our pp.
*
* Lock ordering:
*
* There is a potential but rare deadlock situation
* for page promotion and demotion operations. The problem
* is there are two paths into the freelist manager and
* they have different lock orders:
*
* page_create()
* lock freelist
* page_lock(EXCL)
* unlock freelist
* return
* caller drops page_lock
*
* page_free() and page_reclaim()
* caller grabs page_lock(EXCL)
*
* lock freelist
* unlock freelist
* drop page_lock
*
* What prevents a thread in page_create() from deadlocking
* with a thread freeing or reclaiming the same page is the
* page_trylock() in page_get_freelist(). If the trylock fails
* it skips the page.
*
* The lock ordering for promotion and demotion is the same as
* for page_create(). Since the same deadlock could occur during
* page promotion and freeing or reclaiming of a page on the
* cache list we might have to fail the operation and undo what
* have done so far. Again this is rare.
*/
page_t *
{
uint_t i;
int which_list;
/*
* General algorithm:
* Find the starting page
* Walk each page struct removing it from the freelist,
* and linking it to all the other pages removed.
* Once all pages are off the freelist,
* walk the list, modifying p_szc to new_szc and what
* ever other info needs to be done to create a large free page.
* According to the flags, either return the page or put it
* on the freelist.
*/
/*
* Loop through smaller pages to confirm that all pages
* give the same result for PP_ISNORELOC().
* We can check this reliably here as the protocol for setting
* P_NORELOC requires pages to be taken off the free list first.
*/
/* First page, set requirement. */
return (NULL);
}
}
/* Loop around coalescing the smaller pages into a big page. */
while (pages_left) {
/*
* Remove from the freelist.
*/
/*
* PG_FREE_LIST
*/
} else {
}
} else {
/*
* PG_CACHE_LIST
*
* Since this page comes from the
* cachelist, we must destroy the
* vnode association.
*/
goto fail_promote;
}
/*
* We need to be careful not to deadlock
* with another thread in page_lookup().
* The page_lookup() thread could be holding
* the same phm that we need if the two
* pages happen to hash to the same phm lock.
* At this point we have locked the entire
* freelist and page_lookup() could be trying
* to grab a freelist lock.
*/
if (!mutex_tryenter(phm)) {
goto fail_promote;
}
PP_SETAGED(pp);
}
/*
* Concatenate the smaller page(s) onto
* the large page list.
*/
pages_left -= npgs;
while (npgs--) {
}
}
/*
* return the page to the user if requested
* in the properly locked state.
*/
return (pplist);
}
/*
* Otherwise place the new large page on the freelist
*/
return (NULL);
/*
* A thread must have still been freeing or
* reclaiming the page on the cachelist.
* To prevent a deadlock undo what we have
* done sofar and return failure. This
* situation can only happen while promoting
* PAGESIZE pages.
*/
while (pplist) {
}
return (NULL);
}
/*
* Break up a large page into smaller size pages.
* Pages involved are on the freelist before the call and may
* be returned to the caller if requested, otherwise they will
* be placed back on the freelist.
* The caller is responsible for locking the freelist as well as any other
* accounting which needs to be done for a returned page.
* If flags is not PC_ALLOC, the color argument is ignored, and thus
* technically, any value may be passed in but PC_NO_COLOR is the standard
* which should be followed for clarity's sake.
*/
page_t *
{
/*
* Number of PAGESIZE pages for smaller new_szc
* page.
*/
while (pplist) {
/*
* We either break it up into PAGESIZE pages or larger.
*/
} else {
}
} else {
/*
* Break down into smaller lists of pages.
*/
n = npgs;
while (n--) {
}
} else {
}
}
}
return (ret_pp);
}
int mpss_coalesce_disable = 0;
/*
* Coalesce free pages into a page of the given szc and color if possible.
* Return the pointer to the page created, otherwise, return NULL.
*/
static page_t *
{
int r; /* region size */
int color_stride;
if (mpss_coalesce_disable) {
return (NULL);
}
r = szc;
if (cands == 0) {
return (NULL);
}
full = FULL_REGION_CNT(r);
/* Prevent page_counters dynamic memory from being freed */
idx += color_stride;
for (i = 0; i < buckets_to_check; i++) {
/*
* RFE: For performance maybe we can do something less
* brutal than locking the entire freelist. So far
* this doesn't seem to be a performance problem?
*/
goto skip_this_one;
}
idx;
#if defined(__sparc)
if (PP_ISNORELOC(ret_pp)) {
}
#endif
return (ret_pp);
}
/*
* No point looking for another page if we've
* already tried all of the ones that
* page_ctr_cands indicated. Stash off where we left
* off.
* Note: this is not exact since we don't hold the
* page_freelist_locks before we initially get the
* value of cands for performance reasons, but should
* be a decent approximation.
*/
if (--cands == 0) {
idx;
break;
}
}
idx += color_stride;
}
return (NULL);
}
/*
* For the given mnode, promote as many small pages to large pages as possible.
*/
void
{
int r; /* region size */
if (mpss_coalesce_disable) {
return;
}
/*
* Lock the entire freelist and coalesce what we can.
*
* Always promote to the largest page possible
* first to reduce the number of page promotions.
*/
for (r = mmu_page_sizes - 1; r > 0; r--) {
if (cands == 0) {
continue;
}
full = FULL_REGION_CNT(r);
pfnum <
}
}
}
}
/*
* This is where all polices for moving pages around
* to different page size free lists is implemented.
* Returns 1 on success, 0 on failure.
*
* So far these are the priorities for this algorithm in descending
* order:
*
* 1) When servicing a request try to do so with a free page
* from next size up. Helps defer fragmentation as long
* as possible.
*
* 2) Page coalesce on demand. Only when a freelist
* larger than PAGESIZE is empty and step 1
* will not work since all larger size lists are
* also empty.
*
* If pfnhi is non-zero, search for large page with pfn range less than pfnhi.
*/
page_t *
{
int bin;
/*
* First try to break up a larger page to fill
* current size freelist.
*/
while (nszc < mmu_page_sizes) {
/*
* If page found then demote it.
*/
/*
* If pfnhi is not PFNNULL, look for large page below
* pfnhi. PFNNULL signifies no pfn requirement.
*/
do {
break;
}
}
if (pp) {
if (ret_pp) {
#if defined(__sparc)
if (PP_ISNORELOC(ret_pp)) {
}
#endif
return (ret_pp);
}
}
}
nszc++;
}
/*
* Ok that didn't work. Time to coalesce.
*/
if (szc != 0) {
}
return (ret_pp);
}
/*
* Helper routine used only by the freelist code to lock
* a page. If the page is a large page then it succeeds in
* locking all the constituent pages or none at all.
* Returns 1 on sucess, 0 on failure.
*/
static int
{
/*
* Fail if can't lock first or only page.
*/
return (0);
}
/*
* PAGESIZE: common case.
*/
return (1);
}
/*
* Large page case.
*/
/*
* On failure unlock what we
* have locked so far.
*/
}
return (0);
}
}
return (1);
}
page_t *
{
int i, fill_tried, fill_marker;
/* LINTED */
if (mtype < 0) { /* mnode foes not have memory in mtype range */
return (NULL);
}
/*
* Set how many physical colors for this page size.
*/
/* cpu_page_colors is non-zero if a page color may be in > 1 bin */
/*
* adjust cpucolors to possibly check additional 'equivalent' bins
* to try to minimize fragmentation of large pages by delaying calls
* to page_freelist_fill.
*/
if (colorequiv > 1) {
}
/*
* Only hold one freelist lock at a time, that way we
* can start anywhere and not have to worry about lock
* ordering.
*/
fill_tried = 0;
nwaycnt = 0;
for (i = 0; i <= colors; i++) {
/*
* These were set before the page
* was put on the free list,
* they must still be set.
*/
/*
* Walk down the hash chain.
* 8k pages are linked on p_next
* and p_prev fields. Large pages
* are a contiguous group of
* constituent pages linked together
* on their p_next and p_prev fields.
* The large pages are linked together
* on the hash chain using p_vpnext
* p_vpprev of the base constituent
* page of each large page.
*/
if (szc == 0) {
} else {
}
mnode);
break;
}
}
if (pp) {
if (szc == 0) {
} else {
pp);
}
panic("free page is not. pp %p",
(void *)pp);
#if defined(__sparc)
(flags & PG_NORELOC) == 0);
if (PP_ISNORELOC(pp)) {
}
#endif
pgmf_allocok[szc]);
return (pp);
}
}
}
/*
* Wow! The initial bin is empty.
* If specific color is needed, check if page color may be
* in other bins. cpucolors is:
* 0 if the colors for this cpu is equal to page_colors.
* This means that pages with a particular color are in a
* single bin.
* -1 if colors of cpus (cheetah+) are heterogenous. Need to
* first determine the colors for the current cpu.
* >0 colors of all cpus are homogenous and < page_colors
*/
if (!nwaybins) {
/*
* cpucolors is negative if ecache setsizes
* are heterogenous. determine colors for this
* particular cpu.
*/
if (cpucolors < 0) {
} else {
}
if (nwaybins < 2)
cpucolors = 0;
}
nwaycnt++;
(colors - 1);
goto try_again;
}
}
/* back to initial color if fall-thru */
}
/*
* color bins are all empty if color match. Try and satisfy
* the request by breaking up or coalescing pages from
* a different size freelist of the correct color that
* satisfies the ORIGINAL color requested. If that
* fails then try pages of the same size but different
* colors assuming we are not called with
* PG_MATCH_COLOR.
*/
if (!fill_tried) {
fill_tried = 1;
PFNNULL);
return (pp);
}
}
if (flags & PG_MATCH_COLOR)
break;
/*
* Select next color bin to try.
*/
if (szc == 0) {
/*
* PAGESIZE page case.
*/
if (i == 0) {
bin_marker = bin;
} else {
if (bin == bin_marker) {
bin_marker = bin;
}
}
} else {
/*
* Large page case.
*/
}
/*
* If bin advanced to the next color bin of the
* next larger pagesize, there is a chance the fill
* could succeed.
*/
fill_tried = 0;
}
#if defined(__sparc)
(kcage_freemem >= kcage_lotsfree)) {
/*
* The Cage is ON and with plenty of free mem, and
* we're willing to check for a NORELOC page if we
* couldn't find a RELOC page, so spin again.
*/
flags |= PG_NORELOC;
goto big_try_again;
}
#else
if (flags & PGI_MT_RANGE) {
/* cycle through range of mtypes */
if (mtype >= 0)
goto big_try_again;
}
#endif
return (NULL);
}
/*
* Returns the count of free pages for 'pp' with size code 'szc'.
* Note: This function does not return an exact value as the page freelist
* locks are not held and thus the values in the page_counters may be
* changing as we walk through the data.
*/
static int
{
int i;
/* Make sure pagenum passed in is aligned properly */
/* Prevent page_counters dynamic memory from being freed */
/* Check for completely full region */
return (pgfree);
}
while (--r > 0) {
full = FULL_REGION_CNT(r);
/*
* If cnt here is full, that means we have already
* accounted for these pages earlier.
*/
}
}
}
return (pgfree);
}
/*
* Called from page_geti_contig_pages to exclusively lock constituent pages
* starting from 'spp' for page size code 'szc'.
*
* If 'ptcpthreshold' is set, the number of free pages needed in the 'szc'
* region needs to be greater than or equal to the threshold.
*/
static int
{
goto skipptcpcheck;
/*
* check if there are sufficient free pages available before attempting
* to trylock. Count is approximate as page counters can change.
*/
/* attempt to trylock if there are sufficient already free pages */
return (0);
}
for (i = 0; i < pgcnt; i++) {
while (--i != (pgcnt_t)-1) {
}
return (0);
}
ASSERT(i == 0);
return (0);
}
if (PP_ISNORELOC(pp)) {
while (i != (pgcnt_t)-1) {
i--;
}
return (0);
}
}
return (1);
}
/*
* Claim large page pointed to by 'pp'. 'pp' is the starting set
* of 'szc' constituent pages that had been locked exclusively previously.
* Will attempt to relocate constituent pages in use.
*/
static page_t *
{
while (pgcnt) {
/*
* If this is a PG_FREE_LIST page then its
* size code can change underneath us due to
* page promotion or demotion. As an optimzation
* use page_list_sub_pages() instead of
* page_list_sub().
*/
return (pp);
}
}
continue;
}
PP_SETAGED(pp);
pp++;
pgcnt--;
continue;
}
/*
* page_create_wait freemem accounting done by caller of
* page_get_freelist and not necessary to call it prior to
* calling page_get_replacement_page.
*
* page_get_replacement_page can call page_get_contig_pages
* to acquire a large page (szc > 0); the replacement must be
* smaller than the contig page size to avoid looping or
* szc == 0 and PGI_PGCPSZC0 is set.
*/
if (replpp) {
}
}
/*
* If replacement is NULL or do_page_relocate fails, fail
* coalescing of pages.
*/
/*
* Unlock un-processed target list
*/
while (pgcnt--) {
pp++;
}
/*
* Free the processed target list.
*/
while (pplist) {
}
return (NULL);
}
/* LINTED */
while (npgs--) {
(flags & PGI_PGCPSZC0)));
}
}
return (pplist);
}
/*
* Trim kernel cage from pfnlo-pfnhi and store result in lo-hi. Return code
* of 0 means nothing left after trim.
*/
int
{
int decr;
int rc = 0;
/* lower part of this mseg inside kernel cage */
/* kernel cage may have transitioned past mseg */
rc = 1;
}
}
/* else entire mseg in the cage */
} else {
/* upper part of this mseg inside kernel cage */
/* kernel cage may have transitioned past mseg */
rc = 1;
}
} else {
/* entire mseg outside of kernel cage */
rc = 1;
}
}
return (rc);
}
/*
* called from page_get_contig_pages to search 'pfnlo' thru 'pfnhi' to "claim" a
* page with size code 'szc'. Claiming such a page requires acquiring
* exclusive locks on all constituent pages (page_trylock_contig_pages),
* relocating pages in use and concatenating these constituent pages into a
* large page.
*
* The page lists do not have such a large page and page_freelist_fill has
*
* 'flags' may specify PG_COLOR_MATCH which would limit the search of large
* pages with the same color as 'bin'.
*
* 'pfnflag' specifies the subset of the pfn range to search.
*/
static page_t *
{
return (NULL);
/*
* trim the pfn range to search based on pfnflag. pfnflag is set
* when there have been previous page_get_contig_page failures to
* limit the search.
*
* The high bit in pfnflag specifies the number of 'slots' in the
* pfn range and the remainder of pfnflag specifies which slot.
* For example, a value of 1010b would mean the second slot of
* the pfn range that has been divided into 8 slots.
*/
if (pfnflag > 1) {
int slotlen;
}
memsegs_lock(0);
/*
* loop through memsegs to look for contig page candidates
*/
/* no overlap */
continue;
}
/* mseg too small */
continue;
/* trim off kernel cage pages from pfn range */
if (kcage_on) {
continue;
} else {
}
/* round to szcpgcnt boundaries */
continue;
/*
* set lo to point to the pfn for the desired bin. Large
* page sizes may only have a single page color
*/
/*
* factor in colorequiv to check additional
* 'equivalent' bins.
*/
/* determine bin that lo currently points to */
/*
* set lo to point at appropriate color and set skip
* to arrive at the next szc page of the same color.
*/
} else {
/* check all pages starting from lo */
}
/* mseg cannot satisfy color request */
continue;
/* randomly choose a point between lo and hi to begin search */
do {
colorequiv > 1 ||
/* pages unlocked by page_claim on failure */
memsegs_unlock(0);
return (pp);
}
}
/* start from the beginning */
}
}
memsegs_unlock(0);
return (NULL);
}
/*
* controlling routine that searches through physical memory in an attempt to
* claim a large page based on the input parameters.
* on the page free lists.
*
* calls page_geti_contig_pages with an initial pfn range from the mnode
* and mtype. page_geti_contig_pages will trim off the parts of the pfn range
* that overlaps with the kernel cage or does not match the requested page
* color if PG_MATCH_COLOR is set. Since this search is very expensive,
* page_geti_contig_pages may further limit the search range based on
* previous failure counts (pgcpfailcnt[]).
*
* for PGI_PGCPSZC0 requests, page_get_contig_pages will relocate a base
* pagesize page that satisfies mtype.
*/
page_t *
{
int pfnflag = 0; /* no limit on search if 0 */
/* LINTED */
if (mtype < 0) { /* mnode does not have memory in mtype range */
return (NULL);
}
/* do not limit search and ignore color if hi pri */
/* remove color match to improve chances */
flags &= ~PG_MATCH_COLOR;
do {
/* get pfn range based on mnode and mtype */
if (pfnflag) {
/* double the search size */
}
return (pp);
}
/* LINTED */
} while ((flags & PGI_MT_RANGE) &&
return (NULL);
}
/*
* Find the `best' page on the freelist for this (vp,off) (as,vaddr) pair.
*
* Does its own locking and accounting.
* If PG_MATCH_COLOR is set, then NULL will be returned if there are no
* pages of the proper color even if there are pages of a different color.
*
* Finds a page, removes it, THEN locks it.
*/
/*ARGSUSED*/
page_t *
{
int mnode;
int mtype;
/*
* If we aren't passed a specific lgroup, or passed a freed lgrp
* assume we wish to allocate near to the current thread's home.
*/
if (!LGRP_EXISTS(lgrp))
lgrp = lgrp_home_lgrp();
if (kcage_on) {
/*
* Set a "reserve" of kcage_throttlefree pages for
* PG_PANIC and cageout thread allocations.
*
* Everybody else has to serialize in
* page_create_get_something() to get a cage page, so
* that we don't deadlock cageout!
*/
return (NULL);
}
} else {
flags &= ~PG_NORELOC;
flags |= PGI_NOCAGE;
}
/* LINTED */
/*
* Convert size to page size code.
*/
panic("page_get_freelist: illegal page size request");
/* LINTED */
/* bin is for base pagesize color - convert if larger pagesize. */
if (szc)
/*
* Try to get a local page first, but try remote if we can't
* get a page of the right color.
*/
int, mnode,
return (pp);
}
}
/*
* for non-SZC0 PAGESIZE requests, check cachelist before checking
* remote free lists. Caller expected to call page_get_cachelist which
* will check local cache lists and remote free lists.
*/
return (NULL);
}
/*
* Try to get a non-local freelist page.
*/
int, mnode,
return (pp);
}
}
/*
* when the cage is off chances are page_get_contig_pages() will fail
* to lock a large page chunk therefore when the cage is off it's not
*
* page_get_contig_pages() also called to acquire a base pagesize page
* for page_create_get_something().
*/
(page_get_func != page_get_contig_pages)) {
goto pgretry;
}
pgcpfailcnt[szc]++;
return (NULL);
}
/*
* Find the `best' page on the cachelist for this (vp,off) (as,vaddr) pair.
*
* Does its own locking.
* If PG_MATCH_COLOR is set, then NULL will be returned if there are no
* pages of the proper color even if there are pages of a different color.
* Otherwise, scan the bins for ones with pages. For each bin with pages,
* try to lock one of them. If no page can be locked, try the
* next bin. Return NULL if a page can not be found and locked.
*
* Finds a pages, trys to lock it, then removes it.
*/
/*ARGSUSED*/
page_t *
{
/*LINTED*/
int mnode;
int mtype;
/*
* If we aren't passed a specific lgroup, or pasased a freed lgrp
* assume we wish to allocate near to the current thread's home.
*/
if (!LGRP_EXISTS(lgrp))
lgrp = lgrp_home_lgrp();
if (!kcage_on) {
flags &= ~PG_NORELOC;
flags |= PGI_NOCAGE;
}
/*
* Reserve kcage_throttlefree pages for critical kernel
* threads.
*
* Everybody else has to go to page_create_get_something()
* to get a cage page, so we don't deadlock cageout.
*/
return (NULL);
}
/* LINTED */
/* LINTED */
/*
* Try local cachelists first
*/
int, mnode,
return (pp);
}
}
/*
* Try freelists/cachelists that are farther away
* This is our only chance to allocate remote pages for PAGESIZE
* requests.
*/
0, flags);
int, mnode,
return (pp);
}
int, mnode,
return (pp);
}
}
return (NULL);
}
page_t *
{
int i;
int cpucolors;
/* LINTED */
if (mtype < 0) { /* mnode does not have memory in mtype range */
return (NULL);
}
nwaybins = 0;
/*
* adjust cpucolors to possibly check additional 'equivalent' bins
* to try to minimize fragmentation of large pages by delaying calls
* to page_freelist_fill.
*/
if (colorequiv > 1) {
}
/*
* Only hold one cachelist lock at a time, that way we
* can start anywhere and not have to worry about lock
* ordering.
*/
nwaycnt = 0;
for (i = 0; i <= page_colors; i++) {
/*
* We have searched the
* complete list!
* And all of them (might
* only be one) are locked.
* This can happen since
* these pages can also be
* found via the hash list.
* When found via the hash
* list, they are locked
* first, then removed.
* We give up to let the
* other thread run.
*/
break;
}
mnode);
}
if (pp) {
/*
* Found and locked a page.
* Pull it off the list.
*/
mtype);
/*
* Subtract counters before releasing
* pcm mutex to avoid a race with
* page_freelist_coalesce and
* page_freelist_fill.
*/
#if defined(__sparc)
(flags & PG_NORELOC) == 0 ||
PP_ISNORELOC(pp));
if (PP_ISNORELOC(pp)) {
}
#endif
return (pp);
}
}
}
/*
* Wow! The initial bin is empty or no page in the bin could
* be locked.
*
* If specific color is needed, check if page color may be in
* other bins.
*/
if (!nwaybins) {
if (cpucolors < 0) {
if (nwaybins < 2)
cpucolors = 0;
} else {
}
}
break;
}
continue;
}
if (i == 0) {
bin_marker = bin;
} else {
if (bin == bin_marker) {
bin_marker = bin;
}
}
}
#if defined(__sparc)
(kcage_freemem >= kcage_lotsfree)) {
/*
* The Cage is ON and with plenty of free mem, and
* we're willing to check for a NORELOC page if we
* couldn't find a RELOC page, so spin again.
*/
flags |= PG_NORELOC;
goto big_try_again;
}
#else
if (flags & PGI_MT_RANGE) {
if (mtype >= 0)
goto big_try_again;
}
#endif
return (NULL);
}
#ifdef DEBUG
#define REPL_PAGE_STATS
#endif /* DEBUG */
#ifdef REPL_PAGE_STATS
struct repl_page_stats {
#else /* REPL_PAGE_STATS */
#define REPL_STAT_INCR(v)
#endif /* REPL_PAGE_STATS */
int pgrppgcp;
/*
* The freemem accounting must be done by the caller.
* First we try to get a replacement page of the same size as like_pp,
* if that is not possible, then we just get a set of discontiguous
* PAGESIZE pages.
*/
page_t *
{
int mnode, page_mnode;
int szc;
int mtype;
int flags = 0;
/*
* Now we reset like_pp to the base page_t.
* That way, we won't walk past the end of this 'szc' page.
*/
if (PP_ISNORELOC(like_pp)) {
} else if (pgrflags & PGR_NORELOC) {
flags = PG_NORELOC;
}
/*
* Kernel pages must always be replaced with the same size
* pages, since we cannot properly handle demotion of kernel
* pages.
*/
pgrflags |= PGR_SAMESZC;
/* LINTED */
while (npgs) {
for (;;) {
/*
* If an lgroup was specified, try to get the
* page from that lgroup.
*/
if (LGRP_EXISTS(lgrp)) {
/* Try the lgroup's freelists first */
!= -1) {
flags);
}
/*
* Now try it's cachelists if this is a
* small page. Don't need to do it for
* larger ones since page_freelist_coalesce()
* already failed.
*/
break;
/* Now try it's cachelists */
!= -1) {
}
break;
}
/* Done looking in this lgroup. Bail out. */
break;
}
/*
* No lgroup was specified, so just try to get the
* page as close to like_pp's mnode as possible.
* First try the local freelist...
*/
break;
/*
* ...then the local cachelist. Don't need to do it for
* larger pages cause page_freelist_coalesce() already
* failed there anyway.
*/
if (szc == 0) {
break;
}
}
/* Now try remote freelists */
page_mnode = mnode;
lgrp =
!= -1) {
/*
* Skip local mnode.
*/
if ((mnode == page_mnode) ||
continue;
}
break;
/* Now try remote cachelists */
if (mnode == -1)
break;
/*
* Skip local mnode.
*/
if ((mnode == page_mnode) ||
continue;
break;
}
}
/*
* Break out of while loop under the following cases:
* - If we successfully got a page.
* - If pgrflags specified only returning a specific
* page size and we could not find that page size.
* - If we could not satisfy the request with PAGESIZE
* or larger pages.
*/
break;
/* try to find contig page */
(mnode =
!= -1) {
flags | PGI_PGCPHIPRI);
}
break;
}
/*
* The correct thing to do here is try the next
* page size down using szc--. Due to a bug
* with the processing of HAT_RELOAD_SHARE
* where the sfmmu_ttecnt arrays of all
* hats sharing an ISM segment don't get updated,
* using intermediate size pages for relocation
* can lead to continuous page faults.
*/
szc = 0;
}
int, mnode,
PP_CLRFREE(pp);
PP_CLRAGED(pp);
npgs--;
}
} else {
break;
}
}
if (npgs) {
/*
* We were unable to allocate the necessary number
* of pages.
* We need to free up any pl.
*/
return (NULL);
} else {
return (pl);
}
}
/*
* demote a free large page to it's constituent pages
*/
void
{
int mnode;
}
}