/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
*/
#include <sys/sysmacros.h>
#include <sys/mem_cage.h>
#include <vm/seg_kmem.h>
#include <sys/mem_config.h>
extern pri_t maxclsyspri;
#ifdef DEBUG
#define KCAGE_STATS
#endif
#ifdef KCAGE_STATS
struct kcage_stats_scan {
/* managed by KCAGE_STAT_* macros */
/* set in kcage_cageout() */
/* set in kcage_invalidate_page() */
/* set in kcage_expand() */
};
struct kcage_stats {
/* managed by KCAGE_STAT_* macros */
/* set in kcage_cageout */
/* set in kcage_expand */
/* set in kcage_freemem_add() */
/* set in kcage_freemem_sub() */
/* set in kcage_create_throttle */
/* set in kcage_cageout_wakeup */
/* managed by KCAGE_STAT_* macros */
};
/*
* No real need for atomics here. For the most part the incs and sets are
* done by the kernel cage thread. There are a few that are done by any
* number of other threads. Those cases are noted by comments.
*/
#define KCAGE_STAT_INCR_SCAN(m) \
#define KCAGE_STAT_NINCR_SCAN(m, v) \
#define KCAGE_STAT_SETZ(m, v) \
if (kcage_stats.m == 0) kcage_stats.m = (v)
#define KCAGE_STAT_SET_SCAN(m, v) \
#define KCAGE_STAT_SETZ_SCAN(m, v) \
#define KCAGE_STAT_INC_SCAN_INDEX \
#define KCAGE_STAT_INIT_SCAN_INDEX \
#else /* KCAGE_STATS */
#define KCAGE_STAT_INCR(v)
#define KCAGE_STAT_NINCR(m, v)
#define KCAGE_STAT_INCR_SCAN(v)
#define KCAGE_STAT_NINCR_SCAN(m, v)
#define KCAGE_STAT_SET(m, v)
#define KCAGE_STAT_SETZ(m, v)
#define KCAGE_STAT_SET_SCAN(m, v)
#define KCAGE_STAT_SETZ_SCAN(m, v)
#define KCAGE_STAT_INC_SCAN_INDEX
#define KCAGE_STAT_INIT_SCAN_INDEX
#endif /* KCAGE_STATS */
/*
* Cage expansion happens within a range.
*/
struct kcage_glist {
int decr;
};
/*
* The firstfree element is provided so that kmem_alloc can be avoided
* until that cage has somewhere to go. This is not currently a problem
* as early kmem_alloc's use BOP_ALLOC instead of page_create_va.
*/
/*
* Miscellaneous forward references
*/
static struct kcage_glist *kcage_glist_alloc(void);
static void kcage_cageout(void);
/*
* Kernel Memory Cage counters and thresholds.
*/
int kcage_on = 0;
/* when we use lp for kmem we start the cage at a higher initial value */
#ifdef DEBUG
#else
#define KCAGEPAGETS_INC()
#endif
/* kstats to export what pages are currently caged */
/*
* Startup and Dynamic Reconfiguration interfaces.
* kcage_range_add()
* kcage_range_del()
* kcage_range_delete_post_mem_del()
* kcage_range_init()
* kcage_set_thresholds()
*/
/*
* Called from page_get_contig_pages to get the approximate kcage pfn range
* for exclusion from search for contiguous pages. This routine is called
* without kcage_range lock (kcage routines can call page_get_contig_pages
* through page_relocate) and with the assumption, based on kcage_range_add,
* that kcage_current_glist always contain a valid pointer.
*/
int
{
}
/*
* Called from vm_pagelist.c during coalesce to find kernel cage regions
* within an mnode. Looks for the lowest range between lo and hi.
*
* Kernel cage memory is defined between kcage_glist and kcage_current_glist.
* Non-cage memory is defined between kcage_current_glist and list end.
*
* If incage is set, returns the lowest kcage range. Otherwise returns lowest
* non-cage range.
*
* Returns zero on success and nlo, nhi:
* lo <= nlo < nhi <= hi
* Returns non-zero if no overlapping range is found.
*/
int
{
/*
* Reader lock protects the list, but kcage_get_pfn
* running concurrently may advance kcage_current_glist
* and also update kcage_current_glist->curr. Page
* coalesce can handle this race condition.
*/
/* find the range limits in this element */
} else {
}
/* handle overlap */
break;
}
/* check end of kcage */
break;
}
}
/* return non-zero if no overlapping range found */
return (1);
/* return overlapping range */
return (0);
}
void
{
int ret = 0;
if (d == KCAGE_DOWN) {
}
if (ret)
panic("kcage_range_add_internal failed: "
}
if (ret == 0)
}
/*
* Third arg controls direction of growth: 0: increasing pfns,
* 1: decreasing.
*/
static int
{
if (npgs == 0)
return (EINVAL);
return (EINVAL);
new = kcage_glist_alloc();
return (ENOMEM);
}
else
/*
* Any overlapping existing ranges are removed by deleting
* from the new list as we search for the tail.
*/
lpp = &kcage_glist;
int ret;
if (ret != 0)
return (ret);
}
if (kcage_current_glist == NULL) {
}
return (0);
}
int
{
int ret;
return (ret);
}
/*
* Calls to add and delete must be protected by kcage_range_rwlock
*/
static int
{
if (npgs == 0)
return (EINVAL);
return (EINVAL);
/*
* Check if the delete is OK first as a number of elements
* might be involved and it will be difficult to go
* back and undo (can't just add the range back in).
*/
/*
* If there have been no pages allocated from this
* element, we don't need to check it.
*/
continue;
/*
* If the element does not overlap, its OK.
*/
continue;
/*
* Overlapping element: Does the range to be deleted
* overlap the area already used? If so fail.
*/
return (EBUSY);
}
return (EBUSY);
}
}
}
int
{
int ret;
return (ret);
}
/*
* Calls to add and delete must be protected by kcage_range_rwlock.
* This routine gets called after successful Solaris memory
* delete operation from DR post memory delete routines.
*/
static int
{
if (npgs == 0)
return (EINVAL);
return (EINVAL);
}
int
{
int ret;
return (ret);
}
/*
* No locking is required here as the whole operation is covered
* by kcage_range_rwlock writer lock.
*/
static struct kcage_glist *
kcage_glist_alloc(void)
{
} else if (kernel_cage_enable) {
} else {
/*
* On DR supported platforms we allow memory add
* even when kernel cage is disabled. "kcage_arena" is
* created only when kernel cage is enabled.
*/
}
return (new);
}
static void
{
}
static int
{
/* The delete range overlaps this element. */
/* Delete whole element. */
if (lp == kcage_current_glist) {
/* This can never happen. */
}
continue;
}
/* Partial delete. */
/*
* Remove a section from the middle,
* need to allocate a new element.
*/
new = kcage_glist_alloc();
return (ENOMEM);
}
/*
* Tranfser unused range to new.
* Edit lp in place to preserve
* kcage_current_glist.
*/
} else {
}
/* Insert new. */
} else {
/* Delete part of current block. */
} else {
}
}
}
}
return (0);
}
/*
* If lockit is 1, kcage_get_pfn holds the
* reader lock for kcage_range_rwlock.
* Changes to lp->curr can cause race conditions, but
* they are handled by higher level code (see kcage_next_range.)
*/
static pfn_t
{
return (pfn);
break;
}
} else {
break;
}
}
if (lp)
}
if (lockit)
return (pfn);
}
/*
* Walk the physical address space of the cage.
* This routine does not guarantee to return PFNs in the order
* in which they were allocated to the cage. Instead, it walks
* each range as they appear on the growth list returning the PFNs
* range in ascending order.
*
* To begin scanning at lower edge of cage, reset should be nonzero.
* To step through cage, reset should be zero.
*
* PFN_INVALID will be returned when the upper end of the cage is
* reached -- indicating a full scan of the cage has been completed since
* previous reset. PFN_INVALID will continue to be returned until
* kcage_walk_cage is reset.
*
* It is possible to receive a PFN_INVALID result on reset if a growth
* list is not installed or if none of the PFNs in the installed list have
* been allocated to the cage. In otherwords, there is no cage.
*
* Caller need not hold kcage_range_rwlock while calling this function
* as the front part of the list is static - pages never come out of
* the cage.
*
* The caller is expected to only be kcage_cageout().
*/
static pfn_t
{
if (reset)
lp = kcage_glist;
pfn = PFN_INVALID;
}
if (pfn == PFN_INVALID) {
return (PFN_INVALID);
/*
* In this range the cage grows from the highest
* address towards the lowest.
* Arrange to return pfns from curr to lim-1,
* inclusive, in ascending order.
*/
} else {
/*
* In this range the cage grows from the lowest
* address towards the highest.
* Arrange to return pfns from base to curr,
* inclusive, in ascending order.
*/
}
}
/* Don't go beyond the static part of the glist. */
if (lp == kcage_current_glist)
else
pfn = PFN_INVALID;
goto again;
}
} else { /* incrementing pfn */
/* Don't go beyond the static part of the glist. */
if (lp == kcage_current_glist)
else
pfn = PFN_INVALID;
goto again;
}
}
return (pfn++);
}
/*
* Callback functions for to recalc cage thresholds after
*/
/*ARGSUSED*/
static void
{
}
/*ARGSUSED*/
static int
{
/* TODO: when should cage refuse memory delete requests? */
return (0);
}
/*ARGSUSED*/
static void
{
}
};
/*
* This is called before a CPR suspend and after a CPR resume. We have to
* turn off kcage_cageout_ready before a suspend, and turn it back on after a
* restart.
*/
/*ARGSUSED*/
static boolean_t
{
if (code == CB_CODE_CPR_CHKPT) {
kcage_cageout_ready = 0;
return (B_TRUE);
} else if (code == CB_CODE_CPR_RESUME) {
ASSERT(kcage_cageout_ready == 0);
kcage_cageout_ready = 1;
return (B_TRUE);
}
return (B_FALSE);
}
/*
* kcage_recalc_preferred_size() increases initial cage size to improve large
* page availability when lp for kmem is enabled and kpr is disabled
*/
static pgcnt_t
{
if (SEGKMEM_USE_LARGEPAGES && segkmem_reloc == 0) {
if (lpmincage == 0) {
}
(segkmem_kmemlp_max / PAGESIZE));
}
return (preferred_size);
}
/*
* Kcage_init() builds the cage and initializes the cage thresholds.
* The size of the cage is determined by the argument preferred_size.
* or the actual amount of memory, whichever is smaller.
*/
static void
{
extern void page_list_noreloc_startup(page_t *);
/* increase preferred cage size for lp for kmem */
/* Debug note: initialize this now so early expansions can stat */
/*
* Initialize cage thresholds and install kphysm callback.
* If we can't arrange to have the thresholds track with
* available physical memory, then the cage thresholds may
* end up over time at levels that adversly effect system
* performance; so, bail out.
*/
ASSERT(0); /* Catch this in DEBUG kernels. */
return;
}
/*
* Limit startup cage size within the range of kcage_minfree
* and availrmem, inclusively.
*/
/*
* Construct the cage. PFNs are allocated from the glist. It
* is assumed that the list has been properly ordered for the
* platform by the platform code. Typically, this is as simple
* as calling kcage_range_init(phys_avail, decr), where decr is
* 1 if the kernel has been loaded into upper end of physical
* memory, or 0 if the kernel has been loaded at the low end.
*
* Note: it is assumed that we are in the startup flow, so there
* is no reason to grab the page lock.
*/
kcage_freemem = 0;
while (wanted != 0) {
break;
/*
* Set the noreloc state on the page.
* If the page is free and not already
* on the noreloc list then move it.
*/
if (PP_ISNORELOC(pp) == 0)
} else {
}
}
wanted -= 1;
}
/*
* Need to go through and find kernel allocated pages
* and capture them into the Cage. These will primarily
* be pages gotten through boot_alloc().
*/
do {
if (PP_ISNORELOC(pp) == 0) {
}
}
kcage_on = 1;
/*
* CB_CL_CPR_POST_KERNEL is the class that executes from cpr_suspend()
* after the cageout thread is blocked, and executes from cpr_resume()
* before the cageout thread is restarted. By executing in this class,
* we are assured that the kernel cage thread won't miss wakeup calls
* and also CPR's larger kmem_alloc requests will not fail after
* CPR shuts down the cageout kernel thread.
*/
"cageout");
/*
* Coalesce pages to improve large page availability. A better fix
* would to coalesce pages as they are included in the cage
*/
if (SEGKMEM_USE_LARGEPAGES) {
extern void page_freelist_coalesce_all(int mnode);
}
}
}
static int
{
if (rw == KSTAT_WRITE)
return (EACCES);
count = 0;
count++;
}
} else {
count++;
}
}
}
return (0);
}
static int
{
struct memunit {
} *kspmem;
if (rw == KSTAT_WRITE)
return (EACCES);
break;
}
} else {
}
}
}
return (0);
}
void
{
/* TODO: any reason to take more care than this with live editing? */
if (first) {
first = 0;
} else {
}
if (kcage_lotsfree == 0)
if (kcage_minfree == 0)
if (kcage_desfree == 0)
if (kcage_throttlefree == 0)
if (kcage_reserve == 0)
if (kcage_cageout_ready) {
if (kcage_freemem < kcage_desfree)
if (kcage_needfree) {
}
}
}
/*
* Pageout interface:
* kcage_cageout_init()
*/
void
{
if (kcage_on) {
}
}
/*
* VM Interfaces:
* kcage_create_throttle()
* kcage_freemem_add()
* kcage_freemem_sub()
*/
/*
* Wakeup cageout thread and throttle waiting for the number of pages
* requested to become available. For non-critical requests, a
* timeout is added, since freemem accounting is separate from cage
* freemem accounting: it's possible for us to get stuck and not make
* forward progress even though there was sufficient freemem before
* arriving here.
*/
int
{
/*
* Obviously, we can't throttle the cageout thread since
* we depend on it. We also can't throttle the panic thread.
*/
return (KCT_CRIT);
}
/*
* Don't throttle threads which are critical for proper
* vm management if we're above kcage_throttlefree or
* if freemem is very low.
*/
if (NOMEMWAIT()) {
return (KCT_CRIT);
return (KCT_CRIT);
}
}
/*
* Don't throttle real-time threads if kcage_freemem > kcage_reserve.
*/
return (KCT_CRIT);
}
/*
* Cause all other threads (which are assumed to not be
* critical to cageout) to wait here until their request
* can be satisfied. Be a little paranoid and wake the
* kernel cage on each loop through this logic.
*/
if (kcage_cageout_ready) {
kcage_needfree += npages;
kcage_needfree -= npages;
} else {
/*
* NOTE: atomics are used just in case we enter
* mp operation before the cageout thread is ready.
*/
}
return (KCT_CRIT);
}
return (KCT_FAILURE);
} else {
return (KCT_NONCRIT);
}
}
}
return (KCT_NONCRIT);
}
void
{
extern void wakeup_pcgs(void);
wakeup_pcgs(); /* wakeup threads in pcgs() */
if (kcage_needfree != 0 &&
}
}
void
{
if (kcage_freemem < kcage_desfree) {
}
}
/*
* return 0 on failure and 1 on success.
*/
static int
{
return (0);
}
if (szc == 0) {
/*
* The szc of a locked page can only change for pages that are
* non-swapfs (i.e. anonymous memory) file system pages.
*/
return (1);
}
}
return (1);
}
/*
* Attempt to convert page to a caged page (set the P_NORELOC flag).
* If successful and pages is free, move page to the tail of whichever
* list it is on.
* Returns:
* EBUSY page already locked, assimilated but not free.
* ENOMEM page assimilated, but memory too low to relocate. Page not free.
* EAGAIN page not assimilated. Page not free.
* ERANGE page assimilated. Page not root.
* 0 page assimilated. Page free.
* *nfreedp number of pages freed.
* NOTE: With error codes ENOMEM, EBUSY, and 0 (zero), there is no way
* to distinguish between a page that was already a NORELOC page from
* those newly converted to NORELOC pages by this invocation of
* kcage_assimilate_page.
*/
static int
{
if (PP_ISNORELOC(pp)) {
*nfreedp = 0;
return (0);
} else {
return (EBUSY);
}
/*NOTREACHED*/
}
} else {
if (PP_ISNORELOC(pp))
goto check_free_and_return;
} else {
return (EAGAIN);
}
return (EAGAIN);
}
/*
* Need to upgrade the lock on it and set the NORELOC
* bit. If it is free then remove it from the free
* list so that the platform free list code can keep
* NORELOC pages where they should be.
*/
/*
* Before doing anything, get the exclusive lock.
* This may fail (eg ISM pages are left shared locked).
* If the page is free this will leave a hole in the
* cage. There is no solution yet to this.
*/
if (!page_tryupgrade(pp)) {
return (EAGAIN);
}
}
*nfreedp = 1;
return (0);
} else {
return (EAGAIN);
}
} else {
}
}
/*NOTREACHED*/
}
static int
{
int did_something = 0;
/* TODO: we don't really need n any more? */
pgcnt_t n;
/*
* Expand the cage if available cage memory is really low. Calculate
* the amount required to return kcage_freemem to the level of
* kcage_lotsfree, or to satisfy throttled requests, whichever is
* more. It is rare for their sum to create an artificial threshold
* above kcage_lotsfree, but it is possible.
*
* Exit early if expansion amount is equal to or less than zero.
* (<0 is possible if kcage_freemem rises suddenly.)
*
* Exit early when freemem drops below pageout_reserve plus the request.
*/
if (wanted <= 0) {
return (0);
return (0);
}
/*
* Assimilate more pages from the global page pool into the cage.
*/
n = 0; /* number of pages PP_SETNORELOC'd */
nf = 0; /* number of those actually free */
goto terminate;
}
continue;
}
/*
* Sanity check. Skip this pfn if it is
* being deleted.
*/
if (pfn_is_being_deleted(pfn)) {
continue;
}
if (PP_ISNORELOC(pp)) {
continue;
}
case 0: /* assimilated, page is free */
did_something = 1;
n++;
break;
case EBUSY: /* assimilated, page not free */
case ERANGE: /* assimilated, page not root */
did_something = 1;
n++;
break;
case ENOMEM: /* assimilated, but no mem */
did_something = 1;
n++;
goto terminate;
case EAGAIN: /* can't assimilate */
break;
default: /* catch this with debug kernels */
ASSERT(0);
break;
}
}
/*
* Realign cage edge with the nearest physical address
* boundry for big pages. This is done to give us a
* better chance of actually getting usable big pages
* in the cage.
*/
return (did_something);
}
/*
* Relocate page opp (Original Page Pointer) from cage pool to page rpp
* (Replacement Page Pointer) in the global pool. Page opp will be freed
* if relocation is successful, otherwise it is only unlocked.
* On entry, page opp must be exclusively locked and not free.
* *nfreedp: number of pages freed.
*/
static int
{
int result;
if (result == 0) {
while (npgs-- > 0) {
}
return (0); /* success */
}
return (result);
}
/*
* Based on page_invalidate_pages()
*
* Kcage_invalidate_page() uses page_relocate() twice. Both instances
* of use must be updated to match the new page_relocate() when it
* becomes available.
*
* Return result of kcage_relocate_page or zero if page was directly freed.
* *nfreedp: number of pages freed.
*/
static int
{
int result;
#if defined(__sparc)
#endif /* __sparc */
/*
* Is this page involved in some I/O? shared?
* The page_struct_lock need not be acquired to
* examine these fields since the page has an
* "exclusive" lock.
*/
#ifdef KCAGE_STATS
if (result == 0)
#endif
return (result);
}
/*
* Unload the mappings and check if mod bit is set.
*/
#ifdef KCAGE_STATS
if (result == 0)
#endif
return (result);
}
if (!page_try_demote_pages(pp)) {
return (EAGAIN);
}
/* LINTED: constant in conditional context */
*nfreedp = 1;
return (0);
}
/*
* Expand cage only if there is not enough memory to satisfy
* current request. We only do one (complete) scan of the cage.
* Dirty pages and pages with shared mappings are skipped;
* Locked pages (p_lckcnt and p_cowcnt) are also skipped.
* All other pages are freed (if they can be locked).
* This may affect caching of user pages which are in cage by freeing/
* reclaiming them more often. However cage is mainly for kernel (heap)
* pages and we want to keep user pages outside of cage. The above policy
* should also reduce cage expansion plus it should speed up cage mem
* allocations.
*/
static void
{
int did_something;
#ifdef KCAGE_STATS
#endif
callb_generic_cpr, "cageout");
loop:
/*
* Wait here. Sooner or later, kcage_freemem_sub() will notice
* that kcage_freemem is less than kcage_desfree. When it does
* notice, kcage_freemem_sub() will wake us up via call to
* kcage_cageout_wakeup().
*/
#ifdef KCAGE_STATS
scan_start = ddi_get_lbolt();
#endif
if (!kcage_on)
goto loop;
did_something = 0;
PFN_INVALID) {
break;
}
if (start_pfn == PFN_INVALID)
/*
* Did a complete walk of kernel cage, but didn't free
* any pages. If only one cpu is active then
* stop kernel cage walk and try expanding.
*/
break;
}
}
continue;
}
/*
* Do a quick PP_ISNORELOC() and PP_ISFREE test outside
* of the lock. If one is missed it will be seen next
* time through.
*
* Skip non-caged-pages. These pages can exist in the cage
* because, if during cage expansion, a page is
* encountered that is long-term locked the lock prevents the
* expansion logic from setting the P_NORELOC flag. Hence,
* non-caged-pages surrounded by caged-pages.
*/
if (!PP_ISNORELOC(pp)) {
case 0:
did_something = 1;
nfreed);
break;
case EBUSY:
case ERANGE:
did_something = 1;
break;
case EAGAIN:
case ENOMEM:
break;
default:
/* catch this with debug kernels */
ASSERT(0);
break;
}
continue;
} else {
continue;
}
continue;
}
/* P_NORELOC bit should not have gone away. */
continue;
}
continue;
}
did_something = 1;
}
/*
* No need to drop the page lock here.
* Kcage_invalidate_page has done that for us
* either explicitly or through a page_free.
*/
}
}
(void) kcage_expand();
if (kcage_on && kcage_cageout_ready)
goto loop;
/*NOTREACHED*/
}
void
{
if (mutex_tryenter(&kcage_cageout_mutex)) {
if (kcage_cageout_ready) {
/*
* Available cage memory is really low. Time to
* start expanding the cage. However, the
* kernel cage thread is not yet ready to
* do the work. Use *this* thread, which is
* most likely to be t0, to do the work.
*/
(void) kcage_expand();
}
}
/* else, kernel cage thread is already running */
}
void
{
/*
* Once per second we wake up all the threads throttled
* waiting for cage memory, in case we've become stuck
* and haven't made forward progress expanding the cage.
*/
if (kcage_on && kcage_cageout_ready)
}