page_lock.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* VM - page locking primitives
*/
#include <sys/lockstat.h>
#include <sys/condvar_impl.h>
#include <vm/seg_enum.h>
/*
* This global mutex is for logical page locking.
* The following fields in the page structure are protected
* by this lock:
*
* p_lckcnt
* p_cowcnt
*/
/*
* This is a global lock for the logical page free list. The
* logical free list, in this implementation, is maintained as two
* separate physical lists - the cache list and the free list.
*/
/*
* The hash table, page_hash[], the p_selock fields, and the
* list of pages associated with vnodes are protected by arrays of mutexes.
*
* Unless the hashes are changed radically, the table sizes must be
* a power of two. Also, we typically need more mutexes for the
* vnodes since these locks are occasionally held for long periods.
* And since there seem to be two special vnodes (kvp and swapvp),
* we make room for private mutexes for them.
*
* The pse_mutex[] array holds the mutexes to protect the p_selock
* fields of all page_t structures.
*
* PAGE_SE_MUTEX(pp) returns the address of the appropriate mutex
* when given a pointer to a page_t.
*
* PSE_TABLE_SIZE must be a power of two. One could argue that we
* should go to the trouble of setting it up at run time and base it
* on memory size rather than the number of compile time CPUs.
*
* XX64 We should be using physmem size to calculate PSE_TABLE_SIZE,
* PSE_SHIFT, PIO_SHIFT.
*
* These might break in 64 bit world.
*/
#define PAGE_SE_MUTEX(pp) \
#define PAGE_IO_MUTEX(pp) \
#define PSZC_MTX_TABLE_SIZE 128
#define PSZC_MTX_TABLE_SHIFT 7
#define PAGE_SZC_MUTEX(_pp) \
/*
* The vph_mutex[] array holds the mutexes to protect the vnode chains,
* (i.e., the list of pages anchored by v_pages and connected via p_vpprev
* and p_vpnext).
*
* The page_vnode_mutex(vp) function returns the address of the appropriate
* mutex from this array given a pointer to a vnode. It is complicated
* by the fact that the kernel's vnode and the swapfs vnode are referenced
* frequently enough to warrent their own mutexes.
*
* The VP_HASH_FUNC returns the index into the vph_mutex array given
* an address of a vnode.
*/
/*
* XX64 VPH_TABLE_SIZE and VP_HASH_FUNC might break in 64 bit world.
* Need to review again.
*/
#define VP_HASH_FUNC(vp) \
& (VPH_TABLE_SIZE - 1))
/*
* Initialize the locks used by the Virtual Memory Management system.
*/
void
{
}
/*
* At present we only use page ownership to aid debugging, so it's
* OK if the owner field isn't exact. In the 32-bit world two thread ids
* can map to the same owner because we just 'or' in 0x80000000 and
* then clear the second highest bit, so that (for example) 0x2faced00
* and 0xafaced00 both map to 0xafaced00.
* In the 64-bit world, p_selock may not be large enough to hold a full
* thread pointer. If we ever need precise ownership (e.g. if we implement
* priority inheritance for page locks) then p_selock should become a
* uintptr_t and SE_WRITER should be -((uintptr_t)curthread >> 2).
*/
#define SE_READER 1
/*
* A page that is deleted must be marked as such using the
* page_lock_delete() function. The page must be exclusively locked.
* The SE_DELETED marker is put in p_selock when this function is called.
* SE_DELETED must be distinct from any SE_WRITER value.
*/
#ifdef VM_STATS
#endif /* VM_STATS */
#ifdef VM_STATS
#endif /* VM_STATS */
/*
*
* Returns 1 on success and locks the page appropriately.
* 0 on failure and does not lock the page.
*
* If `lock' is non-NULL, it will be dropped and reacquired in the
* failure case. This routine can block, and if it does
* it will always return a failure since the page identity [vp, off]
* or state may have changed.
*/
int
{
}
/*
* With the addition of reader-writer lock semantics to page_lock_es,
* callers wanting an exclusive (writer) lock may prevent shared-lock
* (reader) starvation by setting the es parameter to SE_EXCL_WANTED.
* In this case, when an exclusive lock cannot be acquired, p_selock's
* SE_EWANTED bit is set.
* This bit, along with the se and es parameters, are used to decide
* if the requested lock should be granted:
*
* Lock wanted SE_EXCL_WANTED p_selock/SE_EWANTED Action
* ---------- -------------- ------------------- ---------
* SE_EXCL no dont-care/1 deny lock
* SE_SHARED not applicable shared/0 grant
* SE_SHARED not applicable unlocked/0 grant
* SE_SHARED not applicable shared/1 deny
* SE_SHARED not applicable unlocked/1 deny
*
* Note: the code grants an exclusive lock to the caller and clears
* SE_EWANTED whenever p_selock is unlocked, regardless of the SE_EWANTED
* bit's value. This was deemed acceptable as we are not concerned about
* exclusive-lock starvation. If this ever becomes an issue, a priority or
* fifo mechanism should also be implemented.
*/
int
{
int retval;
int upgraded;
int reclaim_it;
upgraded = 0;
reclaim_it = 0;
/*
* Current uses of 'es':
* es == 1 page_lookup_create will attempt page relocation
* es == SE_EXCL_WANTED caller wants SE_EWANTED set (eg. delete
* memory thread); this prevents reader-starvation of waiting
* writer thread(s).
*/
}
reclaim_it = 1;
/*
* This is an interesting situation.
*
* Remember that p_free can only change if
* p_selock < 0.
* p_free does not depend on our holding `pse'.
* And, since we hold `pse', p_selock can not change.
* So, if p_free changes on us, the page is already
* exclusively held, and we would fail to get p_selock
* regardless.
*
* We want to avoid getting the share
* lock on a free page that needs to be reclaimed.
* It is possible that some other thread has the share
* lock and has left the free page on the cache list.
* pvn_vplist_dirty() does this for brief periods.
* If the se_share is currently SE_EXCL, we will fail
* to acquire p_selock anyway. Blocking is the
* right thing to do.
* If we need to reclaim this page, we must get
* exclusive access to it, force the upgrade now.
* Again, we will fail to acquire p_selock if the
* page is not free and block.
*/
upgraded = 1;
}
}
/*
* if the caller wants a writer lock (but did not
* specify exclusive access), and there is a pending
* writer that wants exclusive access, return failure
*/
retval = 0;
/* this clears our setting of the SE_EWANTED bit */
retval = 1;
} else {
/* page is locked */
if (es == SE_EXCL_WANTED) {
/* set the SE_EWANTED bit */
}
retval = 0;
}
} else {
retval = 0;
/* readers are not allowed when excl wanted */
retval = 1;
}
}
}
if (retval == 0) {
return (retval);
}
#ifdef VM_STATS
if (upgraded) {
}
#endif
if (lock) {
}
/*
* Now, wait for the page to be unlocked and
* release the lock protecting p_cv and p_selock.
*/
/*
* The page identity may have changed while we were
* blocked. If we are willing to depend on "pp"
* still pointing to a valid page structure (i.e.,
* assuming page structures are not dynamically allocated
* or freed), we could try to lock the page if its
* identity hasn't changed.
*
* This needs to be measured, since we come back from
* cv_wait holding pse (the expensive part of this
* operation) we might as well try the cheap part.
* Though we would also have to confirm that dropping
* `lock' did not cause any grief to the callers.
*/
if (lock) {
}
} else {
/*
* We have the page lock.
* If we needed to reclaim the page, and the page
* needed reclaiming (ie, it was free), then we
* have the page exclusively locked. We may need
* to downgrade the page.
*/
/*
* We now hold this page's lock, either shared or
* exclusive. This will prevent its identity from changing.
* The page, however, may or may not be free. If the caller
* requested, and it is free, go reclaim it from the
* free list. If the page can't be reclaimed, return failure
* so that the caller can start all over again.
*
* NOTE:page_reclaim() releases the page lock (p_selock)
* if it can't be reclaimed.
*/
if (reclaim_it) {
retval = 0;
} else {
if (upgraded) {
}
}
}
}
return (retval);
}
/*
* Clear the SE_EWANTED bit from p_selock. This function allows
* callers of page_lock_es and page_try_reclaim_lock to clear
* their setting of this bit if they decide they no longer wish
* to gain exclusive access to the page. Currently only
* delete_memory_thread uses this when the delete memory
* operation is cancelled.
*/
void
{
}
/*
* Read the comments inside of page_lock_es() carefully.
*
* SE_EXCL callers specifying es == SE_EXCL_WANTED will cause the
* SE_EWANTED bit of p_selock to be set when the lock cannot be obtained.
* This is used by threads subject to reader-starvation (eg. memory delete).
*
* When a thread using SE_EXCL_WANTED does not obtain the SE_EXCL lock,
* it is expected that it will retry at a later time. Threads that will
* not retry the lock *must* call page_lock_clr_exclwanted to clear the
* SE_EWANTED bit. (When a thread using SE_EXCL_WANTED obtains the lock,
* the bit is cleared.)
*/
int
{
}
if (old >= 0) {
/* readers are not allowed when excl wanted */
if (!(old & SE_EWANTED)) {
return (1);
}
}
return (0);
}
/*
* The page is free, so we really want SE_EXCL (below)
*/
}
/*
* The caller wants a writer lock. We try for it only if
* SE_EWANTED is not set, or if the caller specified
* SE_EXCL_WANTED.
*/
if ((old & ~SE_EWANTED) == 0) {
/* this clears out our setting of the SE_EWANTED bit */
return (1);
}
}
if (es == SE_EXCL_WANTED) {
/* page is locked, set the SE_EWANTED bit */
}
return (0);
}
/*
* Returns 1 on success, 0 on failure.
*/
int
{
/* fail if a thread wants exclusive access */
return (0);
}
return (1);
}
} else {
return (1);
}
}
return (0);
}
/*
* who might be waiting for it.
*/
void
{
} else if (old < 0) {
} else {
}
}
/*
* Try to upgrade the lock on the page from a "shared" to an
* "exclusive" lock. Since this upgrade operation is done while
* holding the mutex protecting this page, no one else can acquire this page's
* lock and change the page. Thus, it is safe to drop the "shared"
* lock and attempt to acquire the "exclusive" lock.
*
* Returns 1 on success, 0 on failure.
*/
int
{
/* no threads want exclusive access, try upgrade */
/* convert to exclusive lock */
return (1);
}
}
return (0);
}
/*
* Downgrade the "exclusive" lock on the page to a "shared" lock
* while holding the mutex protecting this page's p_selock field.
*/
void
{
int excl_waiting;
}
void
{
}
/*
* Implement the io lock for pages
*/
void
{
pp->p_iolock_state = 0;
}
/*
* Acquire the i/o lock on a page.
*/
void
{
}
}
/*
* Release the i/o lock on a page.
*/
void
{
}
/*
* Try to acquire the i/o lock on a page without blocking.
* Returns 1 on success, 0 on failure.
*/
int
{
return (0);
return (0);
}
return (1);
}
/*
* Assert that the i/o lock on a page is held.
* Returns 1 on success, 0 on failure.
*/
int
{
}
/*
* Wrapper exported to kernel routines that are built
* platform-independent (the macro is platform-dependent;
* the size of vph_mutex[] is based on NCPU).
*
* Note that you can do stress testing on this by setting the
* variable page_vnode_mutex_stress to something other than
* zero in a DEBUG kernel in a debugger after loading the kernel.
* Setting it after the kernel is running may not work correctly.
*/
#ifdef DEBUG
static int page_vnode_mutex_stress = 0;
#endif
kmutex_t *
{
return (&vph_mutex[VPH_TABLE_SIZE + 0]);
#ifdef DEBUG
if (page_vnode_mutex_stress != 0)
return (&vph_mutex[0]);
#endif
}
kmutex_t *
{
return (PAGE_SE_MUTEX(pp));
}
#ifdef VM_STATS
#endif
/*
* Find, take and return a mutex held by hat_page_demote().
* Called by page_demote_vp_pages() before hat_page_demote() call and by
* routines that want to block hat_page_demote() but can't do it
* via locking all constituent pages.
*
* Return NULL if p_szc is 0.
*
* It should only be used for pages that can be demoted by hat_page_demote()
* i.e. non swapfs file system pages. The logic here is lifted from
* sfmmu_mlspl_enter() except there's no need to worry about p_szc increase
* since the page is locked and not free.
*
* Hash of the root page is used to find the lock.
* To find the root in the presense of hat_page_demote() chageing the location
* of the root this routine relies on the fact that hat_page_demote() changes
* root last.
*
* If NULL is returned pp's p_szc is guaranteed to be 0. If non NULL is
* returned pp's p_szc may be any value.
*/
kmutex_t *
{
if (pszc == 0) {
VM_STAT_ADD(pszclck_stat[0]);
return (NULL);
}
/* The lock lives in the root page */
/*
* since p_szc can only decrease if pp == rootpp
* rootpp will be always the same i.e we have the right root
* regardless of rootpp->p_szc.
* If location of pp's root didn't change after we took
* the lock we have the right root. return mutex hashed off it.
*/
return (mtx);
}
/*
* root location changed because page got demoted.
* locate the new root.
*/
goto again;
}
/*
* current hat_page_demote not done yet.
* wait for it to finish.
*/
goto again;
}
int
{
return (MUTEX_HELD(mtx));
}