page.h revision 3cff2f43d248498a56a2055f97f4aa514273fd24
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * CDDL HEADER START
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * The contents of this file are subject to the terms of the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Common Development and Distribution License (the "License").
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * You may not use this file except in compliance with the License.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * See the License for the specific language governing permissions
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * and limitations under the License.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * When distributing Covered Code, include this CDDL HEADER in each
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * If applicable, add the following below this CDDL HEADER, with the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * fields enclosed by brackets "[]" replaced with your own identifying
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * information: Portions Copyright [yyyy] [name of copyright owner]
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * CDDL HEADER END
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Use is subject to license terms.
a32c32954ffe171a0437ff1bc1287e9086f0bd26Chad Kienle/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
a32c32954ffe171a0437ff1bc1287e9086f0bd26Chad Kienle/* All Rights Reserved */
a32c32954ffe171a0437ff1bc1287e9086f0bd26Chad Kienle * University Copyright- Copyright (c) 1982, 1986, 1988
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The Regents of the University of California
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * All Rights Reserved
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * University Acknowledgment- Portions of this document are derived from
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * software developed by the University of California, Berkeley, and its
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * contributors.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#pragma ident "%Z%%M% %I% %E% SMI"
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Types of page locking supported by page_lock & friends.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienletypedef enum {
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * For requesting that page_lock reclaim the page from the free list.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienletypedef enum {
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Callers of page_try_reclaim_lock and page_lock_es can use this flag
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * to get SE_EXCL access before reader/writers are given access.
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * All page_*lock() requests will be denied unless this flag is set in
1e7cf46779cd3edd805fffdd3e5c3c8f5d36a75cBrendan Mmiller * the 'es' parameter.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#endif /* _KERNEL | _KMEMUSER */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Define VM_STATS to turn on all sorts of statistic gathering about
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * the VM layer. By default, it is only turned on when DEBUG is
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * also defined.
a32c32954ffe171a0437ff1bc1287e9086f0bd26Chad Kienle#endif /* DEBUG */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define VM_STAT_COND_ADD(cond, stat) ((void) (!(cond) || (stat)++))
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#endif /* VM_STATS */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Macros to acquire and release the page logical lock.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define page_struct_lock(pp) mutex_enter(&page_llock)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define page_struct_unlock(pp) mutex_exit(&page_llock)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#endif /* _KERNEL */
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * Each physical page has a page structure, which is used to maintain
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * these pages as a cache. A page can be found via a hashed lookup
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * based on the [vp, offset]. If a page has an [vp, offset] identity,
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * then it is entered on a doubly linked circular list off the
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * vnode using the vpnext/vpprev pointers. If the p_free bit
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * is on, then the page is also on a doubly linked circular free
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * list using next/prev pointers. If the "p_selock" and "p_iolock"
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * are held, then the page is currently being read in (exclusive p_selock)
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * or written back (shared p_selock). In this case, the next/prev pointers
a17f398e6526e91a7a3df8dbe0e7f8dd1e8e581dBrendan Mmiller * are used to link the pages together for a consecutive i/o request. If
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * the page is being brought in from its backing store, then other processes
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * will wait for the i/o to complete before attaching to the page since it
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * will have an "exclusive" lock.
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * Each page structure has the locks described below along with
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * the fields they protect:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_selock This is a per-page shared/exclusive lock that is
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * used to implement the logical shared/exclusive
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * lock for each page. The "shared" lock is normally
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * used in most cases while the "exclusive" lock is
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * required to destroy or retain exclusive access to
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * a page (e.g., while reading in pages). The appropriate
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * lock is always held whenever there is any reference
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * to a page structure (e.g., during i/o).
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * (Note that with the addition of the "writer-lock-wanted"
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * semantics (via SE_EWANTED), threads must not acquire
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * multiple reader locks or else a deadly embrace will
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * occur in the following situation: thread 1 obtains a
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * reader lock; next thread 2 fails to get a writer lock
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * but specified SE_EWANTED so it will wait by either
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * blocking (when using page_lock_es) or spinning while
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * retrying (when using page_try_reclaim_lock) until the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * reader lock is released; then thread 1 attempts to
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * get another reader lock but is denied due to
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * SE_EWANTED being set, and now both threads are in a
1e7cf46779cd3edd805fffdd3e5c3c8f5d36a75cBrendan Mmiller * deadly embrace.)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_iolock This is a binary semaphore lock that provides
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * exclusive access to the i/o list links in each
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page structure. It is always held while the page
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * is on an i/o list (i.e., involved in i/o). That is,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * even though a page may be only `shared' locked
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * while it is doing a write, the following fields may
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * change anyway. Normally, the page must be
5848ca5e742e748dfd18fc53d810fa017949f809Jim Mitchener * `exclusively' locked to change anything in it.
5848ca5e742e748dfd18fc53d810fa017949f809Jim Mitchener * The following fields are protected by the global page_llock:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The following lists are protected by the global page_freelock:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_cachelist
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_freelist
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The following, for our purposes, are protected by
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * the global freemem_lock:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * freemem_wait
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * freemem_cv
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The following fields are protected by hat layer lock(s). When a page
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * structure is not mapped and is not associated with a vnode (after a call
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * to page_hashout() for example) the p_nrm field may be modified with out
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * holding the hat layer lock:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The following field is file system dependent. How it is used and
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * the locking strategies applied are up to the individual file system
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * implementation.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The page structure is used to represent and control the system's
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * physical pages. There is one instance of the structure for each
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page that is not permenately allocated. For example, the pages that
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * hold the page structures are permanently held by the kernel
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * and hence do not need page structures to track them. The array
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * of page structures is allocated early on in the kernel's life and
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * is based on the amount of available physical memory.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Each page structure may simultaneously appear on several linked lists.
a1d206a2a22b5cde9b00633ea4472ae0b144d695Brendan Mmiller * The lists are: hash list, free or in i/o list, and a vnode's page list.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Each type of list is protected by a different group of mutexes as described
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The hash list is used to quickly find a page when the page's vnode and
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * offset within the vnode are known. Each page that is hashed is
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * connected via the `p_hash' field. The anchor for each hash is in the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * array `page_hash'. An array of mutexes, `ph_mutex', protects the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * lists anchored by page_hash[]. To either search or modify a given hash
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * list, the appropriate mutex in the ph_mutex array must be held.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The free list contains pages that are `free to be given away'. For
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * efficiency reasons, pages on this list are placed in two catagories:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * pages that are still associated with a vnode, and pages that are not
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * associated with a vnode. Free pages always have their `p_free' bit set,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * free pages that are still associated with a vnode also have their
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * `p_age' bit set. Pages on the free list are connected via their
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * `p_next' and `p_prev' fields. When a page is involved in some sort
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * of i/o, it is not free and these fields may be used to link associated
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * pages together. At the moment, the free list is protected by a
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * single mutex `page_freelock'. The list of free pages still associated
a1d206a2a22b5cde9b00633ea4472ae0b144d695Brendan Mmiller * with a vnode is anchored by `page_cachelist' while other free pages
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * are anchored in architecture dependent ways (to handle page coloring etc.).
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Pages associated with a given vnode appear on a list anchored in the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * vnode by the `v_pages' field. They are linked together with
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * `p_vpnext' and `p_vpprev'. The field `p_offset' contains a page's
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * offset within the vnode. The pages on this list are not kept in
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * offset order. These lists, in a manner similar to the hash lists,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * are protected by an array of mutexes called `vph_hash'. Before
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * searching or modifying this chain the appropriate mutex in the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * vph_hash[] array must be held.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Again, each of the lists that a page can appear on is protected by a
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * mutex. Before reading or writing any of the fields comprising the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * list, the appropriate lock must be held. These list locks should only
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * be held for very short intervals.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * In addition to the list locks, each page structure contains a
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * shared/exclusive lock that protects various fields within it.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * To modify one of these fields, the `p_selock' must be exclusively held.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * To read a field with a degree of certainty, the lock must be at least
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * held shared.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Removing a page structure from one of the lists requires holding
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * the appropriate list lock and the page's p_selock. A page may be
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * prevented from changing identity, being freed, or otherwise modified
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * by acquiring p_selock shared.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * To avoid deadlocks, a strict locking protocol must be followed. Basically
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * there are two cases: In the first case, the page structure in question
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * is known ahead of time (e.g., when the page is to be added or removed
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * from a list). In the second case, the page structure is not known and
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * must be found by searching one of the lists.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * When adding or removing a known page to one of the lists, first the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page must be exclusively locked (since at least one of its fields
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * will be modified), second the lock protecting the list must be acquired,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * third the page inserted or deleted, and finally the list lock dropped.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The more interesting case occures when the particular page structure
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * is not known ahead of time. For example, when a call is made to
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_lookup(), it is not known if a page with the desired (vnode and
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * offset pair) identity exists. So the appropriate mutex in ph_mutex is
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * acquired, the hash list searched, and if the desired page is found
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * an attempt is made to lock it. The attempt to acquire p_selock must
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * not block while the hash list lock is held. A deadlock could occure
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * if some other process was trying to remove the page from the list.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The removing process (following the above protocol) would have exclusively
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * locked the page, and be spinning waiting to acquire the lock protecting
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * the hash list. Since the searching process holds the hash list lock
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * and is waiting to acquire the page lock, a deadlock occurs.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The proper scheme to follow is: first, lock the appropriate list,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * search the list, and if the desired page is found either use
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_trylock() (which will not block) or pass the address of the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * list lock to page_lock(). If page_lock() can not acquire the page's
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * lock, it will drop the list lock before going to sleep. page_lock()
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * returns a value to indicate if the list lock was dropped allowing the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * calling program to react appropriately (i.e., retry the operation).
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * If the list lock was dropped before the attempt at locking the page
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * was made, checks would have to be made to ensure that the page had
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * not changed identity before its lock was obtained. This is because
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * the interval between dropping the list lock and acquiring the page
9f1e7c4b37140496d6fb5ad0a3ae507e4f9db37cAndi Egloff * lock is indeterminate.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * In addition, when both a hash list lock (ph_mutex[]) and a vnode list
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * lock (vph_mutex[]) are needed, the hash list lock must be acquired first.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The routine page_hashin() is a good example of this sequence.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * This sequence is ASSERTed by checking that the vph_mutex[] is not held
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * just before each acquisition of one of the mutexs in ph_mutex[].
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * So, as a quick summary:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * pse_mutex[]'s protect the p_selock and p_cv fields.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_selock protects the p_free, p_age, p_vnode, p_offset and p_hash,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * ph_mutex[]'s protect the page_hash[] array and its chains.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * vph_mutex[]'s protect the v_pages field and the vp page chains.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * First lock the page, then the hash chain, then the vnode chain. When
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * this is not possible `trylocks' must be used. Sleeping while holding
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * any of these mutexes (p_selock is not a mutex) is not allowed.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * field reading writing ordering
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * ======================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_vnode p_selock(E,S) p_selock(E)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_hash p_selock(E,S) p_selock(E) && p_selock, ph_mutex
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * ph_mutex[]
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_vpnext p_selock(E,S) p_selock(E) && p_selock, vph_mutex
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_vpprev vph_mutex[]
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * When the p_free bit is set:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_next p_selock(E,S) p_selock(E) && p_selock,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_prev page_freelock page_freelock
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * When the p_free bit is not set:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_next p_selock(E,S) p_selock(E) && p_selock, p_iolock
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_prev p_iolock
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_selock pse_mutex[] pse_mutex[] can`t acquire any
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_cv other mutexes or
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * sleep while holding
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_lckcnt p_selock(E,S) p_selock(E) &&
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_cowcnt page_llock
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_nrm hat layer lock hat layer lock
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * E----> exclusive version of p_selock.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * S----> shared version of p_selock.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Global data structures and variable:
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * field reading writing ordering
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_hash[] ph_mutex[] ph_mutex[] can hold this lock
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * before acquiring
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * a vph_mutex or
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * vp->v_pages vph_mutex[] vph_mutex[] can only acquire
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * a pse_mutex while
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * holding this lock.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_cachelist page_freelock page_freelock can't acquire any
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_freelist page_freelock page_freelock
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * freemem freemem_lock freemem_lock can't acquire any
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * freemem_wait other mutexes while
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * freemem_cv holding this mutex.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * =====================================================================
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Page relocation, PG_NORELOC and P_NORELOC.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Pages may be relocated using the page_relocate() interface. Relocation
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * involves moving the contents and identity of a page to another, free page.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * To relocate a page, the SE_EXCL lock must be obtained. The way to prevent
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * a page from being relocated is to hold the SE_SHARED lock (the SE_EXCL
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * lock must not be held indefinitely). If the page is going to be held
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * SE_SHARED indefinitely, then the PG_NORELOC hint should be passed
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * to page_create_va so that pages that are prevented from being relocated
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * can be managed differently by the platform specific layer.
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * Pages locked in memory using page_pp_lock (p_lckcnt/p_cowcnt != 0)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * are guaranteed to be held in memory, but can still be relocated
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * providing the SE_EXCL lock can be obtained.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The P_NORELOC bit in the page_t.p_state field is provided for use by
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * the platform specific code in managing pages when the PG_NORELOC
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * hint is used.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Memory delete and page locking.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The set of all usable pages is managed using the global page list as
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * implemented by the memseg structure defined below. When memory is added
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * or deleted this list changes. Additions to this list guarantee that the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * list is never corrupt. In order to avoid the necessity of an additional
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * lock to protect against failed accesses to the memseg being deleted and,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * more importantly, the page_ts, the memseg structure is never freed and the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_t virtual address space is remapped to a page (or pages) of
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * zeros. If a page_t is manipulated while it is p_selock'd, or if it is
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * locked indirectly via a hash or freelist lock, it is not possible for
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * memory delete to collect the page and so that part of the page list is
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * prevented from being deleted. If the page is referenced outside of one
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * of these locks, it is possible for the page_t being referenced to be
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * deleted. Examples of this are page_t pointers returned by
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_numtopp_nolock, page_first and page_next. Providing the page_t
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * is re-checked after taking the p_selock (for p_vnode != NULL), the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * remapping to the zero pages will be detected.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Page size (p_szc field) and page locking.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_szc field of free pages is changed by free list manager under freelist
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * locks and is of no concern to the rest of VM subsystem.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * p_szc changes of allocated anonymous (swapfs) can only be done only after
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * exclusively locking all constituent pages and calling hat_pageunload() on
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * each of them. To prevent p_szc changes of non free anonymous (swapfs) large
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * pages it's enough to either lock SHARED any of constituent pages or prevent
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * hat_pageunload() by holding hat level lock that protects mapping lists (this
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * method is for hat code only)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * To increase (promote) p_szc of allocated non anonymous file system pages
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * one has to first lock exclusively all involved constituent pages and call
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * hat_pageunload() on each of them. To prevent p_szc promote it's enough to
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * either lock SHARED any of constituent pages that will be needed to make a
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * large page or prevent hat_pageunload() by holding hat level lock that
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * protects mapping lists (this method is for hat code only).
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * To decrease (demote) p_szc of an allocated non anonymous file system large
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page one can either use the same method as used for changeing p_szc of
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * anonymous large pages or if it's not possible to lock all constituent pages
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * exclusively a different method can be used. In the second method one only
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * has to exclusively lock one of constituent pages but then one has to
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller * acquire further locks by calling page_szc_lock() and
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * hat_page_demote(). hat_page_demote() acquires hat level locks and then
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * demotes the page. This mechanism relies on the fact that any code that
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * needs to prevent p_szc of a file system large page from changeing either
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * locks all constituent large pages at least SHARED or locks some pages at
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * least SHARED and calls page_szc_lock() or uses hat level page locks.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Demotion using this method is implemented by page_demote_vp_pages().
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Please see comments in front of page_demote_vp_pages(), hat_page_demote()
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * and page_szc_lock() for more details.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Lock order: p_selock, page_szc_lock, ph_mutex/vph_mutex/freelist,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * hat level locks.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienletypedef struct page {
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle u_offset_t p_offset; /* offset into vnode for this page */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle struct vnode *p_vnode; /* vnode that this page is named by */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle selock_t p_selock; /* shared/exclusive lock on the page */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uint_t p_vpmref; /* vpm ref - index of the vpmap_t */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle struct page *p_hash; /* hash by [vnode, offset] */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle struct page *p_vpnext; /* next page in vnode list */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle struct page *p_vpprev; /* prev page in vnode list */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle struct page *p_next; /* next page in free/intrans lists */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle struct page *p_prev; /* prev page in free/intrans lists */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle ushort_t p_lckcnt; /* number of locks on page data */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle ushort_t p_cowcnt; /* number of copy on write lock */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle kcondvar_t p_cv; /* page struct's condition var */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uchar_t p_fsdata; /* file system dependent byte */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uchar_t p_nrm; /* non-cache, ref, mod readonly bits */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uchar_t p_embed; /* x86 - changes p_mapping & p_index */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uchar_t p_index; /* MPSS mapping info. Not used on x86 */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uchar_t p_toxic; /* page has an unrecoverable error */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle void *p_mapping; /* hat specific translation info */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uint_t p_sharepad; /* pad for growing p_share */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uint_t p_msresv_1; /* reserved for future use */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uint_t p_kpmref; /* number of kpm mapping sharers */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle struct kpme *p_kpmelist; /* kpm specific mapping info */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle /* index of entry in p_map when p_embed is set */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uint64_t p_msresv_2; /* page allocation debugging */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Page hash table is a power-of-two in size, externally chained
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * through the hash field. PAGE_HASHAVELEN is the average length
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * desired for this chain, from which the size of the page_hash
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * table is derived at boot time and stored in the kernel variable
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_hashsz. In the hash function it is given by PAGE_HASHSZ.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * PAGE_HASH_FUNC returns an index into the page_hash[] array. This
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * index is also used to derive the mutex that protects the chain.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * In constructing the hash function, first we dispose of unimportant bits
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * (page offset from "off" and the low 3 bits of "vp" which are zero for
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * struct alignment). Then shift and sum the remaining bits a couple times
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * in order to get as many source bits from the two source values into the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * resulting hashed value. Note that this will perform quickly, since the
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * shifting/summing are fast register to register operations with no additional
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * memory references).
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The amount to use for the successive shifts in the hash function below.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The actual value is LOG2(PH_TABLE_SIZE), so that as many bits as
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * possible will filter thru PAGE_HASH_FUNC() and PAGE_HASH_MUTEX().
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle ((uintptr_t)(off) >> (PAGESHIFT + PH_SHIFT_SIZE)) + \
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle ((uintptr_t)(vp) >> (3 + 2 * PH_SHIFT_SIZE))) & \
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The page hash value is re-hashed to an index for the ph_mutex array.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * For 64 bit kernels, the mutex array is padded out to prevent false
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * sharing of cache sub-blocks (64 bytes) of adjacent mutexes.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * For 32 bit kernels, we don't want to waste kernel address space with
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * padding, so instead we rely on the hash function to introduce skew of
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * adjacent vnode/offset indexes (the left shift part of the hash function).
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Since sizeof (kmutex_t) is 8, we shift an additional 3 to skew to a different
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * 64 byte sub-block.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienletypedef struct pad_mutex {
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle &(ph_mutex[((x) + ((x) >> VP_SHIFT) + ((x) << 3)) & \
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Flags used while creating pages.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PG_MATCH_COLOR 0x0008 /* SUPPORTED by free list routines */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PG_NORELOC 0x0010 /* Non-relocatable alloc hint. */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle /* Page must be PP_ISNORELOC */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PG_PANIC 0x0020 /* system will panic if alloc fails */
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmiller#define PG_PUSHPAGE 0x0040 /* alloc may use reserve */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * When p_selock has the SE_EWANTED bit set, threads waiting for SE_EXCL
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * access are given priority over all other waiting threads.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PAGE_LOCKED(pp) (((pp)->p_selock & ~SE_EWANTED) != 0)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PAGE_SHARED(pp) (((pp)->p_selock & ~SE_EWANTED) > 0)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle ((se) == SE_EXCL ? PAGE_EXCL(pp) : PAGE_SHARED(pp))
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienleextern kmutex_t page_llock; /* page logical lock mutex */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienleextern pgcnt_t total_pages; /* total pages in the system */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Variables controlling locking of physical memory.
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerextern pgcnt_t pages_pp_maximum; /* tuning: lock + claim <= max */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienleextern void init_pages_pp_maximum(void);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle/* page_list_{add,sub} flags */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle/* which list */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle/* where on list */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle/* called from */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Page frame operations.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlepage_t *page_lookup(struct vnode *, u_offset_t, se_t);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlepage_t *page_lookup_create(struct vnode *, u_offset_t, se_t, page_t *,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlepage_t *page_lookup_nowait(struct vnode *, u_offset_t, se_t);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienleint page_exists_physcontig(vnode_t *, u_offset_t, uint_t, page_t *[]);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienleint page_exists_forreal(struct vnode *, u_offset_t, uint_t *);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlepage_t *page_create(struct vnode *, u_offset_t, size_t, uint_t);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienleint page_alloc_pages(struct vnode *, struct seg *, caddr_t, page_t **,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlepage_t *page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes,
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle uint_t flags, struct seg *seg, caddr_t vaddr, void *arg);
79d12e9411155c9e79c08df9bb6412b2862e6064Brendan Mmillerpage_t *page_create_va(struct vnode *, u_offset_t, size_t, uint_t,
79d12e9411155c9e79c08df9bb6412b2862e6064Brendan Mmillerint page_create_wait(size_t npages, uint_t flags);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlevoid free_vp_pages(struct vnode *, u_offset_t, size_t);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlevoid page_rename(page_t *, struct vnode *, u_offset_t);
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerint page_hashin(page_t *, struct vnode *, u_offset_t, kmutex_t *);
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerint page_num_hashin(pfn_t, struct vnode *, u_offset_t);
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerpage_t *page_get_freelist(struct vnode *, u_offset_t, struct seg *,
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerpage_t *page_get_cachelist(struct vnode *, u_offset_t, struct seg *,
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillervoid page_list_break(page_t **, page_t **, size_t);
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerint page_lock(page_t *, se_t, kmutex_t *, reclaim_t);
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerint page_lock_es(page_t *, se_t, kmutex_t *, reclaim_t, int);
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerint page_try_reclaim_lock(page_t *, se_t, int);
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillervoid page_pp_useclaim(page_t *, page_t *, uint_t);
5c6fc9459842796234027c8bb8f58886d69ebc8fBrendan Mmillerpage_t *page_next_scan_large(page_t *, ulong_t *, void **);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Page relocation interfaces. page_relocate() is generic.
117941f80637bf0fe66cbb3a444430ef88f629a7Jim Mitchener * page_get_replacement_page() is provided by the PSM.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * page_free_replacement_page() is generic.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienleint page_relocate(page_t **, page_t **, int, int, spgcnt_t *, struct lgrp *);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienleint do_page_relocate(page_t **, page_t **, int, spgcnt_t *, struct lgrp *);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlepage_t *page_get_replacement_page(page_t *, struct lgrp *, uint_t);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlevoid page_mark_migrate(struct seg *, caddr_t, size_t, struct anon_map *,
117941f80637bf0fe66cbb3a444430ef88f629a7Jim Mitchenervoid page_migrate(struct seg *, caddr_t, page_t **, pgcnt_t);
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Tell the PIM we are adding physical memory
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienlevoid add_physmem_cb(page_t *, pfn_t); /* callback for page_t part */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * hw_page_array[] is configured with hardware supported page sizes by
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * platform specific code.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienletypedef struct {
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle/* page_get_replacement page flags */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PGR_SAMESZC 0x1 /* only look for page size same as orig */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PGR_NORELOC 0x2 /* allocate a P_NORELOC page */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#endif /* _KERNEL */
9f1e7c4b37140496d6fb5ad0a3ae507e4f9db37cAndi Egloff * Constants used for the p_iolock_state
9f1e7c4b37140496d6fb5ad0a3ae507e4f9db37cAndi Egloff * Constants used for page_release status
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * The p_state field holds what used to be the p_age and p_free
1e7cf46779cd3edd805fffdd3e5c3c8f5d36a75cBrendan Mmiller * bits. These fields are protected by p_selock (see above).
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define P_NORELOC 0x40 /* Page is non-relocatable */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define P_MIGRATE 0x20 /* Migrate page on next touch */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define P_SWAP 0x10 /* belongs to vnode that is V_ISSWAP */
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PP_ISAGED(pp) (((pp)->p_state & P_FREE) && \
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PP_ISNORELOC(pp) ((pp)->p_state & P_NORELOC)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PP_ISNORELOCKERNEL(pp) (PP_ISNORELOC(pp) && PP_ISKVP(pp))
9f1e7c4b37140496d6fb5ad0a3ae507e4f9db37cAndi Egloff#define PP_ISMIGRATE(pp) ((pp)->p_state & P_MIGRATE)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PP_SETFREE(pp) ((pp)->p_state = ((pp)->p_state & ~P_MIGRATE) \
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PP_SETNORELOC(pp) ((pp)->p_state |= P_NORELOC)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PP_SETMIGRATE(pp) ((pp)->p_state |= P_MIGRATE)
9f1e7c4b37140496d6fb5ad0a3ae507e4f9db37cAndi Egloff#define PP_CLRFREE(pp) ((pp)->p_state &= ~P_FREE)
9f1e7c4b37140496d6fb5ad0a3ae507e4f9db37cAndi Egloff#define PP_CLRNORELOC(pp) ((pp)->p_state &= ~P_NORELOC)
9f1e7c4b37140496d6fb5ad0a3ae507e4f9db37cAndi Egloff#define PP_CLRMIGRATE(pp) ((pp)->p_state &= ~P_MIGRATE)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle#define PP_CLRSWAP(pp) ((pp)->p_state &= ~P_SWAP)
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * Flags for page_t p_toxic, for tracking memory hardware errors.
37f9df7d5b474a12668813f98992dceb7c7feacbChad Kienle * These flags are OR'ed into p_toxic with page_settoxic() to track which
* beginning of page_retire.c too.
typedef struct kpm_page {
} kpm_page_t;
* Note: khl_lock offset changes must be reflected in sfmmu_asm.s
typedef struct kpm_hlk {
} kpm_hlk_t;
typedef struct kpm_spage {
} kpm_spage_t;
* Note: kshl_lock offset changes must be reflected in sfmmu_asm.s
typedef struct kpm_shlk {
} kpm_shlk_t;
struct memseg {
#if defined(__sparc)
union _mseg_un {
} mseg_un;
void build_pfn_hash();
#ifdef __cplusplus