hat_sfmmu.c revision fedab560fb18c85777c255ea9f445ffaf6830d30
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* VM - Hardware Address Translation management for Spitfire MMU.
*
* This file implements the machine specific hardware translation
* needed by the VM system. The machine independent interface is
* and data structures are described in <vm/hat_sfmmu.h>.
*
* The hat layer manages the address translation hardware as a cache
* driven by calls from the higher levels in the VM system.
*/
#include <vm/hat_sfmmu.h>
#include <sys/sysmacros.h>
#include <sys/machparam.h>
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <vm/seg_kmem.h>
#include <sys/vm_machparam.h>
#include <sys/machtrap.h>
#include <sys/machlock.h>
#include <sys/cpu_module.h>
#include <sys/prom_debug.h>
#include <sys/mem_config.h>
#include <sys/mem_cage.h>
#include <vm/xhat_sfmmu.h>
#include <vm/mach_kpm.h>
#if defined(SF_ERRATA_57)
extern caddr_t errata57_limit;
#endif
(sizeof (int64_t)))
#define HBLK_RESERVE_CNT 128
#define HBLK_RESERVE_MIN 20
static kmutex_t freehblkp_lock;
static int freehblkcnt;
static kmutex_t hblk_reserve_lock;
static kthread_t *hblk_reserve_thread;
static nucleus_hblk8_info_t nucleus_hblk8;
static nucleus_hblk1_info_t nucleus_hblk1;
/*
* SFMMU specific hat functions
*/
void hat_pagecachectl(struct page *, int);
/* flags for hat_pagecachectl */
#define HAT_CACHE 0x1
#define HAT_UNCACHE 0x2
#define HAT_TMPNC 0x4
/*
* Flag to allow the creation of non-cacheable translations
* to system memory. It is off by default. At the moment this
* flag is used by the ecache error injector. The error injector
* will turn it on when creating such a translation then shut it
* off when it's finished.
*/
int sfmmu_allow_nc_trans = 0;
/*
* Flag to disable large page support.
* value of 1 => disable all large pages.
* bits 1, 2, and 3 are to disable 64K, 512K and 4M pages respectively.
*
* For example, use the value 0x4 to disable 512K pages.
*
*/
#define LARGE_PAGES_OFF 0x1
/*
* a process would page fault indefinitely if it tried to
* access a 512K page.
*/
int disable_large_pages = 0;
int disable_auto_large_pages = 0;
/*
* Private sfmmu data structures for hat management
*/
static struct kmem_cache *sfmmuid_cache;
static struct kmem_cache *mmuctxdom_cache;
/*
* Private sfmmu data structures for tsb management
*/
static struct kmem_cache *sfmmu_tsbinfo_cache;
static struct kmem_cache *sfmmu_tsb8k_cache;
static vmem_t *kmem_tsb_arena;
/*
* sfmmu static variables for hmeblk resource management.
*/
static struct kmem_cache *sfmmu8_cache;
static struct kmem_cache *sfmmu1_cache;
static struct kmem_cache *pa_hment_cache;
/*
* private data for ism
*/
static struct kmem_cache *ism_blk_cache;
static struct kmem_cache *ism_ment_cache;
#define ISMID_STARTADDR NULL
/*
* Whether to delay TLB flushes and use Cheetah's flush-all support
* when removing contexts from the dirty list.
*/
int delay_tlb_flush;
/*
* ``hat_lock'' is a hashed mutex lock for protecting sfmmu TSB lists,
* The lock is hashed on the sfmmup since the case where we need to lock
* all processes is rare but does occur (e.g. we need to unload a shared
* mapping from all processes using the mapping). We have a lot of buckets,
* and each slab of sfmmu_t's can use about a quarter of them, giving us
* a fairly good distribution without wasting too much space and overhead
* when we have to grab them all.
*/
/*
* Hash algorithm optimized for a small number of slabs.
* 7 is (highbit((sizeof sfmmu_t)) - 1)
* This hash algorithm is based upon the knowledge that sfmmu_t's come from a
* kmem_cache, and thus they will be sequential within that cache. In
* addition, each new slab will have a different "color" up to cache_maxcolor
* which will skew the hashing for each successive slab which is allocated.
* If the size of sfmmu_t changed to a larger size, this algorithm may need
* to be revisited.
*/
#define TSB_HASH_SHIFT_BITS (7)
#ifdef DEBUG
int tsb_hash_debug = 0;
(tsb_hash_debug ? &hat_lock[0] : \
#else /* DEBUG */
#endif /* DEBUG */
/* sfmmu_replace_tsb() return codes. */
typedef enum tsb_replace_rc {
/*
* Flags for TSB allocation routines.
*/
#define TSB_ALLOC 0x01
#define TSB_FORCEALLOC 0x02
#define TSB_GROW 0x04
#define TSB_SHRINK 0x08
#define TSB_SWAPIN 0x10
/*
* Support for HAT callbacks.
*/
#define SFMMU_MAX_RELOC_CALLBACKS 10
static id_t sfmmu_cb_nextid = 0;
static id_t sfmmu_tsb_cb_id;
struct sfmmu_callback *sfmmu_cb_table;
/*
* Kernel page relocation is enabled by default for non-caged
* kernel pages. This has little effect unless segkmem_reloc is
* set, since by default kernel memory comes from inside the
* kernel cage.
*/
int hat_kpr_enabled = 1;
/*
* Enable VA->PA translation sanity checking on DEBUG kernels.
* Disabled by default. This is incompatible with some
* drivers (error injector, RSM) so if it breaks you get
* to keep both pieces.
*/
int hat_check_vtop = 0;
/*
* Private sfmmu routines (prototypes)
*/
caddr_t, int);
static void sfmmu_hblks_list_purge(struct hme_blk **);
static struct hme_blk *sfmmu_hblk_steal(int);
static int sfmmu_steal_this_hblk(struct hmehash_bucket *,
struct hme_blk *);
uint_t);
uint_t);
caddr_t, int);
static void sfmmu_tteload_release_hashbucket(struct hmehash_bucket *);
#ifdef VAC
#endif
static void sfmmu_get_ctx(sfmmu_t *);
static void sfmmu_free_sfmmu(sfmmu_t *);
#ifdef VAC
static void sfmmu_page_cache(page_t *, int, int, int);
#endif
pfn_t, int, int, int, int);
pfn_t, int);
static void sfmmu_tlb_range_demap(demap_range_t *);
static void sfmmu_invalidate_ctx(sfmmu_t *);
static void sfmmu_sync_mmustate(sfmmu_t *);
sfmmu_t *);
static void sfmmu_tsb_free(struct tsb_info *);
static void sfmmu_tsbinfo_free(struct tsb_info *);
sfmmu_t *);
static int sfmmu_select_tsb_szc(pgcnt_t);
#ifdef VAC
void sfmmu_cache_flush(pfn_t, int);
void sfmmu_cache_flushcolor(int, pfn_t);
#endif
static int sfmmu_idcache_constructor(void *, void *, int);
static void sfmmu_idcache_destructor(void *, void *);
static int sfmmu_hblkcache_constructor(void *, void *, int);
static void sfmmu_hblkcache_destructor(void *, void *);
static void sfmmu_hblkcache_reclaim(void *);
struct hmehash_bucket *);
static void sfmmu_rm_large_mappings(page_t *, int);
static void hat_lock_init(void);
static void hat_kstat_init(void);
static void sfmmu_check_page_sizes(sfmmu_t *, int);
int fnd_mapping_sz(page_t *);
extern void sfmmu_setup_tsbinfo(sfmmu_t *);
extern void sfmmu_clear_utsbinfo(void);
static void sfmmu_ctx_wrap_around(mmu_ctx_t *);
/* kpm globals */
#ifdef DEBUG
/*
* Enable trap level tsbmiss handling
*/
int kpm_tsbmtl = 1;
/*
* Flush the TLB on kpm mapout. Note: Xcalls are used (again) for the
* required TLB shootdowns in this case, so handle w/ care. Off by default.
*/
int kpm_tlb_flush;
#endif /* DEBUG */
#ifdef DEBUG
static void sfmmu_check_hblk_flist();
#endif
/*
* Semi-private sfmmu data structures. Some of them are initialize in
* startup or in hat_init. Some of them are private but accessed by
* assembly code or mach_sfmmu.c
*/
int uhmehash_num; /* # of buckets in user hash table */
int khmehash_num; /* # of buckets in kernel hash table */
#define DEFAULT_NUM_CTXS_PER_MMU 8192
int cache; /* describes system cache */
int ktsb_szcode; /* kernel 8k-indexed tsb size code */
int ktsb_sz; /* kernel 8k-indexed tsb size */
int ktsb4m_szcode; /* kernel 4m-indexed tsb size code */
int ktsb4m_sz; /* kernel 4m-indexed tsb size */
int kpm_tsbsz; /* kernel seg_kpm 4M TSB size code */
int kpmsm_tsbsz; /* kernel seg_kpm 8K TSB size code */
#ifndef sun4v
int dtlb_resv_ttenum; /* index in TLB of first reserved TTE */
#endif /* sun4v */
/*
* Size to use for TSB slabs. Future platforms that support page sizes
* larger than 4M may wish to change these values, and provide their own
* assembly macros for building and decoding the TSB base register contents.
* Note disable_large_pages will override the value set here.
*/
/* largest TSB size to grow to, will be smaller on smaller memory systems */
int tsb_max_growsize = UTSB_MAX_SZCODE;
/*
* Tunable parameters dealing with TSB policies.
*/
/*
* This undocumented tunable forces all 8K TSBs to be allocated from
* the kernel heap rather than from the kmem_tsb_default_arena arenas.
*/
#ifdef DEBUG
int tsb_forceheap = 0;
#endif /* DEBUG */
/*
* Decide whether to use per-lgroup arenas, or one global set of
* TSB arenas. The default is not to break up per-lgroup, since
* most platforms don't recognize any tangible benefit from it.
*/
int tsb_lgrp_affinity = 0;
/*
* Used for growing the TSB based on the process RSS.
* tsb_rss_factor is based on the smallest TSB, and is
* shifted by the TSB size to determine if we need to grow.
* The default will grow the TSB if the number of TTEs for
* this page size exceeds 75% of the number of TSB entries,
* which should _almost_ eliminate all conflict misses
* (at the expense of using up lots and lots of memory).
*/
#define SELECT_TSB_SIZECODE(pgcnt) ( \
#define TSB_OK_SHRINK() \
#define TSB_OK_GROW() \
int enable_tsb_rss_sizing = 1;
int tsb_rss_factor = (int)TSB_RSS_FACTOR;
/* which TSB size code to use for new address spaces or if rss sizing off */
int default_tsb_size = TSB_8K_SZCODE;
#define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32
#ifdef DEBUG
static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */
static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */
static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */
static int tsb_alloc_fail_mtbf = 0;
static int tsb_alloc_count = 0;
#endif /* DEBUG */
/* if set to 1, will remap valid TTEs when growing TSB. */
int tsb_remap_ttes = 1;
/*
* If we have more than this many mappings, allocate a second TSB.
* This default is chosen because the I/D fully associative TLBs are
* assumed to have at least 8 available entries. Platforms with a
* larger fully-associative TLB could probably override the default.
*/
int tsb_sectsb_threshold = 8;
/*
* kstat data
*/
struct sfmmu_global_stat sfmmu_global_stat;
struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat;
/*
* Global data
*/
#ifdef DEBUG
#endif
/* sfmmu locking operations */
static int sfmmu_mlspl_held(struct page *, int);
void sfmmu_page_exit(kmutex_t *);
int sfmmu_page_spl_held(struct page *);
/* sfmmu internal locking operations - accessed directly */
static hatlock_t *
static hatlock_t *
static void sfmmu_hat_exit(hatlock_t *);
static void sfmmu_hat_lock_all(void);
static void sfmmu_hat_unlock_all(void);
static void sfmmu_ismhat_enter(sfmmu_t *, int);
static void sfmmu_ismhat_exit(sfmmu_t *, int);
/*
* Array of mutexes protecting a page's mapping list and p_nrm field.
*
* The hash function looks complicated, but is made up so that:
*
* "pp" not shifted, so adjacent pp values will hash to different cache lines
*
* "pp" >> mml_shift, incorporates more source bits into the hash result
*
* "& (mml_table_size - 1), should be faster than using remainder "%"
*
* Hopefully, mml_table, mml_table_size and mml_shift are all in the same
* cacheline, since they get declared next to each other below. We'll trust
* ld not to do something random.
*/
#ifdef DEBUG
int mlist_hash_debug = 0;
#else /* !DEBUG */
#endif /* !DEBUG */
/*
* SPL_HASH was improved to avoid false cache line sharing
*/
#define SPL_TABLE_SIZE 128
(SPL_TABLE_SIZE - 1))
/*
* hat_unload_callback() will group together callbacks in order
* to avoid xt_sync() calls. This is the maximum size of the group.
*/
#define MAX_CB_ADDR 32
static char *mmu_ctx_kstat_names[] = {
"mmu_ctx_tsb_exceptions",
"mmu_ctx_tsb_raise_exception",
"mmu_ctx_wrap_around",
};
/*
* Wrapper for vmem_xalloc since vmem_create only allows limited
* parameters for vm_source_alloc functions. This function allows us
* to specify alignment consistent with the size of the object being
* allocated.
*/
static void *
{
}
/* Common code for setting tsb_alloc_hiwater. */
/*
* Set tsb_max_growsize to allow at most all of physical memory to be mapped by
* a single TSB. physmem is the number of physical pages so we need physmem 8K
* TTEs to represent all those physical pages. We round this up by using
* 1<<highbit(). To figure out which size code to use, remember that the size
* code is just an amount to shift the smallest TSB size to get the size of
* this TSB. So we subtract that size, TSB_START_SIZE, from highbit() (or
* highbit() - 1) to get the size code for the smallest TSB that can represent
* all of physical memory, while erring on the side of too much.
*
* If the computed size code is less than the current tsb_max_growsize, we set
* tsb_max_growsize to the computed size code. In the case where the computed
* size code is greater than tsb_max_growsize, we have these restrictions that
* apply to increasing tsb_max_growsize:
* 1) TSBs can't grow larger than the TSB slab size
* 2) TSBs can't grow larger than UTSB_MAX_SZCODE.
*/
#define SFMMU_SET_TSB_MAX_GROWSIZE(pages) { \
int i, szc; \
\
i--; /* 2^n case, round down */ \
szc = i - TSB_START_SIZE; \
if (szc < tsb_max_growsize) \
tsb_max_growsize = szc; \
else if ((szc > tsb_max_growsize) && \
}
/*
* Given a pointer to an sfmmu and a TTE size code, return a pointer to the
* tsb_info which handles that TTE size.
*/
/*
* Return the number of mappings present in the HAT
* for a particular process and page size.
*/
(sfmmup)->sfmmu_iblk? \
/*
* Macro to use to unload entries from the TSB.
* It has knowledge of which page sizes get replicated in the TSB
* and will call the appropriate unload routine for the appropriate size.
*/
{ \
} else { \
} \
}
/* Update tsb_alloc_hiwater after memory is configured. */
/*ARGSUSED*/
static void
{
/* Assumes physmem has already been updated. */
}
/*
* Update tsb_alloc_hiwater before memory is deleted. We'll do nothing here
* and update tsb_alloc_hiwater and tsb_max_growsize after the memory is
* deleted.
*/
/*ARGSUSED*/
static int
{
return (0);
}
/* Update tsb_alloc_hiwater after memory fails to be unconfigured. */
/*ARGSUSED*/
static void
{
/*
* Whether the delete was cancelled or not, just go ahead and update
* tsb_alloc_hiwater and tsb_max_growsize.
*/
}
static kphysm_setup_vector_t sfmmu_update_tsb_vec = {
KPHYSM_SETUP_VECTOR_VERSION, /* version */
sfmmu_update_tsb_post_add, /* post_add */
sfmmu_update_tsb_pre_del, /* pre_del */
sfmmu_update_tsb_post_del /* post_del */
};
/*
* HME_BLK HASH PRIMITIVES
*/
/*
* Enter a hme on the mapping list for page pp.
* When large pages are more prevalent in the system we might want to
* keep the mapping list in ascending order by the hment size. For now,
* small pages are more frequent, so don't slow it down.
*/
{ \
\
} else { \
/* EMPTY */ \
} \
}
/*
* Enter a hme on the mapping list for page pp.
* If we are unmapping a large translation, we need to make sure that the
* change is reflect in the corresponding bit of the p_index field.
*/
{ \
\
panic("hme_remove - no mappings"); \
} \
\
membar_stst(); /* ensure previous stores finish */ \
\
\
} else { \
} \
\
} \
\
/* zero out the entry */ \
\
/* remove mappings for remainder of large pg */ \
} \
}
/*
* This function returns the hment given the hme_blk and a vaddr.
* It assumes addr has already been checked to belong to hme_blk's
* range.
*/
{ \
int index; \
}
/*
* Version of HBLKTOHME that also returns the index in hmeblkp
* of the hment.
*/
{ \
\
} else \
idx = 0; \
\
}
/*
* Disable any page sizes not supported by the CPU
*/
void
{
int i;
for (i = TTE8K; i < max_mmu_page_sizes; i++) {
extern int disable_text_largepages;
extern int disable_initdata_largepages;
if ((mmu_exported_pagesize_mask & (1 << i)) == 0) {
disable_large_pages |= (1 << i);
disable_ism_large_pages |= (1 << i);
disable_text_largepages |= (1 << i);
disable_initdata_largepages |= (1 << i);
} else {
}
}
/*
* Initialize mmu-specific large page sizes.
*/
if ((mmu_page_sizes == max_mmu_page_sizes) &&
(&mmu_large_pages_disabled)) {
}
}
/*
* Initialize the hardware address translation structures.
*/
void
hat_init(void)
{
int i;
/*
* Hardware-only bits in a TTE
*/
/* Initialize the hash locks */
for (i = 0; i < khmehash_num; i++) {
}
for (i = 0; i < uhmehash_num; i++) {
}
khmehash_num--; /* make sure counter starts from 0 */
uhmehash_num--; /* make sure counter starts from 0 */
/*
* Allocate context domain structures.
*
* A platform may choose to modify max_mmu_ctxdoms in
* set_platform_defaults(). If a platform does not define
* a set_platform_defaults() or does not choose to modify
* max_mmu_ctxdoms, it gets one MMU context domain for every CPU.
*
* For sun4v, there will be one global context domain, this is to
* avoid the ldom cpu substitution problem.
*
* For all platforms that have CPUs sharing MMUs, this
* value must be defined.
*/
if (max_mmu_ctxdoms == 0) {
#ifndef sun4v
#else /* sun4v */
max_mmu_ctxdoms = 1;
#endif /* sun4v */
}
/* mmu_ctx_t is 64 bytes aligned */
/*
* MMU context domain initialization for the Boot CPU.
* This needs the context domains array allocated above.
*/
/*
* Intialize ism mapping list lock.
*/
/*
* Each sfmmu structure carries an array of MMU context info
* structures, one per context domain. The size of this array depends
* on the maximum number of context domains. So, the size of the
* sfmmu structure varies per platform.
*
* sfmmu is allocated from static arena, because trap
* handler at TL > 0 is not allowed to touch kernel relocatable
* memory. sfmmu's alignment is changed to 64 bytes from
* default 8 bytes, as the lower 6 bits will be used to pass
* pgcnt to vtag_flush_pgcnt_tl1.
*/
/*
* Since we only use the tsb8k cache to "borrow" pages for TSBs
* from the heap when low on memory or when TSB_FORCEALLOC is
* specified, don't use magazines to cache them--we want to return
* them to the system as quickly as possible.
*/
/*
* Set tsb_alloc_hiwater to 1/tsb_alloc_hiwater_factor of physical
* memory, which corresponds to the old static reserve for TSBs.
* tsb_alloc_hiwater_factor defaults to 32. This caps the amount of
* memory we'll allocate for TSB slabs; beyond this point TSB
* allocations will be taken from the kernel heap (via
* sfmmu_tsb8k_cache) and will be throttled as would any other kmem
* consumer.
*/
if (tsb_alloc_hiwater_factor == 0) {
}
/* Set tsb_max_growsize. */
/*
* On smaller memory systems, allocate TSB memory in smaller chunks
* than the default 4M slab size. We also honor disable_large_pages
* here.
*
* The trap handlers need to be patched with the final slab shift,
* since they need to be able to construct the TSB pointer at runtime.
*/
if (tsb_max_growsize <= TSB_512K_SZCODE)
break;
}
tsb_slab_ttesz = sz;
if (tsb_max_growsize > maxtsb)
/*
* Set up memory callback to update tsb_alloc_hiwater and
* tsb_max_growsize.
*/
i = kphysm_setup_func_register(&sfmmu_update_tsb_vec, (void *) 0);
ASSERT(i == 0);
/*
* kmem_tsb_arena is the source from which large TSB slabs are
* drawn. The quantum of this arena corresponds to the largest
* TSB size we can dynamically allocate for user processes.
* Currently it must also be a supported page size since we
* use exactly one translation entry to map each slab page.
*
* The per-lgroup kmem_tsb_default_arena arenas are the arenas from
* which most TSBs are allocated. Since most TSB allocations are
* typically 8K we have a kmem cache we stack on top of each
* kmem_tsb_default_arena to speed up those allocations.
*
* Note the two-level scheme of arenas is required only
* because vmem_create doesn't allow us to specify alignment
* requirements. If this ever changes the code could be
* simplified to use only one level of arenas.
*/
0, VM_SLEEP);
if (tsb_lgrp_affinity) {
char s[50];
for (i = 0; i < NLGRPS_MAX; i++) {
(void) sprintf(s, "kmem_tsb_lgrp%d", i);
(void) sprintf(s, "sfmmu_tsb_lgrp%d_cache", i);
kmem_tsb_default_arena[i], 0);
}
} else {
VM_SLEEP | VM_BESTFIT);
kmem_tsb_default_arena[0], 0);
}
sfmmu_hblkcache_reclaim, (void *)HME8BLK_SZ,
NULL, (void *)HME1BLK_SZ,
/*
* We grab the first hat for the kernel,
*/
/*
* Initialize hblk_reserve.
*/
#ifndef UTSB_PHYS
/*
* Reserve some kernel virtual address space for the locked TTEs
* that allow us to probe the TSB from TL>0.
*/
#endif
#ifdef VAC
/*
* The big page VAC handling code assumes VAC
* will not be bigger than the smallest big
* page- which is 64K.
*/
}
#endif
(void) xhat_init();
/*
* Initialize relocation locks. kpr_suspendlock is held
* at PIL_MAX to prevent interrupts from pinning the holder
* of a suspended TTE which may access it leading to a
* deadlock condition.
*/
}
/*
* Initialize locking for the hat layer, called early during boot.
*/
static void
{
int i;
/*
* initialize the array of mutexes protecting a page's mapping
* list and p_nrm field.
*/
for (i = 0; i < mml_table_sz; i++)
if (kpm_enable) {
for (i = 0; i < kpmp_table_sz; i++) {
}
}
/*
* Initialize array of mutex locks that protects sfmmu fields and
* TSB lists.
*/
for (i = 0; i < SFMMU_NUM_LOCK; i++)
NULL);
}
#define SFMMU_KERNEL_MAXVA \
/*
* Allocate a hat structure.
* Called when an address space first uses a hat.
*/
struct hat *
{
int i;
sfmmup->sfmmu_flags = 0;
sfmmup->sfmmu_cext = 0;
sfmmup->sfmmu_clrstart = 0;
/*
* hat_kern_setup() will call sfmmu_init_ktsbinfo()
* to setup tsb_info for ksfmmup.
*/
} else {
/*
* Just set to invalid ctx. When it faults, it will
* get a valid ctx. This would avoid the situation
* where we get a ctx, but it gets stolen and then
* we fault when we try to run and so have to get
* another ctx.
*/
sfmmup->sfmmu_cext = 0;
/* initialize original physical page coloring bin */
#ifdef DEBUG
if (tsb_random_size) {
/* chose a random tsb size for stress testing */
} else
#endif /* DEBUG */
}
ASSERT(max_mmu_ctxdoms > 0);
for (i = 0; i < max_mmu_ctxdoms; i++) {
}
for (i = 0; i < max_mmu_page_sizes; i++) {
sfmmup->sfmmu_ttecnt[i] = 0;
sfmmup->sfmmu_ismttecnt[i] = 0;
}
sfmmup->sfmmu_ismhat = 0;
} else {
}
sfmmup->sfmmu_free = 0;
sfmmup->sfmmu_rmstat = 0;
return (sfmmup);
}
/*
* Create per-MMU context domain kstats for a given MMU ctx.
*/
static void
{
} else {
}
}
/*
* plat_cpuid_to_mmu_ctx_info() is a platform interface that returns MMU
* context domain information for a given CPU. If a platform does not
* specify that interface, then the function below is used instead to return
* default information. The defaults are as follows:
*
* - For sun4u systems there's one MMU context domain per CPU.
* This default is used by all sun4u systems except OPL. OPL systems
* provide platform specific interface to map CPU ids to MMU ids
* because on OPL more than 1 CPU shares a single MMU.
* Note that on sun4v, there is one global context domain for
* the entire system. This is to avoid running into potential problem
* with ldom physical cpu substitution feature.
* - The number of MMU context IDs supported on any CPU in the
* system is 8K.
*/
/*ARGSUSED*/
static void
{
#ifndef sun4v
#else /* sun4v */
#endif /* sun4v */
}
/*
* Called during CPU initialization to set the MMU context-related information
* for a CPU.
*
* cpu_lock serializes accesses to mmu_ctxs and mmu_saved_gnum.
*/
void
{
if (&plat_cpuid_to_mmu_ctx_info == NULL)
else
/* Each mmu_ctx is cacheline aligned. */
(void *)ipltospl(DISP_LEVEL));
/*
* Globally for lifetime of a system,
* gnum must always increase.
* mmu_saved_gnum is protected by the cpu_lock.
*/
} else {
}
/*
* The mmu_lock is acquired here to prevent races with
* the wrap-around code.
*/
}
/*
* Called to perform MMU context-related cleanup for a CPU.
*/
void
{
/*
* The mmu_lock is acquired here to prevent races with
* the wrap-around code.
*/
/* mmu_saved_gnum is protected by the cpu_lock. */
return;
}
}
/*
* Hat_setup, makes an address space context the current active one.
* In sfmmu this translates to setting the secondary context with the
* corresponding context.
*/
void
{
/* Init needs some special treatment. */
/*
* Make sure that we have
* 1. a TSB
* 2. a valid ctx that doesn't get stolen after this point.
*/
/*
* Swap in the TSB. hat_init() allocates tsbinfos without
* TSBs, but we need one for init, since the kernel does some
* special things to set up its stack and needs the TSB to
* resolve page faults.
*/
} else {
/*
* sfmmu_setctx_sec takes <pgsz|cnum> as a parameter,
* pagesize bits don't matter in this case since we are passing
* INVALID_CONTEXT to it.
*/
}
}
/*
* Free all the translation resources for the specified address space.
* Called from as_free when an address space is being destroyed.
*/
void
{
}
void
{
int i;
if (sfmmup->sfmmu_ismhat) {
for (i = 0; i < mmu_page_sizes; i++) {
sfmmup->sfmmu_ttecnt[i] = 0;
sfmmup->sfmmu_ismttecnt[i] = 0;
}
} else {
/* EMPTY */
}
if (sfmmup->sfmmu_rmstat) {
}
}
}
/*
* Set up any translation structures, for the specified address space,
* that are needed or preferred when the process is being swapped in.
*/
/* ARGSUSED */
void
{
}
/*
* Free all of the translation resources, for the specified address space,
* that can be freed while the process is swapped out. Called from as_swapout.
* Also, free up the ctx that this process was using.
*/
void
{
struct hmehash_bucket *hmebp;
int i;
struct free_tsb {
}; /* free list of TSBs */
/*
* There is no way to go from an as to all its translations in sfmmu.
* Here is one of the times when we take the big hit and traverse
* the hash looking for hme_blks to free up. Not only do we free up
* this as hme_blks but all those that are free. We are obviously
* swapping because we need memory so let's free up as much
* as we can.
*
* because:
* 1) we free the ctx we're using and throw away the TSB(s);
* 2) processes aren't runnable while being swapped out.
*/
for (i = 0; i <= UHMEHASH_SZ; i++) {
prevpa = 0;
while (hmeblkp) {
NULL, HAT_UNLOAD);
}
} else {
}
}
}
/*
* Now free up the ctx so that others can reuse it.
*/
/*
* Free TSBs, but not tsbinfos, and set SWAPPED flag.
* If TSBs were never swapped in, just return.
* This implies that we don't support partial swapping
* of TSBs -- either all are swapped out, or none are.
*
* We must hold the HAT lock here to prevent racing with another
* thread trying to unmap TTEs from the TSB or running the post-
* relocator after relocating the TSB's memory. Unfortunately, we
* can't free memory while holding the HAT lock or we could
* deadlock, so we build a list of TSBs to be freed after marking
* the tsbinfos as swapped out and free them after dropping the
* lock.
*/
return;
}
/*
* Cast the TSB into a struct free_tsb and put it on the free
* list.
*/
} else {
}
/*
* Zero out the TTE to clear the valid bit.
* Note we can't use a value like 0xbad because we want to
* ensure diagnostic bits are NEVER set on TTEs that might
* be loaded. The intent is to catch any invalid access
* to the swapped TSB, such as a thread running with a valid
* context without first calling sfmmu_tsb_swapin() to
* allocate TSB memory.
*/
}
/* Now we can drop the lock and free the TSB memory. */
}
}
/*
* Duplicate the translations of an as into another newas
*/
/* ARGSUSED */
int
{
if (flag == HAT_DUP_COW) {
panic("hat_dup: HAT_DUP_COW not supported");
}
return (0);
}
/*
* Set up addr to map to page pp with protection prot.
* As an optimization we also load the TSB with the
* corresponding tte but it is no big deal if the tte gets kicked out.
*/
void
{
panic("hat_memload: loading a mapping to free page %p",
(void *)pp);
}
if (hat->sfmmu_xhat_provider) {
return;
}
if (flags & ~SFMMU_LOAD_ALLFLAG)
flags & ~SFMMU_LOAD_ALLFLAG);
if (hat->sfmmu_rmstat)
#if defined(SF_ERRATA_57)
!(flags & HAT_LOAD_SHARE)) {
" page executable");
}
#endif
/*
* Check TSB and TLB page sizes.
*/
if ((flags & HAT_LOAD_SHARE) == 0) {
}
}
/*
* hat_devload can be called to map real memory (e.g.
* for memory, it will be unable to get a shared lock on the
* page (because someone else has it exclusively) and will
* pass dp = NULL. If tteload doesn't get a non-NULL
* page pointer it can't cache memory.
*/
void
{
int use_lgpg = 0;
if (hat->sfmmu_xhat_provider) {
return;
}
if (len == 0)
panic("hat_devload: zero len");
if (flags & ~SFMMU_LOAD_ALLFLAG)
flags & ~SFMMU_LOAD_ALLFLAG);
#if defined(SF_ERRATA_57)
!(flags & HAT_LOAD_SHARE)) {
" page executable");
}
#endif
/*
* If it's a memory page find its pp
*/
} else {
panic("hat_memload: loading "
"a mapping to free page %p",
(void *)pp);
}
panic("hat_memload: loading a mapping "
"to unlocked relocatable page %p",
(void *)pp);
}
}
}
if (hat->sfmmu_rmstat)
if (flags & HAT_LOAD_NOCONSIST) {
use_lgpg = 1;
}
if (!pf_is_memory(pfn)) {
use_lgpg = 1;
switch (attr & HAT_ORDER_MASK) {
case HAT_STRICTORDER:
case HAT_UNORDERED_OK:
/*
* we set the side effect bit for all non
* memory mappings unless merging is ok
*/
attr |= SFMMU_SIDEFFECT;
break;
case HAT_MERGING_OK:
case HAT_LOADCACHING_OK:
case HAT_STORECACHING_OK:
break;
default:
panic("hat_devload: bad attr");
break;
}
}
while (len) {
if (!use_lgpg) {
flags);
len -= MMU_PAGESIZE;
addr += MMU_PAGESIZE;
pfn++;
continue;
}
/*
* Note that 32M/256M page sizes are not (yet) supported.
*/
if ((len >= MMU_PAGESIZE4M) &&
flags);
len -= MMU_PAGESIZE4M;
addr += MMU_PAGESIZE4M;
} else if ((len >= MMU_PAGESIZE512K) &&
flags);
len -= MMU_PAGESIZE512K;
addr += MMU_PAGESIZE512K;
} else if ((len >= MMU_PAGESIZE64K) &&
flags);
len -= MMU_PAGESIZE64K;
addr += MMU_PAGESIZE64K;
} else {
flags);
len -= MMU_PAGESIZE;
addr += MMU_PAGESIZE;
pfn++;
}
}
/*
* Check TSB and TLB page sizes.
*/
if ((flags & HAT_LOAD_SHARE) == 0) {
}
}
/*
* Map the largest extend possible out of the page array. The array may NOT
* be in order. The largest possible mapping a page can have
* is specified in the p_szc field. The p_szc field
* cannot change as long as there any mappings (large or small)
* to any of the pages that make up the large page. (ie. any
* the page free list manager). The array
* should consist of properly aligned contigous pages that are
* part of a big page for a large mapping to be created.
*/
void
{
int ttesz;
int large_pages_disable;
if (hat->sfmmu_xhat_provider) {
return;
}
if (hat->sfmmu_rmstat)
#if defined(SF_ERRATA_57)
!(flags & HAT_LOAD_SHARE)) {
"user page executable");
}
#endif
/* Get number of pages */
if (flags & HAT_LOAD_SHARE) {
} else {
}
return;
}
/*
* Check if this page size is disabled.
*/
continue;
/*
* At this point we have enough pages and
* we know the virtual address and the pfn
* are properly aligned. We still need
* to check for physical contiguity but since
* it is very likely that this is the case
* we will assume they are so and undo
* the request if necessary. It would
* be great if we could get a hint flag
* like HAT_CONTIG which would tell us
* the pages are contigous for sure.
*/
break;
}
}
}
/*
* We were not able to map array using a large page
* batch a hmeblk or fraction at a time.
*/
& (NHMENTS-1);
numpg);
}
}
if (npgs) {
}
/*
* Check TSB and TLB page sizes.
*/
if ((flags & HAT_LOAD_SHARE) == 0) {
}
}
/*
* Function tries to batch 8K pages into the same hme blk.
*/
static void
{
struct hmehash_bucket *hmebp;
int index;
while (npgs) {
/*
* Acquire the hash bucket.
*/
/*
* Find the hment block.
*/
do {
/*
* Make the tte.
*/
/*
* Add the translation.
*/
/*
* Goto next page.
*/
pps++;
npgs--;
/*
* Goto next address.
*/
vaddr += MMU_PAGESIZE;
/*
* Don't crossover into a different hmentblk.
*/
(NHMENTS-1));
/*
* Release the hash bucket.
*/
}
}
/*
* Construct a tte for a page:
*
* tte_valid = 1
* tte_size2 = size & TTE_SZ2_BITS (Panther and Olympus-C only)
* tte_size = size
* tte_nfo = attr & HAT_NOFAULT
* tte_ie = attr & HAT_STRUCTURE_LE
* tte_hmenum = hmenum
* tte_pahi = pp->p_pagenum >> TTE_PASHIFT;
* tte_palo = pp->p_pagenum & TTE_PALOMASK;
* tte_ref = 1 (optimization)
* tte_wr_perm = attr & PROT_WRITE;
* tte_no_sync = attr & HAT_NOSYNC
* tte_lock = attr & SFMMU_LOCKTTE
* tte_cp = !(attr & SFMMU_UNCACHEPTTE)
* tte_cv = !(attr & SFMMU_UNCACHEVTTE)
* tte_e = attr & SFMMU_SIDEFFECT
* tte_priv = !(attr & PROT_USER)
* tte_hwwr = if nosync is set and it is writable we set the mod bit (opt)
* tte_glb = 0
*/
void
{
if (TTE_IS_NOSYNC(ttep)) {
if (TTE_IS_WRITABLE(ttep)) {
}
}
panic("sfmmu_memtte: can't set both NFO and EXEC bits");
}
}
/*
* This function will add a translation to the hme_blk and allocate the
* hme_blk if one does not exist.
* If a page structure is specified then it will add the
* corresponding hment to the mapping list.
* It will also update the hmenum field for the tte.
*/
void
{
}
/*
* Load (ttep != NULL) or unload (ttep == NULL) one entry in the TSB.
* Assumes that a particular page size may only be resident in one TSB.
*/
static void
{
int vpshift = MMU_PAGESHIFT;
int phys = 0;
#ifndef sun4v
#endif
} else {
}
} else {
/*
* If there isn't a TSB for this page size, or the TSB is
* swapped out, there is nothing to do. Note that the latter
* case seems impossible but can occur if hat_pageunload()
* is called on an ISM mapping while the process is swapped
* out.
*/
return;
/*
* If another thread is in the middle of relocating a TSB
* we can't unload the entry so set a flag so that the
* TSB will be flushed before it can be accessed by the
* process.
*/
return;
}
#if defined(UTSB_PHYS)
phys = 1;
#else
#endif
}
} else {
} else {
}
}
}
/*
* Unmap all entries from [start, end) matching the given page size.
*
* This function is used primarily to unmap replicated 64K or 512K entries
* from the TSB that are inserted using the base page size TSB pointer, but
* it may also be called to unmap a range of addresses from the TSB.
*/
void
{
int phys = 0;
/*
* Assumptions:
* If ttesz == 8K, 64K or 512K, we walk through the range 8K
* at a time shooting down any valid entries we encounter.
*
* If ttesz >= 4M we walk the range 4M at a time shooting
* down any valid mappings we find.
*/
#ifndef sun4v
#endif
} else {
}
} else {
/*
* If there isn't a TSB for this page size, or the TSB is
* swapped out, there is nothing to do. Note that the latter
* case seems impossible but can occur if hat_pageunload()
* is called on an ISM mapping while the process is swapped
* out.
*/
return;
/*
* If another thread is in the middle of relocating a TSB
* we can't unload the entry so set a flag so that the
* TSB will be flushed before it can be accessed by the
* process.
*/
return;
}
#if defined(UTSB_PHYS)
phys = 1;
#else
#endif
}
} else {
}
}
}
/*
* Select the optimum TSB size given the number of mappings
* that need to be cached.
*/
static int
{
int szc = 0;
#ifdef DEBUG
if (tsb_grow_stress) {
}
#endif /* DEBUG */
szc++;
return (szc);
}
/*
* This function will add a translation to the hme_blk and allocate the
* hme_blk if one does not exist.
* If a page structure is specified then it will add the
* corresponding hment to the mapping list.
* It will also update the hmenum field for the tte.
* Furthermore, it attempts to create a large page translation
* for <addr,hat> at page array pps. It assumes addr and first
* pp is correctly aligned. It returns 0 if successful and 1 otherwise.
*/
static int
{
struct hmehash_bucket *hmebp;
int ret;
/*
* Get mapping size.
*/
/*
* Acquire the hash bucket.
*/
/*
* Find the hment block.
*/
/*
* Add the translation.
*/
/*
* Release the hash bucket.
*/
return (ret);
}
/*
* Function locks and returns a pointer to the hash bucket for vaddr and size.
*/
static struct hmehash_bucket *
{
struct hmehash_bucket *hmebp;
int hmeshift;
return (hmebp);
}
/*
* Function returns a pointer to an hmeblk in the hash bucket, hmebp. If the
* hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is
* allocated.
*/
static struct hme_blk *
{
int hmeshift;
struct kmem_cache *sfmmu_cache;
/*
* We block until hblk_reserve_lock is released; it's held by
* the thread, temporarily using hblk_reserve, until hblk_reserve is
* replaced by a hblk from sfmmu8_cache.
*/
hblk_reserve_thread != curthread) {
goto ttearray_realloc;
}
} else {
/*
* It is possible for 8k and 64k hblks to collide since they
* have the same rehash value. This is because we
* lazily free hblks and 8K/64K blks could be lingering.
* If we find size mismatch we free the block and & try again.
*/
goto ttearray_realloc;
}
if (hmeblkp->hblk_shw_bit) {
/*
* if the hblk was previously used as a shadow hblk then
* we will change it to a normal hblk
*/
if (hmeblkp->hblk_shw_mask) {
goto ttearray_realloc;
} else {
hmeblkp->hblk_shw_bit = 0;
}
}
}
/*
* hat_memload() should never call kmem_cache_free(); see block
* comment showing the stacktrace in sfmmu_hblk_alloc();
* enqueue each hblk in the list to reserve list if it's created
* from sfmmu8_cache *and* sfmmup == KHATID.
*/
if ((sfmmu_cache == sfmmu8_cache) &&
continue;
}
return (hmeblkp);
}
/*
* Function adds a tte entry into the hmeblk. It returns 0 if successful and 1
* otherwise.
*/
static int
{
#ifdef DEBUG
#endif /* DEBUG */
/*
* remove this panic when we decide to let user virtual address
* space be >= USERLIMIT.
*/
#if defined(TTE_IS_GLOBAL)
if (TTE_IS_GLOBAL(ttep))
panic("sfmmu_tteload: creating global tte");
#endif
#ifdef DEBUG
panic("sfmmu_tteload: non cacheable memory tte");
#endif /* DEBUG */
!TTE_IS_MOD(ttep)) {
/*
* Don't load TSB for dummy as in ISM. Also don't preload
* the TSB if the TTE isn't writable since we're likely to
* fault on it again -- preloading can be fairly expensive.
*/
}
switch (size) {
case TTE8K:
break;
case TTE64K:
break;
case TTE512K:
break;
case TTE4M:
break;
case (TTE32M):
break;
case (TTE256M):
break;
}
/*
* Need to grab mlist lock here so that pageunload
* will not change tte behind us.
*/
if (pp) {
}
/*
* Look for corresponding hment and if valid verify
* pfns are equal.
*/
if (remap) {
if (flags & HAT_LOAD_REMAP) {
/* make sure we are remapping same type of pages */
panic("sfmmu_tteload - tte remap io<->memory");
}
panic("sfmmu_tteload - tte remap pp != NULL");
}
panic("sfmmu_tteload - tte remap, hmeblkp 0x%p",
(void *)hmeblkp);
}
}
if (pp) {
#ifdef VAC
/*
* Handle VAC consistency
*/
}
#endif
} else if (!PP_ISMAPPED(pp) &&
}
}
/*
* sfmmu_pagearray_setup failed so return
*/
return (1);
}
}
/*
* Make sure hment is not on a mapping list.
*/
/* if it is not a remap then hme->next better be NULL */
if (flags & HAT_LOAD_LOCK) {
panic("too high lckcnt-hmeblk %p",
(void *)hmeblkp);
}
}
#ifdef VAC
/*
* If the physical page is marked to be uncacheable, like
* by a vac conflict, make sure the new mapping is also
* uncacheable.
*/
}
#endif
#ifdef DEBUG
#endif /* DEBUG */
}
#ifdef DEBUG
#endif /* DEBUG */
}
if (!TTE_IS_VALID(&tteold)) {
/*
* HAT_RELOAD_SHARE has been deprecated with lpg DISM.
*/
/*
* If this is the first large mapping for the process
* we must force any CPUs running this process to TL=0
* where they will reload the HAT flags from the
* tsbmiss area. This is necessary to make the large
* mappings we are about to load visible to those CPUs;
* otherwise they'll loop forever calling pagefault()
* since we don't search large hash chains by default.
*/
/* no sync mmustate; 64K shares 8K hashes */
} else if (mmu_page_sizes == max_mmu_page_sizes) {
}
}
}
}
}
/*
* If remap and new tte differs from old tte we need
* need to sync ref bit because we currently always set
* ref bit in tteload.
*/
if (TTE_IS_MOD(&tteold)) {
}
}
if ((flags & SFMMU_NO_TSBLOAD) == 0) {
/*
* We only preload 8K and 4M mappings into the TSB, since
* 64K and 512K mappings are replicated and hence don't
* have a single, unique TSB entry. Ditto for 32M/256M.
*/
}
}
if (pp) {
if (!remap) {
/*
* Cannot ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS)
* see pageunload() for comment.
*/
}
}
return (0);
}
/*
* Function unlocks hash bucket.
*/
static void
{
}
/*
* function which checks and sets up page array for a large
* translation. Will set p_vcolor, p_index, p_ro fields.
* Assumes addr and pfnum of first page are properly aligned.
* Will check for physical contiguity. If check fails it return
* non null.
*/
static int
{
#ifdef VAC
int osz;
int cflags = 0;
int vac_err = 0;
#endif
int newidx = 0;
/*
* Save the first pp so we can do HAT_TMPNC at the end.
*/
#ifdef VAC
#endif
/*
* XXX is it possible to maintain P_RO on the root only?
*/
}
}
/*
* If this is a remap we skip vac & contiguity checks.
*/
if (remap)
continue;
/*
* set p_vcolor and detect any vac conflicts.
*/
#ifdef VAC
if (vac_err == 0) {
}
#endif
/*
* Save current index in case we need to undo it.
* Note: "PAGESZ_TO_INDEX(sz) (1 << (sz))"
* "SFMMU_INDEX_SHIFT 6"
* "SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1)"
* "PP_MAPINDEX(p_index) (p_index & SFMMU_INDEX_MASK)"
*
* So: index = PAGESZ_TO_INDEX(ttesz);
* if ttesz == 1 then index = 0x2
* 2 then index = 0x4
* 3 then index = 0x8
* 4 then index = 0x10
* 5 then index = 0x20
* The code below checks if it's a new pagesize (ie, newidx)
* in case we need to take it back out of p_index,
* and then or's the new index into the existing index.
*/
newidx = 1;
/*
* contiguity check
*/
/*
* If we fail the contiguity test then
* the only thing we need to fix is the p_index field.
* We might get a few extra flushes but since this
* path is rare that is ok. The p_ro field will
* get automatically fixed on the next tteload to
* the page. NO TNC bit is set yet.
*/
while (i >= 0) {
if (newidx)
~index);
pps--;
i--;
}
return (1);
}
pfnum++;
addr += MMU_PAGESIZE;
}
#ifdef VAC
if (vac_err) {
/*
* There are some smaller mappings that causes vac
* conflicts. Convert all existing small mappings to
* TNC.
*/
npgs);
} else {
/* EMPTY */
/*
* If there exists an big page mapping,
* that means the whole existing big page
* has TNC setting already. No need to covert to
* TNC again.
*/
}
}
#endif /* VAC */
return (0);
}
#ifdef VAC
/*
* Routine that detects vac consistency for a large page. It also
* sets virtual color for all pp's for this big mapping.
*/
static int
{
return (HAT_TMPNC);
}
if (PP_NEWPAGE(pp)) {
return (0);
}
return (0);
}
if (!PP_ISMAPPED(pp)) {
/*
* Previous user of page had a differnet color
* but since there are no current users
* we just flush the cache and change the color.
* As an optimization for large pages we flush the
* entire cache of that color and set a flag.
*/
}
return (0);
}
/*
* We got a real conflict with a current mapping.
* set flags to start unencaching all mappings
* and return failure so we restart looping
* the pp array from the beginning.
*/
return (HAT_TMPNC);
}
#endif /* VAC */
/*
* creates a large page shadow hmeblk for a tte.
* The purpose of this routine is to allow us to do quick unloads because
* the vm layer can easily pass a very large but sparsely populated range.
*/
static struct hme_blk *
{
struct hmehash_bucket *hmebp;
if (mmu_page_sizes == max_mmu_page_sizes) {
} else {
}
} else {
}
}
if (!hmeblkp->hblk_shw_mask) {
/*
* if this is a unused hblk it was just allocated or could
* potentially be a previous large page hblk so we need to
* set the shadow bit.
*/
}
/*
* Atomically set shw mask bit
*/
do {
} while (newshw_mask != shw_mask);
return (hmeblkp);
}
/*
* This routine cleanup a previous shadow hmeblk and changes it to
* a regular hblk. This happens rarely but it is possible
* when a process wants to use large pages and there are hblks still
* lying around from the previous as that used these hmeblks.
* The alternative was to cleanup the shadow hblks at unload time
* but since so few user processes actually use large pages, it is
* better to be lazy and cleanup at this time.
*/
static void
struct hmehash_bucket *hmebp)
{
if (!hmeblkp->hblk_shw_mask) {
hmeblkp->hblk_shw_bit = 0;
return;
}
}
static void
int hashno)
{
struct hmehash_bucket *hmebp;
/* inline HME_HASH_SEARCH */
prevpa = 0;
while (hmeblkp) {
/* found hme_blk */
if (hmeblkp->hblk_shw_bit) {
if (hmeblkp->hblk_shw_mask) {
shadow = 1;
break;
} else {
hmeblkp->hblk_shw_bit = 0;
}
}
/*
* Hblk_hmecnt and hblk_vcnt could be non zero
* since hblk_unload() does not gurantee that.
*
* XXX - this could cause tteload() to spin
* where sfmmu_shadow_hcleanup() is called.
*/
}
pr_hblk);
} else {
}
}
if (shadow) {
/*
* We found another shadow hblk so cleaned its
* children. We need to go back and cleanup
* the original hblk so we don't change the
* addr.
*/
shadow = 0;
} else {
(1 << hmeshift));
}
}
}
/*
* Release one hardware address translation lock on the given address range.
*/
void
{
struct hmehash_bucket *hmebp;
/*
* Spitfire supports 4 page sizes.
* Most pages are expected to be of the smallest page size (8K) and
* these will not need to be rehashed. 64K pages also don't need to be
* rehashed because an hmeblk spans 64K of address space. 512K pages
* might need 1 rehash and and 4M pages might need 2 rehashes.
*/
/*
* If we encounter a shadow hmeblk then
* we know there are no valid hmeblks mapping
* this address at this size or larger.
* Just increment address by the smallest
* page size.
*/
if (hmeblkp->hblk_shw_bit) {
addr += MMU_PAGESIZE;
} else {
endaddr);
}
hashno = 1;
continue;
}
/*
* We have traversed the whole list and rehashed
* if necessary without finding the address to unlock
* which should never happen.
*/
panic("sfmmu_unlock: addr not found. "
} else {
hashno++;
}
}
}
/*
* Function to unlock a range of addresses in an hmeblk. It returns the
* next address that needs to be unlocked.
* Should be called with the hash lock held.
*/
static caddr_t
{
if (TTE_IS_VALID(&tteold)) {
if (ret < 0)
goto readtte;
if (hmeblkp->hblk_lckcnt == 0)
panic("zero hblk lckcnt");
panic("can't unlock large tte");
} else {
panic("sfmmu_hblk_unlock: invalid tte");
}
sfhme++;
}
return (addr);
}
/*
* Physical Address Mapping Framework
*
* General rules:
*
* (1) Applies only to seg_kmem memory pages. To make things easier,
* seg_kpm addresses are also accepted by the routines, but nothing
* is done with them since by definition their PA mappings are static.
* (2) hat_add_callback() may only be called while holding the page lock
* SE_SHARED or SE_EXCL of the underlying page (e.g., as_pagelock()),
* or passing HAC_PAGELOCK flag.
* (3) prehandler() and posthandler() may not call hat_add_callback() or
* hat_delete_callback(), nor should they allocate memory. Post quiesce
* callbacks may not sleep or acquire adaptive mutex locks.
* (4) Either prehandler() or posthandler() (but not both) may be specified
* as being NULL. Specifying an errhandler() is optional.
*
* Details of using the framework:
*
* registering a callback (hat_register_callback())
*
* Pass prehandler, posthandler, errhandler addresses
* as described below. If capture_cpus argument is nonzero,
* suspend callback to the prehandler will occur with CPUs
* captured and executing xc_loop() and CPUs will remain
* captured until after the posthandler suspend callback
* occurs.
*
* adding a callback (hat_add_callback())
*
* as_pagelock();
* hat_add_callback();
* save returned pfn in private data structures or program registers;
* as_pageunlock();
*
* prehandler()
*
* Stop all accesses by physical address to this memory page.
* Called twice: the first, PRESUSPEND, is a context safe to acquire
* adaptive locks. The second, SUSPEND, is called at high PIL with
* CPUs captured so adaptive locks may NOT be acquired (and all spin
* locks must be XCALL_PIL or higher locks).
*
* May return the following errors:
* EIO: A fatal error has occurred. This will result in panic.
* EAGAIN: The page cannot be suspended. This will fail the
* relocation.
* 0: Success.
*
* posthandler()
*
* Save new pfn in private data structures or program registers;
* not allowed to fail (non-zero return values will result in panic).
*
* errhandler()
*
* called when an error occurs related to the callback. Currently
* the only such error is HAT_CB_ERR_LEAKED which indicates that
* a page is being freed, but there are still outstanding callback(s)
* registered on the page.
*
* removing a callback (hat_delete_callback(); e.g., prior to freeing memory)
*
* stop using physical address
* hat_delete_callback();
*
*/
/*
* Register a callback class. Each subsystem should do this once and
* cache the id_t returned for use in setting up and tearing down callbacks.
*
* There is no facility for removing callback IDs once they are created;
* the "key" should be unique for each module, so in case a module is unloaded
* and subsequently re-loaded, we can recycle the module's previous entry.
*/
hat_register_callback(int key,
int capture_cpus)
{
/*
* Search the table for a pre-existing callback associated with
* the identifier "key". If one exists, we re-use that entry in
* the table for this instance, otherwise we assign the next
* available table slot.
*/
break;
}
if (id == sfmmu_max_cb_id) {
id = sfmmu_cb_nextid++;
if (id >= sfmmu_max_cb_id)
panic("hat_register_callback: out of callback IDs");
}
return (id);
}
#define HAC_COOKIE_NONE (void *)-1
/*
* when relocating the associated page. See the description of pre and
* posthandler above for more details.
*
* If HAC_PAGELOCK is included in flags, the underlying memory page is
* locked internally so the caller must be able to deal with the callback
* running even before this function has returned. If HAC_PAGELOCK is not
* set, it is assumed that the underlying memory pages are locked.
*
* Since the caller must track the individual page boundaries anyway,
* we only allow a callback to be added to a single page (large
* or small). Thus [addr, addr + len) MUST be contained within a single
* page.
*
* Registering multiple callbacks on the same [addr, addr+len) is supported,
* _provided_that_ a unique parameter is specified for each callback.
* If multiple callbacks are registered on the same range the callback will
* be invoked with each unique parameter. Registering the same callback with
* the same argument more than once will result in corrupted kernel state.
*
* Returns the pfn of the underlying kernel page in *rpfn
* on success, or PFN_INVALID on failure.
*
* cookiep (if passed) provides storage space for an opaque cookie
* to return later to hat_delete_callback(). This cookie makes the callback
* deletion significantly quicker by avoiding a potentially lengthy hash
* search.
*
* Returns values:
* 0: success
* ENOMEM: memory allocation failure (e.g. flags was passed as HAC_NOSLEEP)
* EINVAL: callback ID is not valid
* ENXIO: ["vaddr", "vaddr" + len) is not mapped in the kernel's address
* space
* ERANGE: ["vaddr", "vaddr" + len) crosses a page boundary
*/
int
{
struct hmehash_bucket *hmebp;
int locked = 0;
/*
* For KPM mappings, just return the physical address since we
* don't need to register any callbacks.
*/
if (IS_KPM_ADDR(vaddr)) {
return (0);
}
*rpfn = PFN_INVALID;
return (EINVAL);
}
*rpfn = PFN_INVALID;
return (ENOMEM);
}
/* Find the mapping(s) for this page */
hashno++) {
}
*rpfn = PFN_INVALID;
return (ENXIO);
}
if (!TTE_IS_VALID(&tte)) {
*rpfn = PFN_INVALID;
return (ENXIO);
}
/*
* Make sure the boundaries for the callback fall within this
* single mapping.
*/
*rpfn = PFN_INVALID;
return (ERANGE);
}
/*
* The pfn may not have a page_t underneath in which case we
* just return it. This can happen if we are doing I/O to a
* static portion of the kernel's address space, for instance.
*/
if (cookiep)
return (0);
}
if (flags & HAC_PAGELOCK) {
/*
* Somebody is holding SE_EXCL lock. Might
* even be hat_page_relocate(). Drop all
* our locks, lookup the page in &kvp, and
* retry. If it doesn't exist in &kvp, then
* we must be dealing with a kernel mapped
* page which doesn't actually belong to
* segkmem so we punt.
*/
if (cookiep)
return (0);
}
goto rehash;
}
locked = 1;
}
/*
* The page moved before we got our hands on it. Drop
* all the locks and try again.
*/
locked = 0;
goto rehash;
}
/*
* This is not a segkmem page but another page which
* has been kernel mapped. It had better have at least
* a share lock on it. Return the pfn.
*/
if (locked)
if (cookiep)
return (0);
}
/*
* Setup this pa_hment and link its embedded dummy sf_hment into
* the mapping list.
*/
if (locked)
if (cookiep)
return (0);
}
/*
*/
void
void *cookie)
{
struct hmehash_bucket *hmebp;
int locked = 0;
/*
* If the cookie is HAC_COOKIE_NONE then there is no pa_hment to
* remove so just return.
*/
return;
/* Find the mapping(s) for this page */
hashno++) {
}
return;
if (!TTE_IS_VALID(&tte)) {
return;
}
return;
}
if (flags & HAC_PAGELOCK) {
/*
* Somebody is holding SE_EXCL lock. Might
* even be hat_page_relocate(). Drop all
* our locks, lookup the page in &kvp, and
* retry. If it doesn't exist in &kvp, then
* we must be dealing with a kernel mapped
* page which doesn't actually belong to
* segkmem so we punt.
*/
return;
}
goto rehash;
}
locked = 1;
}
/*
* The page moved before we got our hands on it. Drop
* all the locks and try again.
*/
locked = 0;
goto rehash;
}
/*
* This is not a segkmem page but another page which
* has been kernel mapped.
*/
if (locked)
return;
}
} else {
/*
* skip va<->pa mappings
*/
continue;
/*
* if pa_hment matches, remove it
*/
break;
}
}
}
if (!panicstr) {
panic("hat_delete_callback: pa_hment not found, pp %p",
(void *)pp);
}
return;
}
/*
* Note: at this point a valid kernel mapping must still be
* present on this page.
*/
panic("hat_delete_callback: zero p_share");
panic("hat_delete_callback: pa_hment is busy");
/*
* Remove sfhmep from the mapping list for the page.
*/
} else {
}
if (locked)
return;
}
if (locked)
}
/*
* hat_probe returns 1 if the translation for the address 'addr' is
* loaded, zero otherwise.
*
* hat_probe should be used only for advisorary purposes because it may
* occasionally return the wrong value. The implementation must guarantee that
* returning the wrong value is a very rare event. hat_probe is used
* to implement optimizations in the segment drivers.
*
*/
int
{
== PFN_SUSPENDED) {
}
} else {
}
if (pfn != PFN_INVALID)
return (1);
else
return (0);
}
{
if (TTE_IS_VALID(&tte)) {
}
return (-1);
}
static void
{
struct hmehash_bucket *hmebp;
/* support for ISM */
int i;
if (ism_blkp) {
sfmmu_ismhat_enter(sfmmup, 0);
}
break;
}
}
}
if (locked_hatid) {
}
do {
break;
}
hashno++;
}
{
if (TTE_IS_VALID(&tte)) {
return (0);
}
*attr = 0;
return ((uint_t)0xffffffff);
}
/*
* Enables more attributes on specified address range (ie. logical OR)
*/
void
{
if (hat->sfmmu_xhat_provider) {
return;
} else {
/*
* This must be a CPU HAT. If the address space has
* XHATs attached, change attributes for all of them,
* just in case
*/
}
}
/*
* Assigns attributes to the specified address range. All the attributes
* are specified.
*/
void
{
if (hat->sfmmu_xhat_provider) {
return;
} else {
/*
* This must be a CPU HAT. If the address space has
* XHATs attached, change attributes for all of them,
* just in case
*/
}
}
/*
* Remove attributes on the specified address range (ie. loginal NAND)
*/
void
{
if (hat->sfmmu_xhat_provider) {
return;
} else {
/*
* This must be a CPU HAT. If the address space has
* XHATs attached, change attributes for all of them,
* just in case
*/
}
}
/*
* Change attributes on an address range to that specified by attr and mode.
*/
static void
int mode)
{
struct hmehash_bucket *hmebp;
panic("user addr %p in kernel space",
(void *)addr);
}
/*
* We've encountered a shadow hmeblk so skip the range
* of the next smaller mapping size.
*/
if (hmeblkp->hblk_shw_bit) {
} else {
}
hashno = 1;
continue;
}
/*
* We have traversed the whole list and rehashed
* if necessary without finding the address to chgattr.
* This is ok, so we increment the address by the
* smallest hmeblk range for kernel mappings or for
* user mappings with no large pages, and the largest
* hmeblk range, to account for shadow hmeblks, for
* user mappings with large pages and continue.
*/
TTEBYTES(1));
else
hashno = 1;
} else {
hashno++;
}
}
}
/*
* This function chgattr on a range of addresses in an hmeblk. It returns the
* next addres that needs to be chgattr.
* It should be called with the hash lock held.
* XXX It should be possible to optimize chgattr by not flushing every time but
* on the other hand:
* 1. do one flush crosscall.
* 2. only flush if we are increasing permissions (make sure this will work)
*/
static caddr_t
{
int ttesz;
int ret;
int use_demap_range;
#if defined(SF_ERRATA_57)
int check_exec;
#endif
/*
* Flush the current demap region if addresses have been
* skipped or the page size doesn't match.
*/
if (use_demap_range) {
} else {
}
#if defined(SF_ERRATA_57)
#endif
if (TTE_IS_VALID(&tte)) {
/*
* if the new attr is the same as old
* continue
*/
goto next_addr;
}
if (!TTE_IS_WRITABLE(&tteattr)) {
/*
* make sure we clear hw modify bit if we
* removing write protections
*/
}
if (pp) {
}
/*
* tte must have been unloaded.
*/
continue;
}
#if defined(SF_ERRATA_57)
ttemod.tte_exec_perm = 0;
#endif
if (ret < 0) {
/* tte changed underneath us */
if (pml) {
}
continue;
}
/*
* need to sync if we are clearing modify bit.
*/
}
}
}
if (ret > 0 && use_demap_range) {
} else if (ret > 0) {
}
if (pml) {
}
}
sfhmep++;
}
return (addr);
}
/*
* This routine converts virtual attributes to physical ones. It will
* update the tteflags field with the tte mask corresponding to the attributes
* affected and it returns the new attributes. It will also clear the modify
* bit if we are taking away write permission. This is necessary since the
* modify bit is the hardware permission bit and we need to clear it in order
* to detect write faults.
*/
static uint64_t
{
switch (mode) {
case SFMMU_CHGATTR:
/* all attributes specified */
break;
case SFMMU_SETATTR:
/*
* a valid tte implies exec and read for sfmmu
* so no need to do anything about them.
* since priviledged access implies user access
* PROT_USER doesn't make sense either.
*/
if (attr & PROT_WRITE) {
}
break;
case SFMMU_CLRATTR:
/* attributes will be nand with current ones */
}
if (attr & PROT_WRITE) {
/* clear both writable and modify bit */
}
}
break;
default:
}
}
static uint_t
{
if (TTE_IS_WRITABLE(ttep)) {
attr |= PROT_WRITE;
}
if (TTE_IS_EXECUTABLE(ttep)) {
}
if (!TTE_IS_PRIVILEGED(ttep)) {
}
if (TTE_IS_NFO(ttep)) {
attr |= HAT_NOFAULT;
}
if (TTE_IS_NOSYNC(ttep)) {
attr |= HAT_NOSYNC;
}
if (TTE_IS_SIDEFFECT(ttep)) {
attr |= SFMMU_SIDEFFECT;
}
if (!TTE_IS_VCACHEABLE(ttep)) {
}
if (!TTE_IS_PCACHEABLE(ttep)) {
}
return (attr);
}
/*
* hat_chgprot is a deprecated hat call. New segment drivers
* should store all attributes and use hat_*attr calls.
*
* Change the protections in the virtual address range
* given to the specified virtual protection. If vprot is ~PROT_WRITE,
* then remove write permission, leaving the other
* permissions unchanged. If vprot is ~PROT_USER, remove user permissions.
*
*/
void
{
struct hmehash_bucket *hmebp;
if (sfmmup->sfmmu_xhat_provider) {
return;
} else {
/*
* This must be a CPU HAT. If the address space has
* XHATs attached, change attributes for all of them,
* just in case
*/
}
panic("user addr %p vprot %x in kernel space",
}
/*
* We've encountered a shadow hmeblk so skip the range
* of the next smaller mapping size.
*/
if (hmeblkp->hblk_shw_bit) {
} else {
}
hashno = 1;
continue;
}
/*
* We have traversed the whole list and rehashed
* if necessary without finding the address to chgprot.
* This is ok so we increment the address by the
* smallest hmeblk range for kernel mappings and the
* largest hmeblk range, to account for shadow hmeblks,
* for user mappings and continue.
*/
TTEBYTES(1));
else
hashno = 1;
} else {
hashno++;
}
}
}
/*
* This function chgprots a range of addresses in an hmeblk. It returns the
* next addres that needs to be chgprot.
* It should be called with the hash lock held.
* XXX It shold be possible to optimize chgprot by not flushing every time but
* on the other hand:
* 1. do one flush crosscall.
* 2. only flush if we are increasing permissions (make sure this will work)
*/
static caddr_t
{
int ttesz;
int ret;
int use_demap_range;
#if defined(SF_ERRATA_57)
int check_exec;
#endif
#ifdef DEBUG
panic("sfmmu_hblk_chgprot: partial chgprot of large page");
}
#endif /* DEBUG */
#if defined(SF_ERRATA_57)
#endif
/*
* Flush the current demap region if addresses have been
* skipped or the page size doesn't match.
*/
if (use_demap_range) {
} else {
}
if (TTE_IS_VALID(&tte)) {
/*
* if the new protection is the same as old
* continue
*/
goto next_addr;
}
if (pp) {
}
/*
* tte most have been unloaded
* underneath us. Recheck
*/
continue;
}
#if defined(SF_ERRATA_57)
ttemod.tte_exec_perm = 0;
#endif
if (ret < 0) {
/* tte changed underneath us */
if (pml) {
}
continue;
}
if (tteflags & TTE_HWWR_INT) {
/*
* need to sync if we are clearing modify bit.
*/
}
if (pprot & TTE_WRPRM_INT) {
}
}
if (ret > 0 && use_demap_range) {
} else if (ret > 0) {
}
if (pml) {
}
}
sfhmep++;
}
return (addr);
}
/*
* This routine is deprecated and should only be used by hat_chgprot.
* The correct routine is sfmmu_vtop_attr.
* This routine converts virtual page protections to physical ones. It will
* update the tteflags field with the tte mask corresponding to the protections
* affected and it returns the new protections. It will also clear the modify
* bit if we are taking away write permission. This is necessary since the
* modify bit is the hardware permission bit and we need to clear it in order
* to detect write faults.
* It accepts the following special protections:
* ~PROT_WRITE = remove write permissions.
* ~PROT_USER = remove user permissions.
*/
static uint_t
{
return (0); /* will cause wrprm to be cleared */
}
return (0); /* will cause privprm to be cleared */
}
}
switch (vprot) {
case (PROT_READ):
case (PROT_EXEC):
return (TTE_PRIV_INT); /* set prv and clr wrt */
case (PROT_WRITE):
case (PROT_WRITE | PROT_READ):
case (PROT_EXEC | PROT_WRITE):
return (0); /* clr prv and wrt */
case (PROT_USER | PROT_WRITE):
return (TTE_WRPRM_INT); /* clr prv and set wrt */
default:
}
return (0);
}
/*
* Alternate unload for very large virtual ranges. With a true 64 bit VA,
* the normal algorithm would take too long for a very large VA range with
* few real mappings. This routine just walks thru all HMEs in the global
* hash table to find and remove mappings.
*/
static void
{
struct hmehash_bucket *hmebp;
int i;
int addr_cnt = 0;
int a = 0;
if (sfmmup->sfmmu_free) {
} else {
}
/*
* Loop through all the hash buckets of HME blocks looking for matches.
*/
for (i = 0; i <= UHMEHASH_SZ; i++) {
prevpa = 0;
while (hmeblkp) {
/*
* skip if not this context, if a shadow block or
* if the mapping is not in the requested range
*/
hmeblkp->hblk_shw_bit ||
goto next_block;
}
/*
* unload if there are any current valid mappings
*/
hmeblkp->hblk_hmecnt != 0)
/*
* on unmap we also release the HME block itself, once
* all mappings are gone.
*/
if ((flags & HAT_UNLOAD_UNMAP) != 0 &&
!hmeblkp->hblk_hmecnt) {
} else {
}
goto next_block;
/*
* HME blocks may span more than one page, but we may be
* unmapping only one page, so check for a smaller range
* for the callback
*/
if (++addr_cnt == MAX_CB_ADDR) {
}
for (a = 0; a < MAX_CB_ADDR; ++a) {
}
addr_cnt = 0;
}
}
}
}
for (a = 0; a < addr_cnt; ++a) {
}
/*
* Check TSB and TLB page sizes if the process isn't exiting.
*/
if (!sfmmup->sfmmu_free)
}
/*
* Unload all the mappings in the range [addr..addr+len). addr and len must
* be MMU_PAGESIZE aligned.
*/
void
{
struct hmehash_bucket *hmebp;
int addr_count = 0;
int a;
if (sfmmup->sfmmu_xhat_provider) {
return;
} else {
/*
* This must be a CPU HAT. If the address space has
* XHATs attached, unload the mappings for all of them,
* just in case
*/
}
/*
* Probing through a large VA range (say 63 bits) will be slow, even
* at 4 Meg steps between the probes. So, when the virtual address range
* is very large, search the HME entries for what to unload.
*
* len >> TTE_PAGE_SHIFT(TTE4M) is the # of 4Meg probes we'd need
*
* UHMEHASH_SZ is number of hash buckets to examine
*
*/
return;
}
/*
* If the process is exiting, we can save a lot of fuss since
* we'll flush the TLB when we free the ctx anyway.
*/
if (sfmmup->sfmmu_free)
else
/*
* It is likely for the vm to call unload over a wide range of
* addresses that are actually very sparsely populated by
* translations. In order to speed this up the sfmmu hat supports
* the concept of shadow hmeblks. Dummy large page hmeblks that
* correspond to actual small translations are allocated at tteload
* time and are referred to as shadow hmeblks. Now, during unload
* time, we first check if we have a shadow hmeblk for that
* translation. The absence of one means the corresponding address
* range is empty and can be skipped.
*
* The kernel is an exception to above statement and that is why
* we don't use shadow hmeblks and hash starting from the smallest
* page size.
*/
iskernel = 1;
} else {
iskernel = 0;
if (mmu_page_sizes == max_mmu_page_sizes) {
} else {
}
}
/*
* didn't find an hmeblk. skip the appropiate
* address range.
*/
if (iskernel) {
if (hashno < mmu_hashcnt) {
hashno++;
continue;
} else {
+ 1, MMU_PAGESIZE64K);
continue;
}
}
(1 << hmeshift));
continue;
}
continue;
}
if (mmu_page_sizes == max_mmu_page_sizes) {
continue;
}
continue;
}
continue;
} else {
continue;
}
}
/*
* If the valid count is zero we can skip the range
* mapped by this hmeblk.
* We free hblks in the case of HAT_UNMAP. HAT_UNMAP
* is used by segment drivers as a hint
* that the mapping resource won't be used any longer.
* The best example of this is during exit().
*/
if ((flags & HAT_UNLOAD_UNMAP) ||
pr_hblk);
}
if (iskernel) {
continue;
}
continue;
}
continue;
}
if (mmu_page_sizes == max_mmu_page_sizes) {
continue;
}
continue;
}
continue;
} else {
continue;
}
}
if (hmeblkp->hblk_shw_bit) {
/*
* If we encounter a shadow hmeblk we know there is
* smaller sized hmeblks mapping the same address space.
* Decrement the hash size and rehash.
*/
hashno--;
continue;
}
/*
* track callback address ranges.
* only start a new range when it's not contiguous
*/
if (addr_count > 0 &&
--addr_count;
else
}
pr_hblk);
}
/*
* Notify our caller as to exactly which pages
* have been unloaded. We do these in clumps,
* to minimize the number of xt_sync()s that need to occur.
*/
}
for (a = 0; a < MAX_CB_ADDR; ++a) {
}
addr_count = 0;
}
if (iskernel) {
continue;
}
continue;
}
continue;
}
if (mmu_page_sizes == max_mmu_page_sizes) {
continue;
}
continue;
}
} else {
}
}
}
if (callback && addr_count != 0) {
for (a = 0; a < addr_count; ++a) {
}
}
/*
* Check TSB and TLB page sizes if the process isn't exiting.
*/
if (!sfmmup->sfmmu_free)
}
/*
* Unload all the mappings in the range [addr..addr+len). addr and len must
* be MMU_PAGESIZE aligned.
*/
void
{
if (sfmmup->sfmmu_xhat_provider) {
return;
}
}
/*
* Find the largest mapping size for this page.
*/
int
{
int sz;
int p_index;
sz = 0;
sz++;
}
return (sz);
}
/*
* This function unloads a range of addresses for an hmeblk.
* It returns the next address to be unloaded.
* It should be called with the hash lock held.
*/
static caddr_t
{
int ttesz;
long ttecnt;
int ret;
int use_demap_range;
#ifdef DEBUG
panic("sfmmu_hblk_unload: partial unload of large page");
}
#endif /* DEBUG */
if (use_demap_range) {
} else {
}
ttecnt = 0;
if (TTE_IS_VALID(&tte)) {
}
/*
* Verify if hme still points to 'pp' now that
* we have p_mapping lock.
*/
if (pml) {
}
/* Re-start this iteration. */
continue;
}
goto tte_unloaded;
}
/*
* This point on we have both HASH and p_mapping
* lock.
*/
/*
* We need to loop on modify tte because it is
* possible for pagesync to come along and
* change the software bits beneath us.
*
* Page_unload can also invalidate the tte after
* we read tte outside of p_mapping lock.
*/
if (ret <= 0) {
if (TTE_IS_VALID(&tte)) {
goto again;
} else {
/*
* We read in a valid pte, but it
* is unloaded by page_unload.
* hme_page has become NULL and
* we hold no p_mapping lock.
*/
goto tte_unloaded;
}
}
if (!(flags & HAT_UNLOAD_NOSYNC)) {
}
/*
* Ok- we invalidated the tte. Do the rest of the job.
*/
ttecnt++;
if (flags & HAT_UNLOAD_UNLOCK) {
}
/*
* Normally we would need to flush the page
* from the virtual cache at this point in
* order to prevent a potential cache alias
* inconsistency.
* The particular scenario we need to worry
* about is:
* Given: va1 and va2 are two virtual address
* that alias and map the same physical
* address.
* 1. mapping exists from va1 to pa and data
* has been read into the cache.
* 2. unload va1.
* 3. load va2 and modify data using va2.
* 4 unload va2.
* 5. load va1 and reference data. Unless we
* flush the data cache when we unload we will
* get stale data.
* Fortunately, page coloring eliminates the
* above scenario by remembering the color a
* physical page was last or is currently
* mapped to. Now, we delay the flush until
* the loading of translations. Only when the
* new translation is of a different color
* are we forced to flush.
*/
if (use_demap_range) {
/*
* Mark this page as needing a demap.
*/
} else {
if (do_virtual_coloring) {
sfmmup->sfmmu_free, 0);
} else {
CACHE_FLUSH, 0);
}
}
if (pp) {
/*
* Remove the hment from the mapping list
*/
/*
* Again, we cannot
* ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS);
*/
membar_stst();
}
!hmeblkp->hblk_lckcnt);
#ifdef VAC
/*
* If page was temporary
* uncached, try to recache
* it. Note that HME_SUB() was
* called above so p_index and
* mlist had been updated.
*/
/*
* Page is marked to be in VAC conflict
* kpm mapped using only the regular
* pagesize.
*/
}
}
#endif /* VAC */
/*
* TTE is invalid but the hme
* still exists. let pageunload
* complete its job.
*/
goto again;
}
} else if (hmeblkp->hblk_hmecnt != 0) {
/*
* pageunload may have not finished decrementing
* hblk_vcnt and hblk_hmecnt. Find page_t if any and
* wait for pageunload to finish. Rely on pageunload
* to decrement hblk_hmecnt after hblk_vcnt.
*/
if (pf_is_memory(pfn)) {
}
}
}
/*
* At this point, the tte we are looking at
* should be unloaded, and hme has been unlinked
* from page too. This is important because in
* pageunload, it does ttesync() then HME_SUB.
* We need to make sure HME_SUB has been completed
* so we know ttesync() has been completed. Otherwise,
* at exit time, after return from hat layer, VM will
* release as structure which hat_setstat() (called
* by ttesync()) needs.
*/
#ifdef DEBUG
{
}
#endif
if (pml) {
}
sfhmep++;
}
if (ttecnt > 0)
return (addr);
}
/*
* Synchronize all the mappings in the range [addr..addr+len).
* Can be called with clearflag having two states:
* HAT_SYNC_DONTZERO means just return the rm stats
* HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats
*/
void
{
struct hmehash_bucket *hmebp;
(clearflag == HAT_SYNC_ZERORM));
/*
* Spitfire supports 4 page sizes.
* Most pages are expected to be of the smallest page
* size (8K) and these will not need to be rehashed. 64K
* pages also don't need to be rehashed because the an hmeblk
* spans 64K of address space. 512K pages might need 1 rehash and
* and 4M pages 2 rehashes.
*/
/*
* We've encountered a shadow hmeblk so skip the range
* of the next smaller mapping size.
*/
if (hmeblkp->hblk_shw_bit) {
} else {
}
hashno = 1;
continue;
}
/*
* We have traversed the whole list and rehashed
* if necessary without finding the address to sync.
* This is ok so we increment the address by the
* smallest hmeblk range for kernel mappings and the
* largest hmeblk range, to account for shadow hmeblks,
* for user mappings and continue.
*/
TTEBYTES(1));
else
hashno = 1;
} else {
hashno++;
}
}
}
static caddr_t
{
int ttesz;
int ret;
if (TTE_IS_VALID(&tte)) {
if (pp) {
}
/*
* tte most have been unloaded
* underneath us. Recheck
*/
continue;
}
if (clearflag == HAT_SYNC_ZERORM) {
TTE_CLR_RM(&ttemod);
if (ret < 0) {
if (pml) {
}
continue;
}
if (ret > 0) {
hmeblkp, 0, 0);
}
}
if (pml) {
}
}
sfhmep++;
}
return (addr);
}
/*
* This function will sync a tte to the page struct and it will
* update the hat stats. Currently it allows us to pass a NULL pp
* and we will simply update the stats. We may want to change this
* so we only keep stats for pages backed by pp's.
*/
static void
{
int sz;
if (TTE_IS_NOSYNC(ttep)) {
return;
}
if (TTE_IS_REF(ttep)) {
}
if (TTE_IS_MOD(ttep)) {
}
if (rm == 0) {
return;
}
if (sfmmup->sfmmu_rmstat) {
int i;
}
}
/*
* XXX I want to use cas to update nrm bits but they
* they should be.
* The nrm bits are protected by the same mutex as
* the one that protects the page's mapping list.
*/
if (!pp)
return;
/*
* If the tte is for a large page, we need to sync all the
* pages covered by the tte.
*/
}
/* Get number of pages from tte size. */
do {
/*
* Are we done? If not, we must have a large mapping.
* For large mappings we need to sync the rest of the pages
* covered by this tte; goto the next page.
*/
}
/*
* Execute pre-callback handler of each pa_hment linked to pp
*
* Inputs:
* flag: either HAT_PRESUSPEND or HAT_SUSPEND.
* capture_cpus: pointer to return value (below)
*
* Returns:
* Propagates the subsystem callback return values back to the caller;
* returns 0 on success. If capture_cpus is non-NULL, the value returned
* is zero if all of the pa_hments are of a type that do not require
* capturing CPUs prior to suspending the mapping, else it is 1.
*/
static int
{
int ret;
int locked = 0;
if (!sfmmu_mlist_held(pp)) {
locked = 1;
}
if (capture_cpus)
*capture_cpus = 0;
top:
/*
* skip sf_hments corresponding to VA<->PA mappings;
* for pa_hment's, hme_tte.ll is zero
*/
continue;
/*
* skip if pre-handler has been called earlier in this loop
*/
continue;
*capture_cpus = 1;
continue;
}
/*
* Drop the mapping list lock to avoid locking order issues.
*/
if (locked)
if (ret != 0)
return (ret); /* caller must do the cleanup */
if (locked) {
goto top;
}
}
if (locked)
return (0);
}
/*
* Execute post-callback handler of each pa_hment linked to pp
*
* Same overall assumptions and restrictions apply as for
* hat_pageprocess_precallbacks().
*/
static void
{
int locked = 0;
if (!sfmmu_mlist_held(pp)) {
locked = 1;
}
top:
/*
* skip sf_hments corresponding to VA<->PA mappings;
* for pa_hment's, hme_tte.ll is zero
*/
continue;
continue;
continue;
/*
* Convert the base page PFN into the constituent PFN
* which is needed by the callback handler.
*/
/*
* Drop the mapping list lock to avoid locking order issues.
*/
if (locked)
!= 0)
panic("sfmmu: posthandler failed");
if (locked) {
goto top;
}
}
if (locked)
}
/*
* Suspend locked kernel mapping
*/
void
{
/*
* Call into dtrace to tell it we're about to suspend a
* kernel mapping. This prevents us from running into issues
* with probe context trying to touch a suspended page
* in the relocation codepath itself.
*/
if (dtrace_kreloc_init)
(*dtrace_kreloc_init)();
continue;
continue;
/*
* Loop until we successfully set the suspend bit in
* the TTE.
*/
goto again;
/*
* Invalidate TSB entry
*/
/*
* No need to make sure that the TSB for this sfmmu is
* not being relocated since it is ksfmmup and thus it
* will never be relocated.
*/
/*
* Update xcall stats
*/
/* LINTED: constant in conditional context */
/*
* Flush TLB entry on remote CPU's
*/
/*
* Flush TLB entry on local CPU
*/
}
while (index != 0) {
if (index != 0)
cons++;
if (index & 0x1) {
goto retry;
}
}
}
#ifdef DEBUG
#define N_PRLE 1024
struct prle {
int status;
int pausecpus;
};
static int prl_entry;
#define PAGE_RELOCATE_LOG(t, r, s, p) \
mutex_enter(&prl_mutex); \
#else /* !DEBUG */
#define PAGE_RELOCATE_LOG(t, r, s, p)
#endif
/*
* Core Kernel Page Relocation Algorithm
*
* Input:
*
* target : constituent pages are SE_EXCL locked.
* replacement: constituent pages are SE_EXCL locked.
*
* Output:
*
* nrelocp: number of pages relocated
*/
int
{
int old_pil;
int cap_cpus;
int ret;
return (EAGAIN);
}
repl = *replacement;
/*
* unload VA<->PA mappings that are not locked
*/
for (i = 0; i < npages; i++) {
tpp++;
}
/*
* Do "presuspend" callbacks, in a context from which we can still
* block as needed. Note that we don't hold the mapping list lock
* of "targ" at this point due to potential locking order issues;
* we assume that between the hat_pageunload() above and holding
* the SE_EXCL lock that the mapping list *cannot* change at this
* point.
*/
if (ret != 0) {
/*
* EIO translates to fatal error, for all others cleanup
* and return EAGAIN.
*/
return (EAGAIN);
}
/*
* acquire p_mapping list lock for both the target and replacement
* root pages.
*
* low and high refer to the need to grab the mlist locks in a
* specific order in order to prevent race conditions. Thus the
* lower lock must be grabbed before the higher lock.
*
* This will block hat_unload's accessing p_mapping list. Since
* we have SE_EXCL lock, hat_memload and hat_pageunload will be
* blocked. Thus, no one else will be accessing the p_mapping list
* while we suspend and reload the locked mapping below.
*/
#ifdef VAC
/*
* If the replacement page is of a different virtual color
* than the page it is replacing, we need to handle the VAC
* consistency for it just as we would if we were setting up
* a new mapping to a page.
*/
}
}
#endif
/*
* We raise our PIL to 13 so that we don't get captured by
* another CPU or pinned by an interrupt thread. We can't go to
* PIL 14 since the nexus driver(s) may need to interrupt at
* that level in the case of IOMMU pseudo mappings.
*/
} else {
old_pil = -1;
}
/*
* Now do suspend callbacks. In the case of an IOMMU mapping
* this will suspend all DMA activity to the page while it is
* being relocated. Since we are well above LOCK_LEVEL and CPUs
* may be captured at this point we should have acquired any needed
* locks in the presuspend callback.
*/
if (ret != 0) {
goto suspend_fail;
}
/*
* Raise the PIL yet again, this time to block all high-level
* interrupts on this CPU. This is necessary to prevent an
* interrupt routine from pinning the thread which holds the
* mapping suspended and then touching the suspended page.
*
* Once the page is suspended we also need to be careful to
* avoid calling any functions which touch any seg_kmem memory
* since that memory may be backed by the very page we are
* relocating in here!
*/
/*
* Now that we are confident everybody has stopped using this page,
* copy the page contents. Note we use a physical copy to prevent
* locking issues and to avoid fpRAS because we can't handle it in
* this context.
*/
/*
* Copy the contents of the page.
*/
}
/*
* Copy attributes. VAC consistency was handled above,
* if required.
*/
#ifdef VAC
#endif
}
/*
* First, unsuspend the page, if we set the suspend bit, and transfer
* the mapping list from the target page to the replacement page.
* Next process postcallbacks; since pa_hment's are linked only to the
* p_mapping list of root page, we don't iterate over the constituent
* pages.
*/
/*
* Now lower our PIL and release any captured CPUs since we
* are out of the "danger zone". After this it will again be
* safe to acquire adaptive mutex locks, or to drop them...
*/
if (old_pil != -1) {
} else {
}
/*
* Postsuspend callbacks should drop any locks held across
* the suspend callbacks. As before, we don't hold the mapping
* list lock at this point.. our assumption is that the mapping
* list still can't change due to our holding SE_EXCL lock and
* there being no unlocked mappings left. Hence the restriction
* on calling context to hat_delete_callback()
*/
if (ret != 0) {
/*
* The second presuspend call failed: we got here through
* the suspend_fail label above.
*/
return (EAGAIN);
}
/*
* Now that we're out of the performance critical section we can
* take care of updating the hash table, since we still
* hold all the pages locked SE_EXCL at this point we
* needn't worry about things changing out from under us.
*/
/*
* replace targ with replacement in page_hash table
*/
/*
* concatenate target; caller of platform_page_relocate()
* expects target to be concatenated after returning.
*/
}
return (0);
}
/*
* Called when stray pa_hments are found attached to a page which is
* being freed. Notify the subsystem which attached the pa_hment of
* the error if it registered a suitable handler, else panic.
*/
static void
{
return; /* non-fatal */
}
}
/*
* Remove all mappings to page 'pp'.
*/
int
{
#ifdef VAC
#endif
int xhme_blks;
int pa_hments;
xhme_blks = 0;
pa_hments = 0;
pa_hments++;
continue;
}
if (hmeblkp->hblk_xhat_bit) {
struct xhat_hme_blk *xblk =
(struct xhat_hme_blk *)hmeblkp;
xhme_blks = 1;
continue;
}
/*
* If there are kernel mappings don't unload them, they will
* be suspended.
*/
continue;
}
while (index != 0) {
if (index != 0)
cons++;
if (index & 0x1) {
/* Go to leading page */
goto retry;
}
}
/*
* cpuset may be empty if the page was only mapped by segkpm,
* in which case we won't actually cross-trap.
*/
/*
* The page should have no mappings at this point, unless
* we were called from hat_page_relocate() in which case we
* leave the locked mappings which will be suspended later.
*/
(forceflag == SFMMU_KERNEL_RELOC));
#ifdef VAC
} else {
}
}
#endif /* VAC */
/*
* Unlink any pa_hments and free them, calling back
* the responsible subsystem to notify it of the error.
* This can occur in situations such as drivers leaking
* DMA handles: naughty, but common enough that we'd like
* to keep the system running rather than bringing it
* down with an obscure error like "pa_hment leaked"
* which doesn't aid the user in debugging their driver.
*/
}
}
}
/*
* XHAT may not have finished unloading pages
* because some other thread was waiting for
* mlist lock and XHAT_PAGEUNLOAD let it do
* the job.
*/
if (xhme_blks) {
goto retry_xhat;
}
return (0);
}
{
#ifdef DEBUG
#endif /* DEBUG */
int ttesz;
int ret;
if (TTE_IS_VALID(&tte)) {
/*
* Only unload mappings of 'cons' size.
*/
return (cpuset);
/*
* Note that we have p_mapping lock, but no hash lock here.
* hblk_unload() has to have both hash lock AND p_mapping
* lock before it tries to modify tte. So, the tte could
* not become invalid in the sfmmu_modifytte_try() below.
*/
#ifdef DEBUG
#endif /* DEBUG */
if (ret < 0) {
#ifdef DEBUG
/* only R/M bits can change. */
#endif /* DEBUG */
goto readtte;
}
if (ret == 0) {
panic("pageunload: cas failed?");
}
/*
* We need to flush the page from the virtual cache
* in order to prevent a virtual cache alias
* inconsistency. The particular scenario we need
* to worry about is:
* Given: va1 and va2 are two virtual address that
* alias and will map the same physical address.
* 1. mapping exists from va1 to pa and data has
* been read into the cache.
* 2. unload va1.
* 3. load va2 and modify data using va2.
* 4 unload va2.
* 5. load va1 and reference data. Unless we flush
* the data cache when we unload we will get
* stale data.
* This scenario is taken care of by using virtual
* page coloring.
*/
if (sfmmup->sfmmu_ismhat) {
/*
* Flush TSBs, TLBs and caches
* of every process
* sharing this ism segment.
*/
if (do_virtual_coloring)
else
} else if (do_virtual_coloring) {
} else {
CACHE_FLUSH, 0);
}
/*
* Hme_sub has to run after ttesync() and a_rss update.
* See hblk_unload().
*/
membar_stst();
/*
* We can not make ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS)
* since pteload may have done a HME_ADD() right after
* we did the HME_SUB() above. Hmecnt is now maintained
* by cas only. no lock guranteed its value. The only
* gurantee we have is the hmecnt should not be less than
* what it should be so the hblk will not be taken away.
* It's also important that we decremented the hmecnt after
* we are done with hmeblkp so that this hmeblk won't be
* stolen.
*/
/*
* This is bug 4063182.
* XXX: fixme
* ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt ||
* !hmeblkp->hblk_lckcnt);
*/
} else {
panic("invalid tte? pp %p &tte %p",
}
return (cpuset);
}
/*
* While relocating a kernel page, this function will move the mappings
* from tpp to dpp and modify any associated data with these mappings.
* It also unsuspends the suspended kernel mapping.
*/
static void
{
/* Update real mappings to the page */
continue;
/*
* replace old pfn with new pfn in TTE
*/
/*
* clear suspend bit
*/
panic("hat_pagereload(): sfmmu_modifytte_try() failed");
/*
* set hme_page point to new page
*/
}
/*
* move p_mapping list from old page to new page
*/
while (index != 0) {
if (index != 0)
cons++;
if (index & 0x1) {
goto retry;
}
}
if (dtrace_kreloc_fini)
(*dtrace_kreloc_fini)();
}
{
return (PP_GENERIC_ATTR(pp));
}
return (PP_GENERIC_ATTR(pp));
}
return (PP_GENERIC_ATTR(pp));
}
if ((clearflag & HAT_SYNC_STOPON_SHARED) != 0 &&
!(clearflag & HAT_SYNC_ZERORM)) {
return (PP_GENERIC_ATTR(pp));
}
/*
* We need to save the next hment on the list since
* it is possible for pagesync to remove an invalid hment
* from the list.
*/
/*
* If we are looking for large mappings and this hme doesn't
* reach the range we are seeking, just ignore its.
*/
if (hmeblkp->hblk_xhat_bit)
continue;
continue;
/*
* If clearflag is HAT_SYNC_DONTZERO, break out as soon
* as the "ref" or "mod" is set.
*/
index = 0;
break;
}
}
while (index) {
cons++;
if (index & 0x1) {
/* Go to leading page */
goto retry;
}
}
return (PP_GENERIC_ATTR(save_pp));
}
/*
* Get all the hardware dependent attributes for a page struct
*/
static cpuset_t
{
int ret;
(clearflag == HAT_SYNC_ZERORM));
if (TTE_IS_VALID(&tte)) {
if (clearflag == HAT_SYNC_ZERORM) {
TTE_CLR_RM(&ttemod);
if (ret < 0) {
/*
* cas failed and the new value is not what
* we want.
*/
goto sfmmu_pagesync_retry;
}
if (ret > 0) {
/* we win the cas */
}
}
}
return (cpuset);
}
/*
* Remove write permission from a mappings to a page, so that
* we can detect the next modification of it. This requires modifying
* the TTE then invalidating (demap) any TLB entry using that TTE.
* This code is similar to sfmmu_pagesync().
*/
static cpuset_t
{
int ret;
/*
* xhat mappings should never be to a VMODSORT page.
*/
/*
* if cas failed and the new value is not what
* we want retry
*/
if (ret < 0)
goto retry;
/* we win the cas */
if (ret > 0) {
}
}
return (cpuset);
}
/*
* Walk all mappings of a page, removing write permission and clearing the
*/
static void
{
int index;
int cons;
/*
* If we are looking for large mappings and this hme doesn't
* reach the range we are seeking, just ignore its.
*/
continue;
}
while (index) {
cons++;
if (index & 0x1) {
/* Go to leading page */
goto retry;
}
}
}
/*
* For a vnode with a sorted v_pages list, we need to change
* the attributes and the v_pages list together under page_vnode_mutex.
*/
void
{
/*
* nothing to do if attribute already set
*/
return;
}
/*
* Some File Systems examine v_pages for NULL w/o
* grabbing the vphm mutex. Must not let it become NULL when
* pp is the only page on the list.
*/
else
}
}
}
void
{
/*
* For vnode with a sorted v_pages list, we need to change
* the attributes and the v_pages list together under page_vnode_mutex.
*/
}
/*
* Some File Systems examine v_pages for NULL w/o
* grabbing the vphm mutex. Must not let it become NULL when
* pp is the only page on the list.
*/
}
/*
* VMODSORT works by removing write permissions and getting
* a fault when a page is made dirty. At this point
* we need to remove write permission from all mappings
* to this page.
*/
}
}
{
}
/*
* DEBUG kernels: verify that a kernel va<->pa translation
* is safe by checking the underlying page_t is in a page
* relocation-safe state.
*/
#ifdef DEBUG
void
{
if (hat_check_vtop == 0)
return;
return;
if (!pp)
return;
return;
/*
* Handed a large kernel page, we dig up the root page since we
* know the root page might have the lock also.
*/
while (index != 0) {
index >>= 1;
if (index != 0)
cons++;
if (index & 0x1) {
goto again;
}
}
}
return;
/*
* Pages need to be locked or allocated "permanent" (either from
* static_arena arena or explicitly setting PG_NORELOC when calling
* page_create_va()) for VA->PA translations to be valid.
*/
if (!PP_ISNORELOC(pp))
else
}
#endif /* DEBUG */
/*
* Returns a page frame number for a given virtual address.
* Returns PFN_INVALID to indicate an invalid mapping
*/
{
/*
* We would like to
* ASSERT(AS_LOCK_HELD(as, &as->a_lock));
* but we can't because the iommu driver will call this
* routine at interrupt time and it can't grab the as lock
* or it will deadlock: A thread could have the as lock
* and be waiting for io. The io can't complete
* because the interrupt thread is blocked trying to grab
* the as lock.
*/
return (sfmmu_kpm_vatopfn(addr));
== PFN_SUSPENDED) {
}
return (pfn);
} else {
}
}
/*
* hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged.
* Use hat_getpfnum(kas.a_hat, ...) instead.
*
* We'd like to return PFN_INVALID if the mappings have underlying page_t's
* but can't right now due to the fact that some software has grown to use
* this interface incorrectly. So for now when the interface is misused,
* return a warning to the user that in the future it won't work in the
* way they're abusing it, and carry on (after disabling page relocation).
*/
{
int badcaller = 0;
extern int segkmem_reloc;
badcaller = 1;
} else {
== PFN_SUSPENDED) {
}
}
if (badcaller) {
/*
* We can't return PFN_INVALID or the caller may panic
* or corrupt the system. The only alternative is to
* disable page relocation at this point for all kernel
* memory. This will impact any callers of page_relocate()
* such as FMA or DR.
*
* RFE: Add junk here to spit out an ereport so the sysadmin
* can be advised that he should upgrade his device driver
* so that this doesn't happen.
*/
if (hat_kpr_enabled && segkmem_reloc) {
hat_kpr_enabled = 0;
segkmem_reloc = 0;
}
}
return (pfn);
}
{
struct hmehash_bucket *hmebp;
/* support for ISM */
int i;
/*
* Set ism_hatid if vaddr falls in a ISM segment.
*/
if (ism_blkp) {
sfmmu_ismhat_enter(sfmmup, 0);
}
break;
}
}
}
if (locked_hatid) {
}
do {
if (TTE_IS_VALID(&tte)) {
} else {
pfn = PFN_INVALID;
}
return (pfn);
}
hashno++;
return (PFN_INVALID);
}
/*
* For compatability with AT&T and later optimizations
*/
/* ARGSUSED */
void
{
}
/*
* Return the number of mappings to a particular page.
* This number is an approximation of the number of
* number of people sharing the page.
*/
{
/*
* We need to grab the mlist lock to make sure any outstanding
* even though the unload(s) hasn't finished yet.
*/
if (kpm_enable)
/*
* If we have any large mappings, we count the number of
* mappings that this large page is part of.
*/
index >>= 1;
while (index) {
}
index >>= 1;
sz++;
}
return (cnt);
}
/*
* Unload all large mappings to the pp and reset the p_szc field of every
* constituent page according to the remaining mappings.
*
* pp must be locked SE_EXCL. Even though no other constituent pages are
* locked it's legal to unload the large mappings to the pp because all
* constituent pages of large locked mappings have to be locked SE_SHARED.
* This means if we have SE_EXCL lock on one of constituent pages none of the
* large mappings to pp are locked.
*
* Decrease p_szc field starting from the last constituent page and ending
* with the root page. This method is used because other threads rely on the
* root's p_szc to find the lock to syncronize on. After a root page_t's p_szc
* is demoted then other threads will succeed in sfmmu_mlspl_enter(). This
* ensures that p_szc changes of the constituent pages appears atomic for all
* threads that use sfmmu_mlspl_enter() to examine p_szc field.
*
* This mechanism is only used for file system pages where it's not always
* possible to get SE_EXCL locks on all constituent pages to demote the size
* code (as is done for anonymous or kernel large pages).
*
* See more comments in front of sfmmu_mlspl_enter().
*/
void
{
int index;
int sz;
int sync = 0;
if (pszc == 0) {
goto out;
}
if (index) {
sync = 1;
}
while (index) {
if (!(index & 0x1)) {
index >>= 1;
sz++;
continue;
}
continue;
}
if (hmeblkp->hblk_xhat_bit) {
"hat_page_demote: xhat hmeblk");
}
}
if (index >>= 1) {
sz++;
}
}
if (sync) {
#ifdef VAC
}
#endif /* VAC */
}
while (--npgs > 0) {
}
if (sz) {
/*
* make sure before current root's pszc
* is updated all updates to constituent pages pszc
* fields are globally visible.
*/
}
}
}
if (sz == 0) {
/* the loop above doesn't cover this case */
}
out:
}
}
/*
* Refresh the HAT ismttecnt[] element for size szc.
* Caller must have set ISM busy flag to prevent mapping
* lists from changing while we're traversing them.
*/
{
int j;
}
return (npgs);
}
/*
* Yield the memory claim requirement for an address space.
*
* This is currently implemented as the number of bytes that have active
* hardware translations that have page structures. Therefore, it can
* underestimate the traditional resident set size, eg, if the
* physical page is present and the hardware translation is missing;
* and it can overestimate the rss, eg, if there are active
* translations to a frame buffer with page structs.
* Also, it does not take sharing into account.
*
* Note that we don't acquire locks here since this function is most often
* called from the clock thread.
*/
{
int i;
return (0);
for (i = 0; i < mmu_page_sizes; i++)
return (assize);
for (i = 0; i < mmu_page_sizes; i++)
return (assize);
}
int
{
hat->sfmmu_rmstat++;
return (1);
}
void
{
hat->sfmmu_rmstat--;
}
/*
* Routines for entering or removing ourselves from the
* ism_hat's mapping list.
*/
static void
{
if (ism_hat->sfmmu_iment) {
}
}
static void
{
panic("ism map entry remove - no entries");
}
if (iment->iment_prev) {
} else {
}
if (iment->iment_next) {
}
/*
* zero out the entry
*/
}
/*
* Hat_share()/unshare() return an (non-zero) error
* when saddr and daddr are not properly aligned.
*
* The top level mapping element determines the alignment
* requirement for saddr and daddr, depending on different
* architectures.
*
* When hat_share()/unshare() are not supported,
* HATOP_SHARE()/UNSHARE() return 0
*/
int
{
int i, added;
int reload_mmu = 0;
#ifdef DEBUG
#endif /* DEBUG */
/*
* Check the alignment.
*/
return (EINVAL);
/*
* Check size alignment.
*/
return (EINVAL);
/*
* Allocate ism_ment for the ism_hat's mapping list, and an
* ism map blk in case we need one. We must do our
* allocations before acquiring locks to prevent a deadlock
* in the kmem allocator on the mapping list lock.
*/
/*
* Serialize ISM mappings with the ISM busy flag, and also the
* trap handlers.
*/
sfmmu_ismhat_enter(sfmmup, 0);
/*
* Allocate an ism map blk if necessary.
*/
membar_stst(); /* make sure next ptr visible to all CPUs */
reload_mmu = 1;
}
#ifdef DEBUG
/*
* Make sure mapping does not already exist.
*/
while (ism_blkp) {
panic("sfmmu_share: Already mapped!");
}
}
}
#endif /* DEBUG */
}
/*
* Add mapping to first available mapping slot.
*/
added = 0;
while (!added) {
for (i = 0; i < ISM_MAP_SLOTS; i++) {
/*
* imap_seg is checked in ISM_CHECK to see if
* non-NULL, then other info assumed valid.
*/
membar_stst();
/*
* Now add ourselves to the ism_hat's
* mapping list.
*/
ism_hatid->sfmmu_flags = 0;
added = 1;
break;
}
}
membar_stst();
}
}
/*
* Update our counters for this sfmmup's ism mappings.
*/
for (i = 0; i <= ismszc; i++) {
if (!(disable_ism_large_pages & (1 << i)))
(void) ism_tsb_entries(sfmmup, i);
}
/*
* For ISM and DISM we do not support 512K pages, so we only
* only search the 4M and 8K/64K hashes for 4 pagesize cpus, and search
* the 256M or 32M, and 4M and 8K/64K hashes for 6 pagesize cpus.
*/
/*
* If we updated the ismblkpa for this HAT or we need
* to start searching the 256M or 32M or 4M hash, we must
* make sure all CPUs running this process reload their
* tsbmiss area. Otherwise they will fail to load the mappings
* in the tsbmiss handler and will loop calling pagefault().
*/
switch (ismszc) {
case TTE256M:
}
break;
case TTE32M:
}
break;
case TTE4M:
}
break;
default:
break;
}
/*
* Now we can drop the locks.
*/
/*
* Free up ismblk if we didn't use it.
*/
/*
* Check TSB and TLB page sizes.
*/
return (0);
}
/*
* hat_unshare removes exactly one ism_map from
* this process's as. It expects multiple calls
* to hat_unshare for multiple shm segments.
*/
void
{
int found, i;
if (sfmmup->sfmmu_xhat_provider) {
return;
} else {
/*
* This must be a CPU HAT. If the address space has
* XHATs attached, inform all XHATs that ISM segment
* is going away
*/
}
/*
* Make sure that during the entire time ISM mappings are removed,
* the trap handlers serialize behind us, and that no one else
* can be mucking with ISM mappings. This also lets us get away
* with not doing expensive cross calls to flush the TLB -- we
* just discard the context, flush the entire TSB, and call it
* a day.
*/
sfmmu_ismhat_enter(sfmmup, 0);
/*
* Remove the mapping.
*
* We can't have any holes in the ism map.
* The tsb miss code while searching the ism map will
* stop on an empty map slot. So we must move
* everyone past the hole up 1 if any.
*
* Also empty ism map blks are not freed until the
* process exits. This is to prevent a MT race condition
* between sfmmu_unshare() and sfmmu_tsbmiss_exception().
*/
found = 0;
for (i = 0; i < ISM_MAP_SLOTS; i++) {
found = 1;
break;
}
}
if (!found)
}
if (found) {
/*
* First remove ourselves from the ism mapping list.
*/
/*
* Now gurantee that any other cpu
* that tries to process an ISM miss
* will go to tl=0.
*/
/*
* We delete the ism map by copying
* the next map over the current one.
* We will take the next one in the maps
* array or from the next ism_blk.
*/
while (ism_blkp) {
while (i < (ISM_MAP_SLOTS - 1)) {
i++;
}
/* i == (ISM_MAP_SLOTS - 1) */
if (ism_blkp) {
i = 0;
} else {
ism_map[i].imap_vb_shift = 0;
ism_map[i].imap_hatflags = 0;
ism_map[i].imap_sz_mask = 0;
}
}
/*
* Now flush entire TSB for the process, since
* demapping page by page can be too expensive.
* We don't have to flush the TLB here anymore
* since we switch to a new TLB ctx instead.
* Also, there is no need to flush if the process
* is exiting since the TSB will be freed later.
*/
if (!sfmmup->sfmmu_free) {
continue;
}
}
}
/*
* Update our counters for this sfmmup's ism mappings.
*/
for (i = 0; i <= ismszc; i++) {
if (!(disable_ism_large_pages & (1 << i)))
(void) ism_tsb_entries(sfmmup, i);
}
sfmmu_ismhat_exit(sfmmup, 0);
/*
* We must do our freeing here after dropping locks
* to prevent a deadlock in the kmem allocator on the
* mapping list lock.
*/
/*
* Check TSB and TLB page sizes if the process isn't exiting.
*/
if (!sfmmup->sfmmu_free)
}
/* ARGSUSED */
static int
{
/* void *buf is sfmmu_t pointer */
return (0);
}
/* ARGSUSED */
static void
{
/* void *buf is sfmmu_t pointer */
}
/*
* setup kmem hmeblks by bzeroing all members and initializing the nextpa
* field to be the pa of this hmeblk
*/
/* ARGSUSED */
static int
{
#ifdef HBLK_TRACE
#endif /* HBLK_TRACE */
return (0);
}
/* ARGSUSED */
static void
{
#ifdef HBLK_TRACE
#endif /* HBLK_TRACE */
}
#define SFMMU_CACHE_RECLAIM_SCAN_RATIO 8
static int sfmmu_cache_reclaim_scan_ratio = SFMMU_CACHE_RECLAIM_SCAN_RATIO;
/*
* The kmem allocator will callback into our reclaim routine when the system
* is running low in memory. We traverse the hash and free up all unused but
* still cached hme_blks. We also traverse the free list and free them up
* as well.
*/
/*ARGSUSED*/
static void
sfmmu_hblkcache_reclaim(void *cdrarg)
{
int i;
struct hmehash_bucket *hmebp;
static struct hmehash_bucket *uhmehash_reclaim_hand;
static struct hmehash_bucket *khmehash_reclaim_hand;
for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) {
if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) {
prevpa = 0;
while (hmeblkp) {
!hmeblkp->hblk_hmecnt) {
} else {
}
}
}
}
for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) {
if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) {
prevpa = 0;
while (hmeblkp) {
!hmeblkp->hblk_hmecnt) {
} else {
}
}
}
}
}
/*
* sfmmu_get_ppvcolor should become a vm_machdep or hatop interface.
* same goes for sfmmu_get_addrvcolor().
*
* This function will return the virtual color for the specified page. The
* virtual color corresponds to this page current mapping or its last mapping.
* It is used by memory allocators to choose addresses with the correct
* alignment so vac consistency is automatically maintained. If the page
* has no color it returns -1.
*/
/*ARGSUSED*/
int
{
#ifdef VAC
int color;
return (-1);
}
return (color);
#else
return (-1);
#endif /* VAC */
}
/*
* This function will return the desired alignment for vac consistency
* (vac color) given a virtual address. If no vac is present it returns -1.
*/
/*ARGSUSED*/
int
{
#ifdef VAC
return (addr_to_vcolor(vaddr));
} else {
return (-1);
}
#else
return (-1);
#endif /* VAC */
}
#ifdef VAC
/*
* Check for conflicts.
* A conflict exists if the new and existent mappings do not match in
* their "shm_alignment fields. If conflicts exist, the existant mappings
* are flushed unless one of them is locked. If one of them is locked, then
* the mappings are flushed and converted to non-cacheable mappings.
*/
static void
{
int vcolor;
if (PP_NEWPAGE(pp)) {
return;
}
return;
}
/*
* Previous user of page had a different color
* but since there are no current users
* we just flush the cache and change the color.
*/
return;
}
/*
* If we get here we have a vac conflict with a current
* mapping. VAC conflict policy is as follows.
* - The default is to unload the other mappings unless:
* - If we have a large mapping we uncache the page.
* We need to uncache the rest of the large page too.
* - If any of the mappings are locked we uncache the page.
* - If the requested mapping is inconsistent
* with another mapping and that mapping
* is in the same address space we have to
* make it non-cached. The default thing
* to do is unload the inconsistent mapping
* but if they are in the same address space
* we run the risk of unmapping the pc or the
* stack which we will use as we return to the user,
* in which case we can then fault on the thing
* we just unloaded and get into an infinite loop.
*/
if (PP_ISMAPPED_LARGE(pp)) {
int sz;
/*
* Existing mapping is for big pages. We don't unload
* existing big mappings to satisfy new mappings.
* Always convert all mappings to TNC.
*/
return;
}
/*
* check if any mapping is in same as or if it is locked
* since in that case we need to uncache.
*/
if (hmeblkp->hblk_xhat_bit)
continue;
/*
* We have an uncache conflict
*/
return;
}
}
/*
* We have an unload conflict
* We have already checked for LARGE mappings, therefore
* the remaining mapping(s) must be TTE8K.
*/
if (hmeblkp->hblk_xhat_bit)
continue;
}
if (PP_ISMAPPED_KPM(pp))
/*
* Unloads only do TLB flushes so we need to flush the
* cache here.
*/
}
/*
* Whenever a mapping is unloaded and the page is in TNC state,
* we see if the page can be made cacheable again. 'pp' is
* the page that we just unloaded a mapping from, the size
* of mapping that was unloaded is 'ottesz'.
* Remark:
* The recache policy for mpss pages can leave a performance problem
* under the following circumstances:
* . A large page in uncached mode has just been unmapped.
* . All constituent pages are TNC due to a conflicting small mapping.
* . There are many other, non conflicting, small mappings around for
* a lot of the constituent pages.
* . We're called w/ the "old" groupleader page and the old ottesz,
* but this is irrelevant, since we're no more "PP_ISMAPPED_LARGE", so
* we end up w/ TTE8K or npages == 1.
* . We call tst_tnc w/ the old groupleader only, and if there is no
* conflict, we re-cache only this page.
* . All other small mappings are not checked and will be left in TNC mode.
* The problem is not very serious because:
* . mpss is actually only defined for heap and stack, so the probability
* is not very high that a large page mapping exists in parallel to a small
* one (this is possible, but seems to be bad programming style in the
* appl).
* . The problem gets a little bit more serious, when those TNC pages
* have to be mapped into kernel space, e.g. for networking.
* . When VAC alias conflicts occur in applications, this is regarded
* as an application bug. So if kstat's show them, the appl should
* be changed anyway.
*/
void
{
/*
* Determine how big a range we check for TNC and find
* leader page. cursz is the size of the biggest
* mapping that still exist on 'pp'.
*/
if (PP_ISMAPPED_LARGE(pp)) {
} else {
}
} else {
}
while (dopgs != 0) {
curnpgs);
}
if (dopgs == 0) {
break;
}
} else {
}
}
}
/*
* Returns 1 if page(s) can be converted from TNC to cacheable setting,
* returns 0 otherwise. Note that oaddr argument is valid for only
* 8k pages.
*/
int
{
int clr_valid = 0;
int i, ncolors;
if (npages > 1) {
}
for (i = 0; i < npages; i++) {
return (0);
}
clr_valid = 0;
if (PP_ISMAPPED_KPM(pp)) {
clr_valid = 1;
}
if (hmeblkp->hblk_xhat_bit)
continue;
if (npages > 1) {
/*
* If there is a big mapping, make sure
* 8K mapping is consistent with the big
* mapping.
*/
return (0);
}
}
if (!clr_valid) {
clr_valid = 1;
}
return (0);
}
}
}
return (1);
}
void
{
/*
* Fast path caching single unmapped page
*/
return;
}
/*
* We need to capture all cpus in order to change cacheability
* because we can't allow one cpu to access the same physical
* page using a cacheable and a non-cachebale mapping at the same
* time. Since we may end up walking the ism mapping list
* have to grab it's lock now since we can't after all the
* cpus have been captured.
*/
if (npages > 1) {
/*
* Make sure all colors are flushed since the
* sfmmu_page_cache() only flushes one color-
* it does not know big pages.
*/
for (i = 0; i < ncolors; i++) {
}
}
}
for (i = 0; i < npages; i++) {
if (npages > 1) {
} else {
}
bcolor);
}
}
}
/*
* This function changes the virtual cacheability of all mappings to a
* particular page. When changing from uncache to cacheable the mappings will
* only be changed if all of them have the same virtual color.
* We need to flush the cache in all cpus. It is possible that
* a process referenced a page as cacheable but has sinced exited
* and cleared the mapping list. We still to flush it but have no
* state so all cpus is the only alternative.
*/
static void
{
if (hmeblkp->hblk_xhat_bit)
continue;
#ifdef DEBUG
}
#endif
} else { /* flags & HAT_CACHE */
}
if (ret < 0) {
/*
* Since all cpus are captured modifytte should not
* fail.
*/
panic("sfmmu_page_cache: write to tte failed");
}
if (cache_flush_flag == CACHE_FLUSH) {
/*
* Flush TSBs, TLBs and caches
*/
if (sfmmup->sfmmu_ismhat) {
} else {
}
pfn, CACHE_FLUSH);
} else {
}
/*
* all cache entries belonging to this pfn are
* now flushed.
*/
} else {
/*
* Flush only TSBs and TLBs.
*/
if (sfmmup->sfmmu_ismhat) {
} else {
}
} else {
}
}
}
if (PP_ISMAPPED_KPM(pp))
switch (flags) {
default:
panic("sfmmu_pagecache: unknown flags");
break;
case HAT_CACHE:
break;
case HAT_TMPNC:
break;
case HAT_UNCACHE:
break;
}
}
#endif /* VAC */
/*
* Wrapper routine used to return a context.
*
* It's the responsibility of the caller to guarantee that the
* process serializes on calls here by taking the HAT lock for
* the hat.
*
*/
static void
{
/*
* Do a wrap-around if cnum reaches the max # cnum supported by a MMU.
*/
/*
* Let the MMU set up the page sizes to use for
* this context in the TLB. Don't program 2nd dtlb for ism hat.
*/
}
/*
* sfmmu_alloc_ctx and sfmmu_load_mmustate will be performed with
* interrupts disabled to prevent race condition with wrap-around
* ctx invalidatation. In sun4v, ctx invalidation also involves
* a HV call to set the number of TSBs to 0. If interrupts are not
* disabled until after sfmmu_load_mmustate is complete TSBs may
* become assigned to INVALID_CONTEXT. This is not allowed.
*/
}
/*
* When all cnums are used up in a MMU, cnum will wrap around to the
* next generation and start from 2.
*/
static void
{
/* caller must have disabled the preemption */
/* acquire Per-MMU (PM) spin lock */
/* re-check to see if wrap-around is needed */
goto done;
/* update gnum */
(void *)mmu_ctxp);
}
membar_enter(); /* make sure updated gnum visible */
/* xcall to others on the same MMU to invalidate ctx */
/*
* Pass in INVALID_CONTEXT as the first parameter to
* sfmmu_raise_tsb_exception, which invalidates the context
* of any process running on the CPUs in the MMU.
*/
}
if (sfmmu_getctx_sec() != INVALID_CONTEXT) {
}
/*
* No xcall is needed here. For sun4u systems all CPUs in context
* domain share a single physical MMU therefore it's enough to flush
* TLB on local CPU. On sun4v systems we use 1 global context
* domain and flush all remote TLBs in sfmmu_raise_tsb_exception
* handler. Note that vtag_flushall_uctxs() is called
* for Ultra II machine, where the equivalent flushall functionality
* is implemented in SW, and only user ctx TLB entries are flushed.
*/
if (&vtag_flushall_uctxs != NULL) {
} else {
}
/* reset mmu cnum, skips cnum 0 and 1 */
done:
}
/*
* For multi-threaded process, set the process context to INVALID_CONTEXT
* so that it faults and reloads the MMU state from TL=0. For single-threaded
* process, we can just load the MMU state directly without having to
* set context invalid. Caller must hold the hat lock since we don't
* acquire it here.
*/
static void
{
/*
* We check whether the pass'ed-in sfmmup is the same as the
* current running proc. This is to makes sure the current proc
* stays single-threaded if it already is.
*/
/* single-thread */
if (cnum != INVALID_CONTEXT) {
/*
* Disable interrupts to prevent race condition
* with sfmmu_ctx_wrap_around ctx invalidation.
* In sun4v, ctx invalidation involves setting
* TSB to NULL, hence, interrupts should be disabled
* untill after sfmmu_load_mmustate is completed.
*/
curcnum = sfmmu_getctx_sec();
}
} else {
/*
* multi-thread
* or when sfmmup is not the same as the curproc.
*/
}
}
/*
* Replace the specified TSB with a new TSB. This function gets called when
* we grow, shrink or swapin a TSB. When swapping in a TSB (TSB_SWAPIN), the
* TSB_FORCEALLOC flag may be used to force allocation of a minimum-sized TSB
* (8K).
*
* Caller must hold the HAT lock, but should assume any tsb_info
* pointers it has are no longer valid after calling this function.
*
* Return values:
* TSB_ALLOCFAIL Failed to allocate a TSB, due to memory constraints
* TSB_LOSTRACE HAT is busy, i.e. another thread is already doing
* TSB_SUCCESS Operation succeeded
*/
static tsb_replace_rc_t
{
int i;
return (TSB_LOSTRACE);
/*
* Find the tsb_info ahead of this one in the list, and
* also make sure that the tsb_info passed in really
* exists!
*/
/*
* The process is swapped out, so just set the new size
* code. When it swaps back in, we'll allocate a new one
* of the new chosen size.
*/
return (TSB_SUCCESS);
}
/*
* All initialization is done inside of sfmmu_tsbinfo_alloc().
* If we fail to allocate a TSB, exit.
*/
(void) sfmmu_hat_enter(sfmmup);
if (!(flags & TSB_SWAPIN))
return (TSB_ALLOCFAIL);
}
(void) sfmmu_hat_enter(sfmmup);
/*
* Re-check to make sure somebody else didn't muck with us while we
* didn't hold the HAT lock. If the process swapped out, fine, just
* exit; this can happen if we try to shrink the TSB from the context
* of another process (such as on an ISM unmap), though it is rare.
*/
(void) sfmmu_hat_enter(sfmmup);
return (TSB_LOSTRACE);
}
#ifdef DEBUG
/* Reverify that the tsb_info still exists.. for debugging only */
#endif /* DEBUG */
/*
* Quiesce any CPUs running this process on their next TLB miss
* so they atomically see the new tsb_info. We temporarily set the
* context to invalid context so new threads that come on processor
* after we do the xcall to cpusran will also serialize behind the
* HAT lock on TLB miss and will see the new TSB. Since this short
* race with a new thread coming on processor is relatively rare,
* this synchronization mechanism should be cheaper than always
* pausing all CPUs for the duration of the setup, which is what
* the old implementation did. This is particuarly true if we are
* copying a huge chunk of memory around during that window.
*
* The memory barriers are to make sure things stay consistent
* with resume() since it does not hold the HAT lock while
* walking the list of tsb_info structures.
*/
/* The TSB is either growing or shrinking. */
} else {
/*
* It is illegal to swap in TSBs from a process other
* than a process being swapped in. This in turn
* implies we do not have a valid MMU context here
* since a process needs one to resolve translation
* misses.
*/
}
#ifdef DEBUG
ASSERT(max_mmu_ctxdoms > 0);
/*
* Process should have INVALID_CONTEXT on all MMUs
*/
for (i = 0; i < max_mmu_ctxdoms; i++) {
}
#endif
membar_stst(); /* strict ordering required */
if (prevtsb)
else
membar_enter(); /* make sure new TSB globally visible */
/*
* We need to migrate TSB entries from the old TSB to the new TSB
* if tsb_remap_ttes is set and the TSB is growing.
*/
/*
* Drop the HAT lock to free our old tsb_info.
*/
}
(void) sfmmu_hat_enter(sfmmup);
return (TSB_SUCCESS);
}
/*
* This function will re-program hat pgsz array, and invalidate the
* process' context, forcing the process to switch to another
* context on the next TLB miss, and therefore start using the
* TLB that is reprogrammed for the new page sizes.
*/
void
{
int i;
/* USIII+-IV+ optimization, requires hat lock */
if (tmp_pgsz) {
for (i = 0; i < mmu_page_sizes; i++)
}
}
/*
* This function assumes that there are either four or six supported page
* sizes and at most two programmable TLBs, so we need to decide which
* page sizes are most important and then tell the MMU layer so it
* can adjust the TLB page sizes accordingly (if supported).
*
* If these assumptions change, this function will need to be
* updated to support whatever the new limits are.
*
* The growing flag is nonzero if we are growing the address space,
* and zero if it is shrinking. This allows us to decide whether
* to grow or shrink our TSB, depending upon available memory
* conditions.
*/
static void
{
uint8_t i;
int sectsb_thresh;
/*
* Kernel threads, processes with small address spaces not using
* large pages, and dummy ISM HATs need not apply.
*/
return;
return;
for (i = 0; i < mmu_page_sizes; i++) {
}
/* Check pagesizes in use, and possibly reprogram DTLB. */
if (&mmu_check_page_sizes)
/*
* Calculate the number of 8k ttes to represent the span of these
* pages.
*/
if (mmu_page_sizes == max_mmu_page_sizes) {
} else {
}
/*
* Inflate TSB sizes by a factor of 2 if this process
* uses 4M text pages to minimize extra conflict misses
* in the first TSB since without counting text pages
* 8K TSB may become too small.
*
* Also double the size of the second TSB to minimize
* extra conflict misses due to competition between 4M text pages
* and data pages.
*
* We need to adjust the second TSB allocation threshold by the
* inflation factor, since there is no point in creating a second
* TSB when we know all the mappings can fit in the I/D TLBs.
*/
tte8k_cnt <<= 1;
tte4m_cnt <<= 1;
sectsb_thresh <<= 1;
}
/*
* Check to see if our TSB is the right size; we may need to
* grow or shrink it. If the process is small, our work is
* finished at this point.
*/
return;
}
}
static void
{
int tsb_bits;
/*
* If we're growing, select the size based on RSS. If we're
* shrinking, leave some room so we don't have to turn around and
* grow again immediately.
*/
if (growing)
else
}
/*
* With the TLB and first TSB out of the way, we need to see if
* we need a second TSB for 4M pages. If we managed to reprogram
* the TLB page sizes above, the process will start using this new
* TSB right away; otherwise, it will start using it on the next
* context switch. Either way, it's no big deal so there's no
* synchronization with the trap handlers here unless we grow the
* TSB (in which case it's required to prevent using the old one
* after it's freed). Note: second tsb is required for 32M/256M
* page sizes.
*/
if (tte4m_cnt > sectsb_thresh) {
/*
* If we're growing, select the size based on RSS. If we're
* shrinking, leave some room so we don't have to turn
* around and grow again immediately.
*/
if (growing)
else
0 : TSB_ALLOC;
/*
* Try to allocate a TSB for 4[32|256]M pages. If we
* can't get the size we want, retry w/a minimum sized
* TSB. If that still didn't work, give up; we can
* still run without one.
*/
allocflags, sfmmup) != 0) &&
return;
}
return;
} else {
/*
* It's annoying, but possible for us
* to get here.. we dropped the HAT lock
* because of locking order in the kmem
* allocator, and while we were off getting
* our memory, some other thread decided to
* do us a favor and won the race to get a
* second TSB for this process. Sigh.
*/
return;
}
}
/*
* We have a second TSB, see if it's big enough.
*/
/*
* Check to see if our second TSB is the right size;
* we may need to grow or shrink it.
* To prevent thrashing (e.g. growing the TSB on a
* subsequent map operation), only try to shrink if
* the TSB reach exceeds twice the virtual address
* space size.
*/
TSB_OK_GROW()) {
}
}
}
/*
* Get the preferred page size code for a hat.
* This is only advice, so locking is not done;
* this transitory information could change
* following the call anyway. This interface is
* sun4 private.
*/
/*ARGSUSED*/
{
if (maptype == MAPPGSZ_ISM) {
continue;
return (szc);
}
return (TTE4M);
} else if (&mmu_preferred_pgsz) { /* USIII+-USIV+ */
} else { /* USIII, USII, Niagara */
continue;
return (szc);
}
return (TTE8K);
}
}
/*
* Free up a sfmmu
* Since the sfmmu is currently embedded in the hat struct we simply zero
* out our fields and free up the ism map blk list if any.
*/
static void
{
#ifdef DEBUG
int i;
#endif
sfmmup->sfmmu_free = 0;
sfmmup->sfmmu_ismhat = 0;
while (blkp) {
#ifdef DEBUG
for (i = 0; i < ISM_MAP_SLOTS; i++) {
}
#endif
}
}
/*
* Locking primitves accessed by HATLOCK macros
*/
#define SFMMU_SPL_MTX (0x0)
#define SFMMU_ML_MTX (0x1)
kmutex_t *
{
}
void
{
}
int
{
}
kmutex_t *
{
}
void
{
}
int
{
}
/*
* Common code for sfmmu_mlist_enter() and sfmmu_page_enter(). For
* sfmmu_mlist_enter() case mml_table lock array is used and for
* sfmmu_page_enter() sfmmu_page_lock lock array is used.
*
* The lock is taken on a root page so that it protects an operation on all
* constituent pages of a large page pp belongs to.
*
* The routine takes a lock from the appropriate array. The lock is determined
* by hashing the root page. After taking the lock this routine checks if the
* root page has the same size code that was used to determine the root (i.e
* that root hasn't changed). If root page has the expected p_szc field we
* have the right lock and it's returned to the caller. If root's p_szc
* decreased we release the lock and retry from the beginning. This case can
* happen due to hat_page_demote() decreasing p_szc between our load of p_szc
* value and taking the lock. The number of retries due to p_szc decrease is
* limited by the maximum p_szc value. If p_szc is 0 we return the lock
* determined by hashing pp itself.
*
* If our caller doesn't hold a SE_SHARED or SE_EXCL lock on pp it's also
* possible that p_szc can increase. To increase p_szc a thread has to lock
* all constituent pages EXCL and do hat_pageunload() on all of them. All the
* callers that don't hold a page locked recheck if hmeblk through which pp
* was found still maps this pp. If it doesn't map it anymore returned lock
* is immediately dropped. Therefore if sfmmu_mlspl_enter() hits the case of
* p_szc increase after taking the lock it returns this lock without further
* retries because in this case the caller doesn't care about which lock was
* taken. The caller will drop it right away.
*
* After the routine returns it's guaranteed that hat_page_demote() can't
* change p_szc field of any of constituent pages of a large page pp belongs
* to as long as pp was either locked at least SHARED prior to this call or
* the caller finds that hment that pointed to this pp still references this
* pp (this also assumes that the caller holds hme hash bucket lock so that
* the same pp can't be remapped into the same hmeblk after it was unmapped by
* hat_pageunload()).
*/
static kmutex_t *
{
if (pszc == 0) {
return (mtx);
}
/* The lock lives in the root page */
/*
* Return mml in the following 3 cases:
*
* 1) If pp itself is root since if its p_szc decreased before we took
* the lock pp is still the root of smaller szc page. And if its p_szc
* increased it doesn't matter what lock we return (see comment in
* front of this routine).
*
* 2) If pp's not root but rootpp is the root of a rootpp->p_szc size
* large page we have the right lock since any previous potential
* hat_page_demote() is done demoting from greater than current root's
* p_szc because hat_page_demote() changes root's p_szc last. No
* further hat_page_demote() can start or be in progress since it
* would need the same lock we currently hold.
*
* 3) If rootpp's p_szc increased since previous iteration it doesn't
* matter what lock we return (see comment in front of this routine).
*/
return (mtx);
}
/*
* hat_page_demote() could have decreased root's p_szc.
* In this case pp's p_szc must also be smaller than pszc.
* Retry.
*/
goto again;
}
/*
* pp's p_szc increased after it was decreased.
* page cannot be mapped. Return current lock. The caller
* will drop it right away.
*/
return (mtx);
}
/*
* root's p_szc is greater than pp's p_szc.
* hat_page_demote() is not done with all pages
* yet. Wait for it to complete.
*/
goto again;
}
static int
{
/* The lock lives in the root page */
return (MUTEX_HELD(mtx));
}
static uint_t
{
/*
* If the current thread is owning hblk_reserve,
* let it succede even if freehblkcnt is really low.
*/
return (0);
}
freehblkcnt--;
return (1);
}
}
return (0);
}
static uint_t
{
/*
* If the current thread is mapping into kernel space,
* let it succede even if freehblkcnt is max
* so that it will avoid freeing it to kmem.
* This will prevent stack overflow due to
* possible recursion since kmem_cache_free()
* might require creation of a slab which
* in turn needs an hmeblk to map that slab;
* let's break this vicious chain at the first
* opportunity.
*/
freehblkcnt++;
return (1);
}
}
/*
* Bring down freehblkcnt to HBLK_RESERVE_CNT. We are here
* only if freehblkcnt is at least HBLK_RESERVE_CNT *and*
* we are not in the process of mapping into kernel space.
*/
while (freehblkcnt > HBLK_RESERVE_CNT) {
if (freehblkcnt > HBLK_RESERVE_CNT) {
freehblkcnt--;
continue;
}
}
return (0);
}
static void
{
struct hmehash_bucket *hmebp;
#ifdef DEBUG
#endif
old = HBLK_RESERVE;
/*
* save pa before bcopy clobbers it
*/
/*
* acquire hash bucket lock.
*/
/*
* copy contents from old to new
*/
/*
* add new to hash chain
*/
/*
* search hash chain for hblk_reserve; this needs to be performed
* after adding new, otherwise prevpa and prev won't correspond
* to the hblk which is prior to old in hash chain when we call
* sfmmu_hblk_hash_rm to remove old later.
*/
panic("sfmmu_hblk_swap: hblk_reserve not found");
/*
* p_mapping list is still pointing to hments in hblk_reserve;
* fix up p_mapping list so that they point to hments in new.
*
* Since all these mappings are created by hblk_reserve_thread
* on the way and it's using at least one of the buffers from each of
* the newly minted slabs, there is no danger of any of these
* mappings getting unloaded by another thread.
*
* Since all of these hments hold mappings established by segkmem
* have no meaning for the mappings in hblk_reserve. hments in
*/
if (TTE_IS_VALID(&tte)) {
panic("sfmmu_hblk_swap: page not mapped");
panic("sfmmu_hblk_swap: mapping changed");
}
}
/*
* remove old from hash chain
*/
#ifdef DEBUG
panic("sfmmu_hblk_swap: new hblk not found");
#endif
/*
* Reset hblk_reserve
*/
}
/*
* Grab the mlist mutex for both pages passed in.
*
* low and high will be returned as pointers to the mutexes for these pages.
* low refers to the mutex residing in the lower bin of the mlist hash, while
* high refers to the mutex residing in the higher bin of the mlist hash. This
* is due to the locking order restrictions on the same thread grabbing
* multiple mlist mutexes. The low lock must be acquired before the high lock.
*
* If both pages hash to the same mutex, only grab that single mutex, and
* high will be returned as NULL
* If the pages hash to different bins in the hash, grab the lower addressed
* lock first and then the higher addressed lock in order to follow the locking
* rules involved with the same thread grabbing multiple mlist mutexes.
* low and high will both have non-NULL values.
*/
static void
{
/*
* no need to do the dance around szc as in sfmmu_mlist_enter()
* because this routine is only called by hat_page_relocate() and all
* targ and repl pages are already locked EXCL so szc can't change.
*/
} else {
} else {
}
}
mutex_enter(*low);
if (*high)
mutex_enter(*high);
}
static void
{
if (high)
}
static hatlock_t *
{
return (hatlockp);
}
return (NULL);
}
static hatlock_t *
{
return (NULL);
return (hatlockp);
}
return (NULL);
}
static void
{
}
static void
sfmmu_hat_lock_all(void)
{
int i;
for (i = 0; i < SFMMU_NUM_LOCK; i++)
}
static void
sfmmu_hat_unlock_all(void)
{
int i;
for (i = SFMMU_NUM_LOCK - 1; i >= 0; i--)
}
int
{
}
/*
* Locking primitives to provide consistency between ISM unmap
* and other operations. Since ISM unmap can take a long time, we
* use HAT_ISMBUSY flag (protected by the hatlock) to avoid creating
* contention on the hatlock buckets while ISM segments are being
* unmapped. The tradeoff is that the flags don't prevent priority
* inversion from occurring, so we must request kernel priority in
* case we have to sleep to keep from getting buried while holding
* the HAT_ISMBUSY flag set, which in turn could block other kernel
* threads from running (for example, in sfmmu_uvatopfn()).
*/
static void
{
if (!hatlock_held)
if (!hatlock_held)
}
static void
{
if (!hatlock_held)
if (!hatlock_held)
}
/*
*
* Algorithm:
*
* (1) if segkmem is not ready, allocate hblk from an array of pre-alloc'ed
* hblks.
*
* (2) if we are allocating an hblk for mapping a slab in sfmmu_cache,
*
* (a) try to return an hblk from reserve pool of free hblks;
* (b) if the reserve pool is empty, acquire hblk_reserve_lock
* and return hblk_reserve.
*
* (3) call kmem_cache_alloc() to allocate hblk;
*
* (a) if hblk_reserve_lock is held by the current thread,
* atomically replace hblk_reserve by the hblk that is
* returned by kmem_cache_alloc; release hblk_reserve_lock
* and call kmem_cache_alloc() again.
* (b) if reserve pool is not full, add the hblk that is
* returned by kmem_cache_alloc to reserve pool and
* call kmem_cache_alloc again.
*
*/
static struct hme_blk *
{
int sleep;
/*
* If segkmem is not created yet, allocate from static hmeblks
* created at the end of startup_modules(). See the block comment
* in startup_modules() describing how we estimate the number of
* static hmeblks that will be needed during re-map.
*/
if (!hblk_alloc_dynamic) {
/*
* If we panic here, see startup_modules() to
* make sure that we are calculating the
* number of hblk8's that we need correctly.
*/
panic("no nucleus hblk8 to allocate");
}
hmeblkp =
} else {
/*
* If we panic here, see startup_modules()
* and H8TOH1; most likely you need to
* update the calculation of the number
* of hblk1's the kernel needs to boot.
*/
panic("no nucleus hblk1 to allocate");
}
hmeblkp =
}
goto hblk_init;
}
if (mmu_page_sizes == max_mmu_page_sizes) {
} else {
}
}
/*
* We are really in a tight spot. We already own
* hblk_reserve and we need another hblk. In anticipation
* of this kind of scenario, we specifically set aside
* HBLK_RESERVE_MIN number of hblks to be used exclusively
* by owner of hblk_reserve.
*/
panic("sfmmu_hblk_alloc: reserve list is empty");
goto hblk_verify;
}
if ((flags & HAT_NO_KALLOC) == 0) {
} else {
/*
* if we are the owner of hblk_reserve,
* swap hblk_reserve with hmeblkp and
* start a fresh life. Hope things go
* better this time.
*/
if (hblk_reserve_thread == curthread) {
goto fill_hblk;
}
/*
* let's donate this hblk to our reserve list if
* we are not mapping kernel range
*/
if (sfmmu_put_free_hblk(hmeblkp, 0))
goto fill_hblk;
}
} else {
/*
* We are here to map the slab in sfmmu8_cache; let's
* check if we could tap our reserve list; if successful,
* this will avoid the pain of going thru sfmmu_hblk_swap
*/
if (!sfmmu_get_free_hblk(&hmeblkp, 0)) {
/*
* let's start hblk_reserve dance
*/
owner = 1;
}
}
if (hmeblkp != HBLK_RESERVE) {
/*
* This is really tricky!
*
* vmem_alloc(vmem_seg_arena)
* vmem_alloc(vmem_internal_arena)
* segkmem_alloc(heap_arena)
* vmem_alloc(heap_arena)
* page_create()
* hat_memload()
* kmem_cache_free()
* kmem_cache_alloc()
* kmem_slab_create()
* vmem_alloc(kmem_internal_arena)
* segkmem_alloc(heap_arena)
* vmem_alloc(heap_arena)
* page_create()
* hat_memload()
* kmem_cache_free()
* ...
*
* Thus, hat_memload() could call kmem_cache_free
* for enough number of times that we could easily
* hit the bottom of the stack or run out of reserve
* list of vmem_seg structs. So, we must donate
* this hblk to reserve list if it's allocated
* from sfmmu8_cache *and* mapping kernel range.
* We don't need to worry about freeing hmeblk1's
* to kmem since they don't map any kmem slabs.
*
* Note: When segkmem supports largepages, we must
* free hmeblk1's to reserve list as well.
*/
goto re_verify;
}
} else {
/*
* Hey! we don't need hblk_reserve any more.
*/
owner = 0;
}
/*
* let's check if the goodies are still present
*/
/*
* return newhblkp if it's not hblk_reserve;
* if newhblkp is hblk_reserve, return it
* _only if_ we are the owner of hblk_reserve.
*/
return (newhblkp);
} else {
/*
* we just hit hblk_reserve in the hash and
* we are not the owner of that;
*
* block until hblk_reserve_thread completes
* swapping hblk_reserve and try the dance
* once again.
*/
goto fill_hblk;
}
} else {
/*
* it's no more! try the dance once again.
*/
goto fill_hblk;
}
}
hmeblkp->hblk_nextpa = 0;
return (hmeblkp);
}
/*
* This function performs any cleanup required on the hme_blk
* and returns it to the free list.
*/
/* ARGSUSED */
static void
{
int size;
if (shw_hblkp) {
if (mmu_page_sizes == max_mmu_page_sizes) {
} else {
}
/*
* Atomically clear shadow mask bit
*/
do {
} while (newshw_mask != shw_mask);
}
hmeblkp->hblk_shw_bit = 0;
if (hmeblkp->hblk_nuc_bit == 0) {
return;
}
}
static void
{
}
}
#define BUCKETS_TO_SEARCH_BEFORE_UNLOAD 30
static uint_t sfmmu_hblk_steal_twice;
/*
* Steal a hmeblk
* Enough hmeblks were allocated at startup (nucleus hmeblks) and also
* hmeblks were added dynamically. We should never ever not be able to
*/
static struct hme_blk *
sfmmu_hblk_steal(int size)
{
struct hmehash_bucket *hmebp;
int i;
for (;;) {
BUCKETS_TO_SEARCH_BEFORE_UNLOAD; i++) {
prevpa = 0;
while (hmeblkp) {
/*
* check if it is a hmeblk that is not locked
* and not shared. skip shadow hmeblks with
* shadow_mask set i.e valid count non zero.
*/
(hmeblkp->hblk_shw_bit == 0 ||
(hmeblkp->hblk_lckcnt == 0)) {
/*
* there is a high probability that we
* will find a free one. search some
* buckets for a free hmeblk initially
* before unloading a valid hmeblk.
*/
hmeblkp->hblk_hmecnt == 0) || (i >=
pr_hblk)) {
/*
* Hblk is unloaded
* successfully
*/
break;
}
}
}
}
}
break;
/*
* in the worst case, look for a free one in the kernel
* hash table.
*/
prevpa = 0;
while (hmeblkp) {
/*
* check if it is free hmeblk
*/
(hmeblkp->hblk_lckcnt == 0) &&
(hmeblkp->hblk_hmecnt == 0)) {
break;
} else {
/*
* Cannot fail since we have
* hash lock.
*/
panic("fail to steal?");
}
}
}
}
break;
}
return (hmeblkp);
}
/*
* This routine does real work to prepare a hblk to be "stolen" by
* unloading the mappings, updating shadow counts ....
* It returns 1 if the block is ready to be reused (stolen), or 0
* means the block cannot be stolen yet- pageunload is still working
* on this hblk.
*/
static int
{
/*
* check if the hmeblk is free, unload if necessary
*/
/*
* Pageunload is working on the same hblk.
*/
return (0);
}
}
if (shw_hblkp) {
/*
* Atomically clear shadow mask bit
*/
do {
} while (newshw_mask != shw_mask);
}
/*
* remove shadow bit if we are stealing an unused shadow hmeblk.
* sfmmu_hblk_alloc needs it that way, will set shadow bit later if
* we are indeed allocating a shadow hmeblk.
*/
hmeblkp->hblk_shw_bit = 0;
return (1);
}
struct hme_blk *
{
struct hme_blk *hblk_dummy = 0;
/*
* No dummy sf_hments, please.
*/
return (hmeblkp);
}
/*
* On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag.
* If we can't get appropriately sized TSB(s), try for 8K TSB(s) using
* KM_SLEEP allocation.
*
* Return 0 on success, -1 otherwise.
*/
static void
{
}
} else {
return;
}
/*
* Loop over all tsbinfo's replacing them with ones that actually have
* a TSB. If any of the replacements ever fail, bail out of the loop.
*/
if (rc != TSB_SUCCESS) {
break;
}
}
switch (rc) {
case TSB_SUCCESS:
return;
case TSB_ALLOCFAIL:
break;
default:
panic("sfmmu_replace_tsb returned unrecognized failure code "
"%d", rc);
}
/*
* In this case, we failed to get one of our TSBs. If we failed to
* get the first TSB, get one of minimum size (8KB). Walk the list
* and throw away the tsbinfos, starting where the allocation failed;
* we can get by with just one TSB as long as we don't leave the
* SWAPPED tsbinfo structures lying around.
*/
}
/*
* If we don't have any TSBs, get a single 8K TSB for 8K, 64K and 512K
* pages.
*/
if (!gotfirst) {
}
}
/*
* Handle exceptions for low level tsb_handler.
*
* There are many scenarios that could land us here:
*
* If the context is invalid we land here. The context can be invalid
* for 3 reasons: 1) we couldn't allocate a new context and now need to
* perform a wrap around operation in order to allocate a new context.
* 2) Context was invalidated to change pagesize programming 3) ISMs or
* TSBs configuration is changeing for this process and we are forced into
* here to do a syncronization operation. If the context is valid we can
* be here from window trap hanlder. In this case just call trap to handle
* the fault.
*
* Note that the process will run in INVALID_CONTEXT before
* faulting into here and subsequently loading the MMU registers
* (including the TSB base register) associated with this process.
* For this reason, the trap handlers must all test for
* INVALID_CONTEXT before attempting to access any registers other
* than the context registers.
*/
void
{
char lwp_save_state;
/*
* First, make sure we come out of here with a valid ctx,
* since if we don't get one we'll simply loop on the
* faulting instruction.
*
* If the ISM mappings are changing, the TSB is being relocated, or
* the process is swapped out we serialize behind the controlling
* thread with the sfmmu_flags and sfmmu_tsb_cv condition variable.
* Otherwise we synchronize with the context stealer or the thread
* that required us to change out our MMU registers (such
* as a thread changing out our TSB while we were running) by
* locking the HAT and grabbing the rwlock on the context as a
* reader temporarily.
*/
ctxnum == INVALID_CONTEXT);
if (ctxnum == INVALID_CONTEXT) {
/*
* Must set lwp state to LWP_SYS before
* trying to acquire any adaptive lock
*/
goto retry;
}
}
/*
* Wait for ISM maps to be updated.
*/
goto retry;
}
/*
* If we're swapping in, get TSB(s). Note that we must do
* this before we get a ctx or load the MMU state. Once
* we swap in we have to recheck to make sure the TSB(s) and
* ISM mappings didn't change while we slept.
*/
goto retry;
}
/*
* Must restore lwp_state if not calling
* trap() for further processing. Restore
* it anyway.
*/
return;
}
if (traptype == T_DATA_PROT) {
}
}
}
/*
* sfmmu_vatopfn_suspended is called from GET_TTE when TL=0 and
* TTE_SUSPENDED bit set in tte we block on aquiring a page lock
* rather than spinning to avoid send mondo timeouts with
* interrupts enabled. When the lock is acquired it is immediately
* released and we return back to sfmmu_vatopfn just after
* the GET_TTE call.
*/
void
{
}
/*
* sfmmu_tsbmiss_suspended is called from GET_TTE when TL>0 and
* TTE_SUSPENDED bit set in tte. We do this so that we can handle
* cross traps which cannot be handled while spinning in the
* trap handlers. Simply enter and exit the kpr_suspendlock spin
* mutex, which is held by the holder of the suspend bit, and then
* retry the trapped instruction after unwinding.
*/
/*ARGSUSED*/
void
{
}
/*
* Special routine to flush out ism mappings- TSBs, TLBs and D-caches.
* This routine may be called with all cpu's captured. Therefore, the
* caller is responsible for holding all locks and disabling kernel
* preemption.
*/
/* ARGSUSED */
static void
{
#ifdef VAC
int vcolor;
#endif
int ttesz;
/*
* Walk the ism_hat's mapping list and flush the page
* from every hat sharing this ism_hat. This routine
* may be called while all cpu's have been captured.
* Therefore we can't attempt to grab any locks. For now
* this means we will protect the ism mapping list under
* a single lock which will be grabbed by the caller.
* problem then we may need to re-think ism mapping list locking.
*/
/*
* Flush TSB of ISM mappings.
*/
} else {
}
#ifdef VAC
/*
* Flush D$
* When flushing D$ we must flush all
* cpu's. See sfmmu_cache_flush().
*/
if (cache_flush_flag == CACHE_FLUSH) {
}
#endif /* VAC */
}
}
/*
* Demaps the TSB, CPU caches, and flushes all TLBs on all CPUs of
* a particular virtual address and ctx. If noflush is set we do not
* HAT lock held.
*/
static void
int hat_lock_held)
{
#ifdef VAC
int vcolor;
#endif
#endif
/*
* There is no longer a need to protect against ctx being
* stolen here since we don't store the ctx in the TSB anymore.
*/
#ifdef VAC
#endif
/*
* We must hold the hat lock during the flush of TLB,
* to avoid a race with sfmmu_invalidate_ctx(), where
* sfmmu_cnum on a MMU could be set to INVALID_CONTEXT,
* causing TLB demap routine to skip flush on that MMU.
* If the context on a MMU has already been set to
* INVALID_CONTEXT, we just get an extra flush on
* that MMU.
*/
if (!hat_lock_held && !tlb_noflush)
if (!tlb_noflush) {
/*
* Flush the TSB and TLB.
*/
}
if (!hat_lock_held && !tlb_noflush)
#ifdef VAC
/*
* Flush the D$
*
* Even if the ctx is stolen, we need to flush the
* cache. Our ctx stealer only flushes the TLBs.
*/
if (cache_flush_flag == CACHE_FLUSH) {
if (cpu_flag & FLUSH_ALL_CPUS) {
} else {
}
}
#endif /* VAC */
}
/*
* Demaps the TSB and flushes all TLBs on all cpus for a particular virtual
* address and ctx. If noflush is set we do not currently do anything.
* This function may or may not be called with the HAT lock held.
*/
static void
int tlb_noflush, int hat_lock_held)
{
/*
* If the process is exiting we have nothing to do.
*/
if (tlb_noflush)
return;
/*
* Flush TSB.
*/
if (!hat_lock_held)
if (!hat_lock_held)
}
/*
* Special case of sfmmu_tlb_demap for MMU_PAGESIZE hblks. Use the xcall
* call handler that can flush a range of pages to save on xcalls.
*/
static int sfmmu_xcall_save;
static void
{
int pgunload = 0;
int dirtypg = 0;
/*
* Flush TSB and calculate number of pages to flush.
*/
while (bitvec != 0) {
dirtypg = 0;
/*
* Find the first page to flush and then count how many
* pages there are after it that also need to be flushed.
* This way the number of TSB flushes is minimized.
*/
while ((bitvec & 1) == 0) {
pgcnt++;
addr += MMU_PAGESIZE;
bitvec >>= 1;
}
while (bitvec & 1) {
dirtypg++;
bitvec >>= 1;
}
}
if (sfmmup->sfmmu_free == 0) {
/*
* make sure it has SFMMU_PGCNT_SHIFT bits only,
* as it will be used to pack argument for xt_some
*/
/*
* Encode pgcnt as (pgcnt -1 ), and pass (pgcnt - 1) in
* the low 6 bits of sfmmup. This is doable since pgcnt
* always >= 1.
*/
/*
* We must hold the hat lock during the flush of TLB,
* to avoid a race with sfmmu_invalidate_ctx(), where
* sfmmu_cnum on a MMU could be set to INVALID_CONTEXT,
* causing TLB demap routine to skip flush on that MMU.
* If the context on a MMU has already been set to
* INVALID_CONTEXT, we just get an extra flush on
* that MMU.
*/
if (bitvec & 1)
addr += MMU_PAGESIZE;
}
}
dmrp->dmr_bitvec = 0;
}
/*
* handlers, _and_ need to flush the TLB, it's a lot easier to
* throw away the context from the process than to do a
* special song and dance to keep things consistent for the
* handlers.
*
* Since the process suddenly ends up without a context and our caller
* holds the hat lock, threads that fault after this function is called
* will pile up on the lock. We can then do whatever we need to
* atomically from the context of the caller. The first blocked thread
* to resume executing will get the process a new context, and the
* process will resume executing.
*
* One added advantage of this approach is that on MMUs that
* support a "flush all" operation, we will delay the flush until
* cnum wrap-around, and then flush the TLB one time. This
* is rather rare, so it's a lot less expensive than making 8000
* x-calls to flush the TLB 8000 times.
*
* A per-process (PP) lock is used to synchronize ctx allocations in
* resume() and ctx invalidations here.
*/
static void
{
int i;
/* set HAT cnum invalid across all context domains. */
for (i = 0; i < max_mmu_ctxdoms; i++) {
if (cnum == INVALID_CONTEXT) {
continue;
}
}
membar_enter(); /* make sure globally visible to all CPUs */
if (!CPUSET_ISNULL(cpuset)) {
}
/*
* If the hat to-be-invalidated is the same as the current
* process on local CPU we need to invalidate
* this CPU context as well.
*/
if ((sfmmu_getctx_sec() == currcnum) &&
(currcnum != INVALID_CONTEXT)) {
}
/*
* we hold the hat lock, so nobody should allocate a context
* for us yet
*/
}
#ifdef VAC
/*
* We need to flush the cache in all cpus. It is possible that
* a process referenced a page as cacheable but has sinced exited
* and cleared the mapping list. We still to flush it but have no
* state so all cpus is the only alternative.
*/
void
{
}
void
{
}
#endif /* VAC */
/*
* We need to prevent processes from accessing the TSB using a cached physical
* address. It's alright if they try to access the TSB via virtual address
* since they will just fault on that virtual address once the mapping has
* been suspended.
*/
#pragma weak sendmondo_in_recover
/* ARGSUSED */
static int
{
extern uint32_t sendmondo_in_recover;
if (flags != HAT_PRESUSPEND)
return (0);
/*
* For Cheetah+ Erratum 25:
* Wait for any active recovery to finish. We can't risk
* relocating the TSB of the thread running mondo_recover_proc()
* since, if we did that, we would deadlock. The scenario we are
* trying to avoid is as follows:
*
* THIS CPU RECOVER CPU
* -------- -----------
* Begins recovery, walking through TSB
* hat_pagesuspend() TSB TTE
* TLB miss on TSB TTE, spins at TL1
* xt_sync()
* send_mondo_timeout()
* mondo_recover_proc()
* ((deadlocked))
*
* The second half of the workaround is that mondo_recover_proc()
* checks to see if the tsb_info has the RELOC flag set, and if it
* does, it skips over that TSB without ever touching tsbinfop->tsb_va
* and hence avoiding the TLB miss that could result in a deadlock.
*/
if (&sendmondo_in_recover) {
membar_enter(); /* make sure RELOC flag visible */
while (sendmondo_in_recover) {
drv_usecwait(1);
}
}
return (0);
}
/* ARGSUSED */
static int
{
if (flags != HAT_POSTUNSUSPEND)
return (0);
/*
* The process may have swapped out while we were relocating one
* of its TSBs. If so, don't bother doing the setup since the
* process can't be using the memory anymore.
*/
}
}
membar_exit();
return (0);
}
/*
* Allocate and initialize a tsb_info structure. Note that we may or may not
* allocate a TSB here, depending on the flags passed in.
*/
static int
{
int err;
return (err);
}
/*
* Bump the TSB size counters for this TSB size.
*/
(*(((int *)&sfmmu_tsbsize_stat) + tsb_szc))++;
return (0);
}
static void
{
/*
* If we allocated this TSB from relocatable kernel memory, then we
* need to uninstall the callback handler.
*/
int ret;
0, NULL);
}
if (kmem_cachep != NULL) {
} else {
}
}
static void
{
}
}
/*
* Setup all the references to physical memory for this tsbinfo.
* The underlying page(s) must be locked.
*/
static void
{
#ifndef sun4v
} else {
/*
* Round down PA and use a large mapping; the handlers will
* compute the TSB pointer at the correct offset into the
* big virtual page. NOTE: this assumes all TSBs larger
* than 8K must come from physically contiguous slabs of
* size tsb_slab_size.
*/
}
#else /* sun4v */
#endif /* sun4v */
}
/*
* Returns zero on success, ENOMEM if over the high water mark,
* or EAGAIN if the caller needs to retry with a smaller TSB
* size (or specify TSB_FORCEALLOC if the allocation can't fail).
*
* This call cannot fail to allocate a TSB if TSB_FORCEALLOC
* is specified and the TSB requested is PAGESIZE, though it
* may sleep waiting for memory if sufficient memory is not
* available.
*/
static int
{
int lowmem = 0;
int ret;
/*
* If not allocating a TSB, set up the tsbinfo, set TSB_SWAPPED, and
* return.
*/
return (0);
}
#ifdef DEBUG
/*
* For debugging:
* Randomly force allocation failures every tsb_alloc_mtbf
* tries if TSB_FORCEALLOC is not specified. This will
* return ENOMEM if tsb_alloc_mtbf is odd, or EAGAIN if
* it is even, to allow testing of both failure paths...
*/
(tsb_alloc_count++ == tsb_alloc_mtbf)) {
tsb_alloc_count = 0;
}
#endif /* DEBUG */
/*
* Enforce high water mark if we are not doing a forced allocation
* and are not shrinking a process' TSB.
*/
if ((flags & TSB_SHRINK) == 0 &&
if ((flags & TSB_FORCEALLOC) == 0)
return (ENOMEM);
lowmem = 1;
}
/*
* Allocate from the correct location based upon the size of the TSB
* compared to the base page size, and what memory conditions dictate.
* Note we always do nonblocking allocations from the TSB arena since
* we don't want memory fragmentation to cause processes to block
* indefinitely waiting for memory; until the kernel algorithms that
* coalesce large pages are improved this is our best option.
*
* Algorithm:
* If allocating a "large" TSB (>8K), allocate from the
* appropriate kmem_tsb_default_arena vmem arena
* else if low on memory or the TSB_FORCEALLOC flag is set or
* tsb_forceheap is set
* Allocate from kernel heap via sfmmu_tsb8k_cache with
* KM_SLEEP (never fails)
* else
* Allocate from appropriate sfmmu_tsb_cache with
* KM_NOSLEEP
* endif
*/
if (tsb_lgrp_affinity)
lgrpid = 0; /* use lgrp of boot CPU */
if (tsbbytes > MMU_PAGESIZE) {
#ifdef DEBUG
#else /* !DEBUG */
#endif /* DEBUG */
} else {
}
return (EAGAIN);
}
/*
* If we are allocating from outside the cage, then we need to
* register a relocation callback handler. Note that for now
* since pseudo mappings always hang off of the slab's root page,
* we need only lock the first 8K of the TSB slab. This is a bit
* hacky but it is good for performance.
*/
if (kmem_cachep != sfmmu_tsb8k_cache) {
/*
* Need to free up resources if we could not successfully
* add the callback function and return an error condition.
*/
if (ret != 0) {
if (kmem_cachep) {
} else {
}
S_WRITE);
return (EAGAIN);
}
} else {
/*
* Since allocation of 8K TSBs from heap is rare and occurs
* during memory pressure we allocate them from permanent
* memory rather than using callbacks to get the PFN.
*/
}
if (kmem_cachep != sfmmu_tsb8k_cache) {
}
return (0);
}
/*
* Initialize per cpu tsb and per cpu tsbmiss_area
*/
void
sfmmu_init_tsbs(void)
{
int i;
#ifndef sun4v
extern int dcache_line_mask;
#endif /* sun4v */
extern uint_t vac_colors;
/*
* Init. tsb miss area.
*/
/*
* initialize the tsbmiss area.
* Do this for all possible CPUs as some may be added
* while the system is running. There is no cost to this.
*/
#ifndef sun4v
#endif /* sun4v */
}
if (kpm_enable == 0)
return;
/* -- Begin KPM specific init -- */
if (kpm_smallpages) {
/*
* If we're using base pagesize pages for seg_kpm
* mappings, we use the kernel TSB since we can't afford
* to allocate a second huge TSB for these mappings.
*/
} else {
/*
* In VAC conflict case, just put the entries in the
* kernel 8K indexed TSB for now so we can find them.
* This could really be changed in the future if we feel
* the need...
*/
}
/*
* Initialize the kpmtsbm area.
* Do this for all possible CPUs as some may be added
* while the system is running. There is no cost to this.
*/
if (kpm_smallpages == 0) {
} else {
}
#ifdef DEBUG
#endif /* DEBUG */
if (ktsb_phys)
}
/* -- End KPM specific init -- */
}
/* Avoid using sfmmu_tsbinfo_alloc() to avoid kmem_alloc - no real reason */
/*
* Called from hat_kern_setup() to setup the tsb_info for ksfmmup.
*/
void
{
/*
* Allocate tsbinfos for kernel and copy in data
* to make debug easier and sun4v setup easier.
*/
/* Link them into ksfmmup. */
}
/*
* Cache the last value returned from va_to_pa(). If the VA specified
* in the current call to cached_va_to_pa() maps to the same Page (as the
* previous call to cached_va_to_pa()), then compute the PA using
* cached info, else call va_to_pa().
*
* Note: this function is neither MT-safe nor consistent in the presence
* of multiple, interleaved threads. This function was created to enable
* an optimization used during boot (at a point when there's only one thread
* executing on the "boot CPU", and before startup_vm() has been called).
*/
static uint64_t
cached_va_to_pa(void *vaddr)
{
static uint64_t prev_vaddr_base = 0;
} else {
/*
* Computed physical address is valid. Cache its
* related info for the next cached_va_to_pa() call.
*/
}
return (pa);
}
}
/*
* Carve up our nucleus hblk region. We may allocate more hblks than
* asked due to rounding errors but we are guaranteed to have at least
* enough space to allocate the requested number of hblk8's and hblk1's.
*/
void
{
size_t i;
ulong_t j = 0, k = 0;
/* Need to use proper structure alignment */
nucleus_hblk8.index = 0;
/*
* Use as much memory as possible for hblk8's since we
* expect all bop_alloc'ed memory to be allocated in 8k chunks.
* We need to hold back enough space for the hblk1's which
* we'll allocate next.
*/
for (i = 0; i <= hblk8_bound; i += hme8blk_sz, j++) {
addr += hme8blk_sz;
}
nucleus_hblk8.len = j;
nucleus_hblk1.index = 0;
addr += hme1blk_sz;
}
nucleus_hblk1.len = k;
}
/*
* This function is currently not supported on this platform. For what
* it's supposed to do, see hat.c and hat_srmmu.c
*/
/* ARGSUSED */
{
return (FC_NOSUPPORT);
}
/*
* Searchs the mapping list of the page for a mapping of the same size. If not
* found the corresponding bit is cleared in the p_index field. When large
* pages are more prevalent in the system, we can maintain the mapping list
* in order and we don't have to traverse the list each time. Just check the
* next and prev entries, and if both are of different size, we clear the bit.
*/
static void
{
int index;
/*
* Traverse mapping list looking for another mapping of same size.
* since we only want to clear index field if all mappings of
* that size are gone.
*/
if (hmeblkp->hblk_xhat_bit)
continue;
/*
* another mapping of the same size. don't clear index.
*/
return;
}
}
/*
* Clear the p_index bit for large page.
*/
while (npgs-- > 0) {
}
}
/*
* return supported features
*/
/* ARGSUSED */
int
{
switch (feature) {
case HAT_SHARED_PT:
case HAT_DYNAMIC_ISM_UNMAP:
case HAT_VMODSORT:
return (1);
default:
return (0);
}
}
void
{
}
}
void
{
}
}
/*ARGSUSED*/
void
{
}
static void
hat_kstat_init(void)
{
KSTAT_TYPE_RAW, sizeof (struct sfmmu_global_stat),
if (ksp) {
}
KSTAT_TYPE_RAW, sizeof (struct sfmmu_tsbsize_stat),
if (ksp) {
}
if (ksp) {
}
}
/* ARGSUSED */
static int
{
int i;
if (rw == KSTAT_READ) {
} else {
cpu_kstat->sf_tsb_hits = 0;
}
}
} else {
/* KSTAT_WRITE is used to clear stats */
tsbm->itlb_misses = 0;
tsbm->dtlb_misses = 0;
tsbm->utsb_misses = 0;
tsbm->ktsb_misses = 0;
tsbm->uprot_traps = 0;
tsbm->kprot_traps = 0;
kpmtsbm->kpm_dtlb_misses = 0;
kpmtsbm->kpm_tsb_misses = 0;
}
}
return (0);
}
#ifdef DEBUG
/*
* A tte checker. *orig_old is the value we read before cas.
* *cur is the value returned by cas.
* *new is the desired value when we do the cas.
*
* *hmeblkp is currently unused.
*/
/* ARGSUSED */
void
{
pfn_t i, j, k;
#ifdef lint
#endif
if (TTE_IS_VALID(orig_old)) {
if (TTE_IS_VALID(cur)) {
i = TTE_TO_TTEPFN(orig_old);
j = TTE_TO_TTEPFN(cur);
k = TTE_TO_TTEPFN(new);
if (i != j) {
/* remap error? */
panic("chk_tte: bad pfn, 0x%lx, 0x%lx", i, j);
}
if (i != k) {
/* remap error? */
panic("chk_tte: bad pfn2, 0x%lx, 0x%lx", i, k);
}
} else {
if (TTE_IS_VALID(new)) {
panic("chk_tte: invalid cur? ");
}
i = TTE_TO_TTEPFN(orig_old);
k = TTE_TO_TTEPFN(new);
if (i != k) {
panic("chk_tte: bad pfn3, 0x%lx, 0x%lx", i, k);
}
}
} else {
if (TTE_IS_VALID(cur)) {
j = TTE_TO_TTEPFN(cur);
if (TTE_IS_VALID(new)) {
k = TTE_TO_TTEPFN(new);
if (j != k) {
panic("chk_tte: bad pfn4, 0x%lx, 0x%lx",
j, k);
}
} else {
panic("chk_tte: why here?");
}
} else {
if (!TTE_IS_VALID(new)) {
panic("chk_tte: why here2 ?");
}
}
}
}
#endif /* DEBUG */
extern void prefetch_tsbe_read(struct tsbe *);
extern void prefetch_tsbe_write(struct tsbe *);
/*
* We want to prefetch 7 cache lines ahead for our read prefetch. This gives
* us optimal performance on Cheetah+. You can only have 8 outstanding
* prefetches at any one time, so we opted for 7 read prefetches and 1 write
* prefetch to make the most utilization of the prefetch capability.
*/
#define TSBE_PREFETCH_STRIDE (7)
void
{
int new_offset;
int i;
int vpshift;
int last_prefetch;
} else {
/*
* A TSBE is 16 bytes which means there are four TSBE's per
* P$ line (64 bytes), thus every 4 TSBE's we prefetch.
*/
for (i = 0; i < old_entries; i++, old++) {
/*
* We have a valid TTE to remap. Check the
* size. We won't remap 64K or 512K TTEs
* because they span more than one TSB entry
* and are indexed using an 8K virt. page.
* Ditto for 32M and 256M TTEs.
*/
continue;
if (mmu_page_sizes == max_mmu_page_sizes) {
continue;
}
/* clear the lower 22 bits of the va */
/* turn va into a virtual pfn */
/*
* or in bits from the offset in the tsb
* to get the real virtual pfn. These
* correspond to bits [21:13] in the va
*/
vpshift =
0x1ff;
}
}
}
}
/*
* unused in sfmmu
*/
void
hat_dump(void)
{
}
/*
* Called when a thread is exiting and we have switched to the kernel address
* space. Perform the same VM initialization resume() uses when switching
* processes.
*
* Note that sfmmu_load_mmustate() is currently a no-op for kernel threads, but
* we call it anyway in case the semantics change in the future.
*/
/*ARGSUSED*/
void
{
#ifdef sun4u
#endif
/*
* Note that sfmmu_load_mmustate() is currently a no-op for
* kernel threads. We need to disable interrupts here,
* simply because otherwise sfmmu_load_mmustate() would panic
* if the caller does not disable interrupts.
*/
}