sfmmu.c revision 125be069be21bcb7b94bd4525c19eb424bc56276
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <vm/hat_sfmmu.h>
#include <sys/sysmacros.h>
#include <sys/machparam.h>
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <vm/seg_kmem.h>
#include <sys/vm_machparam.h>
#include <sys/prom_isa.h>
#include <sys/prom_plat.h>
#include <sys/prom_debug.h>
#include <sys/privregs.h>
#include <sys/bootconf.h>
#include <sys/memlist_plat.h>
#include <sys/cpu_module.h>
/*
* Static routines
*/
static void sfmmu_reloc_trap_handler(void *, void *, size_t);
/*
* External routines
*/
extern void sfmmu_remap_kernel(void);
extern void sfmmu_patch_utsb(void);
/*
* Global Data:
*/
extern int enable_bigktsb;
extern int kmem64_smchunks;
int sfmmu_kern_mapped = 0;
/*
* DMMU primary context register for the kernel context. Machine specific code
* inserts correct page size codes when necessary
*/
#ifdef DEBUG
static int ndata_middle_hole_detected = 0;
#endif
/* Extern Global Data */
extern int page_relocate_ready;
/*
* Controls the logic which enables the use of the
* QUAD_LDD_PHYS ASI for TSB accesses.
*/
extern int ktsb_phys;
/*
* Global Routines called from within:
*/
{
if (tba_taken_over)
#if !defined(C_OBP)
if (!kmem64_smchunks &&
prom_panic("va_to_pfn: kmem64_pabase not init");
}
#endif /* !C_OBP */
(valid == -1)) {
}
return (PFN_INVALID);
}
{
return ((uint64_t)-1);
}
void
hat_kern_setup(void)
{
struct translation *trans_root;
extern void startup_fixup_physavail(void);
/*
* These are the steps we take to take over the mmu from the prom.
*
* (1) Read the prom's mappings through the translation property.
* (2) Remap the kernel text and kernel data with 2 locked 4MB ttes.
* Create the the hmeblks for these 2 ttes at this time.
* (3) Create hat structures for all other prom mappings. Since the
* kernel text and data hme_blks have already been created we
* skip the equivalent prom's mappings.
* (4) Initialize the tsb and its corresponding hardware regs.
* (5) Take over the trap table (currently in startup).
* (6) Up to this point it is possible the prom required some of its
* locked tte's. Now that we own the trap table we remove them.
*/
if (kpm_enable) {
if (kpm_smallpages == 0) {
}
}
if (!shctx_on) {
}
if (&mmu_enable_pgsz_search) {
}
/*
* The 8K-indexed kernel TSB space is used to hold
* translations below...
*/
/*
* We invalidate 8K kernel TSB because we used it in
* sfmmu_map_prom_mappings()
*/
sfmmu_kern_mapped = 1;
/*
* hments have been created for mapped pages, and thus we're ready
* for kmdb to start using its own trap table. It walks the hments
* to resolve TLB misses, and can't be used until they're ready.
*/
}
/*
* Macro used below to convert the prom's 32-bit high and low fields into
* a value appropriate for the 64-bit kernel.
*/
/*
* Track larges pages used.
* Provides observability for this feature on non-debug kernels.
*/
/*
* This function traverses the prom mapping list and creates equivalent
* mappings in the sfmmu mapping hash.
*/
static void
{
struct translation *promt;
unsigned long i;
extern struct memlist *virt_avail;
char buf[256];
/*
* hack until we get rid of map-for-unix
*/
continue;
#if defined(TTE_IS_GLOBAL)
if (TTE_IS_GLOBAL(ttep)) {
/*
* The prom better not use global translations
* because a user process might use the same
* virtual addresses
*/
prom_panic("sfmmu_map_prom_mappings: global"
" translation");
}
#endif
if (TTE_IS_LOCKED(ttep)) {
/* clear the lock bits */
}
offset = 0;
while (size) {
/*
* make sure address is not in virt-avail list
*/
size)) {
prom_panic("sfmmu_map_prom_mappings:"
" inconsistent translation/avail lists");
}
if (pf_is_memory(pfn)) {
if (attr & SFMMU_UNCACHEPTTE) {
prom_panic("sfmmu_map_prom_mappings:"
" uncached prom memory page");
}
} else {
if (!(attr & SFMMU_SIDEFFECT)) {
prom_panic("sfmmu_map_prom_mappings:"
" prom i/o page without"
" side-effect");
}
}
/*
* skip kmem64 area
*/
if (!kmem64_smchunks &&
vaddr >= kmem64_base &&
vaddr < kmem64_aligned_end) {
#if !defined(C_OBP)
prom_panic("sfmmu_map_prom_mappings:"
" unexpected kmem64 prom mapping");
#else /* !C_OBP */
prom_panic("sfmmu_map_prom_mappings:"
" unexpected kmem64 prom mapping");
}
break;
}
continue;
#endif /* !C_OBP */
}
ASSERT(page_relocate_ready == 0);
if (oldpfn != PFN_INVALID) {
/*
* mapping already exists.
* Verify they are equal
*/
"sfmmu_map_prom_mappings: mapping"
" conflict (va = 0x%p, pfn = 0x%p,"
" oldpfn = 0x%p)", (void *)vaddr,
}
size -= MMU_PAGESIZE;
offset += MMU_PAGESIZE;
continue;
}
"sfmmu_map_prom_mappings: prom-mapped"
" page (va = 0x%p, pfn = 0x%p) on free list",
}
size -= MMU_PAGESIZE;
offset += MMU_PAGESIZE;
}
}
/*
* We claimed kmem64 from prom, so now we need to load tte.
*/
int pszc;
pszc = kmem64_szc;
#ifdef sun4u
}
#endif /* sun4u */
vaddr = kmem64_base;
while (vaddr < kmem64_end) {
}
}
}
/*
* This routine reads in the "translations" property in to a buffer and
* returns a pointer to this buffer and the number of translations.
*/
static struct translation *
{
char *prop = "translations";
struct translation *transroot;
/*
* the "translations" property is associated with the mmu node
*/
/*
* We use the TSB space to read in the prom mappings. This space
* is currently not being used because we haven't taken over the
* trap table yet. It should be big enough to hold the mappings.
*/
}
return (transroot);
}
/*
* Init routine of the nucleus data memory allocator.
*
* The nucleus data memory allocator is organized in ecache_alignsize'd
* memory chunks. Memory allocated by ndata_alloc() will never be freed.
*
* The ndata argument is used as header of the ndata freelist.
* Other freelist nodes are placed in the nucleus memory itself
* at the beginning of a free memory chunk. Therefore a freelist
* node (struct memlist) must fit into the smallest allocatable
* memory chunk (ecache_alignsize bytes).
*
* The memory interval [base, end] passed to ndata_alloc_init() must be
* bzero'd to allow the allocator to return bzero'd memory easily.
*/
void
{
}
/*
* Deliver the size of the largest free memory chunk.
*/
{
}
return (chunksize);
}
/*
* Allocate the last properly aligned memory chunk.
* This function is called when no more large nucleus memory chunks
* will be allocated. The remaining free nucleus memory at the end
* of the nucleus can be added to the phys_avail list.
*/
void *
{
#ifdef DEBUG
static int called = 0;
if (called++ > 0)
#endif /* DEBUG */
/*
* The alignment needs to be a multiple of ecache_alignsize.
*/
}
return (NULL);
#ifdef DEBUG
#endif
return (NULL);
}
else
} else {
}
return ((void *)base);
}
/*
* Select the best matching buffer, avoid memory fragmentation.
*/
static struct memlist *
{
/*
* Look for the best matching buffer, avoid memory fragmentation.
* The following strategy is used, try to find
* 1. an exact fitting buffer
* 2. avoid wasting any space below the buffer, take first
* fitting buffer
* 3. avoid wasting any space above the buffer, take first
* fitting buffer
* 4. avoid wasting space, take first fitting buffer
* 5. take the last buffer in chain
*/
continue;
if (unused == 0)
return (frlist);
break;
if (below < best_below) {
best_below = below;
}
if (above < best_above) {
best_above = above;
}
if (unused < best_unused) {
fnd_unused = frlist;
}
}
if (best_below == 0)
return (fnd_below);
if (best_above == 0)
return (fnd_above);
if (best_unused < ULONG_MAX)
return (fnd_unused);
return (frlist);
}
/*
* Nucleus data memory allocator.
* The granularity of the allocator is ecache_alignsize.
* See also comment for ndata_alloc_init().
*/
void *
{
/*
* Look for the best matching buffer, avoid memory fragmentation.
*/
return (NULL);
/*
* Allocate the nucleus data buffer.
*/
if (below >= ecache_alignsize) {
/*
* There is free memory below the allocated memory chunk.
*/
if (above) {
}
return ((void *)base);
}
/*
* The first chunk (ndata) is selected.
*/
if (above) {
} else {
}
return ((void *)base);
}
/*
* Not the first chunk.
*/
if (above) {
} else {
}
return ((void *)base);
}
/*
* Size the kernel TSBs based upon the amount of physical
* memory in the system.
*/
static void
{
if (npages <= TSB_FREEMEM_MIN) {
enable_bigktsb = 0;
enable_bigktsb = 0;
} else if (npages <= TSB_FREEMEM_LARGE) {
enable_bigktsb = 0;
enable_bigktsb == 0) {
enable_bigktsb = 0;
} else {
}
/*
* We choose the TSB to hold kernel 4M mappings to have twice
* the reach as the primary kernel TSB since this TSB will
* potentially (currently) be shared by both mappings to all of
* physical memory plus user TSBs. If this TSB has to be in nucleus
* (only for Spitfire and Cheetah) limit its size to 64K.
*/
}
}
/*
* Allocate kernel TSBs from nucleus data memory.
* The function return 0 on success and -1 on failure.
*/
int
{
/*
* Set ktsb_phys to 1 if the processor supports ASI_QUAD_LDD_PHYS.
*/
/*
* Size the kernel TSBs based upon the amount of physical
* memory in the system.
*/
/*
* Allocate the 8K kernel TSB if it belongs inside the nucleus.
*/
if (enable_bigktsb == 0) {
return (-1);
}
/*
* Next, allocate 4M kernel TSB from the nucleus since it's small.
*/
if (ktsb4m_szcode <= TSB_64K_SZCODE) {
if (ktsb4m_base == NULL)
return (-1);
}
return (0);
}
{
/*
* The number of buckets in the hme hash tables
* is a power of 2 such that the average hash chain length is
* HMENT_HASHAVELEN. The number of buckets for the user hash is
* a function of physical memory and a predefined overmapping factor.
* The number of buckets for the kernel hash is a function of
* physical memory only.
*/
if (uhmehash_num > USER_BUCKETS_THRESHOLD) {
/*
* if uhmehash_num is not power of 2 round it down to the
* next power of 2.
*/
} else
}
{
return (alloc_base);
}
/*
* Allocate hat structs from the nucleus data memory.
*/
int
{
/*
* For the page mapping list mutex array we allocate one mutex
* for every 128 pages (1 MB) with a minimum of 64 entries and
* a maximum of 8K entries. For the initial computation npages
* is rounded up (ie. 1 << highbit(npages * 1.5 / 128))
*
* mml_shift is roughly log2(mml_table_sz) + 3 for MLIST_HASH
*/
if (mml_table_sz < 64)
mml_table_sz = 64;
else if (mml_table_sz > 8192)
mml_table_sz = 8192;
return (-1);
if (sfmmu_cb_table == NULL)
return (-1);
return (0);
}
int
{
/*
* For the kpm_page mutex array we allocate one mutex every 16
* kpm pages (64MB). In smallpage mode we allocate one mutex
* every 8K pages. The minimum is set to 64 entries and the
* maximum to 8K entries.
*/
if (kpm_smallpages == 0) {
if (kpmp_table == NULL)
return (-1);
kpmp_stable_sz = 0;
kpmp_stable = NULL;
} else {
if (kpmp_stable == NULL)
return (-1);
kpmp_table_sz = 0;
kpmp_table = NULL;
}
return (0);
}
/*
* This function bop allocs kernel TSBs.
*/
{
if (enable_bigktsb) {
" 8K bigktsb");
}
if (ktsb4m_szcode > TSB_64K_SZCODE) {
if (vaddr != ktsb4m_base)
" 4M bigktsb");
ktsb4m_base = vaddr;
}
return (tsbbase);
}
/*
* Moves code assembled outside of the trap table into the trap
* table taking care to relocate relative branches to code outside
* of the trap handler.
*/
static void
{
size_t i;
int disp;
if (op == 1) {
/* call */
continue;
} else if (op == 0) {
/* branch or sethi */
switch (op2) {
case 0x3: /* BPr */
(inst & 0x3fff);
continue;
inst &= ~0x303fff;
break;
case 0x2: /* Bicc */
continue;
inst &= ~0x3fffff;
break;
case 0x1: /* Bpcc */
continue;
inst &= ~0x7ffff;
break;
}
}
}
}
/*
* Routine to allocate a large page to use in the TSB caches.
*/
/*ARGSUSED*/
static page_t *
{
int pgflags;
if ((vmflag & VM_NOSLEEP) == 0)
if (vmflag & VM_PUSHPAGE)
pgflags |= PG_PUSHPAGE;
}
/*
* Allocate a large page to back the virtual address range
* [addr, addr + size). If addr is NULL, allocate the virtual address
* space as well.
*/
static void *
void *pcarg)
{
int i = 0;
/*
* Assuming that only TSBs will call this with size > PAGESIZE
* There is no reason why this couldn't be expanded to 8k pages as
* well, or other page sizes in the future .... but for now, we
* only support fixed sized page requests.
*/
return (NULL);
return (NULL);
}
return (NULL);
}
}
/*
* Load the locked entry. It's OK to preload the entry into
* the TSB since we now support large mappings in the kernel TSB.
*/
for (--i; i >= 0; --i) {
page_unlock(ppa[i]);
}
return (addr);
}
/* Called to import new spans into the TSB vmem arenas */
void *
{
if (tsb_lgrp_affinity) {
/*
* Search for the vmp->lgrpid mapping by brute force;
* some day vmp will have an lgrp, until then we have
* to do this the hard way.
*/
;
if (lgrpid == NLGRPS_MAX)
}
}
/* Called to free spans from the TSB vmem arenas */
void
{
panic("sfmmu_tsb_segkmem_free: page not found");
if (--pgs_left == 0) {
/*
* similar logic to segspt_free_pages, but we know we
* have one large page.
*/
}
}
}