mach_vm_dep.c revision d2a70789f056fc6c9ce3ab047b52126d80b0e3da
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
/*
* Portions of this source code were derived from Berkeley 4.3 BSD
* under license from the Regents of the University of California.
*/
/*
* UNIX machine dependent virtual memory support.
*/
#include <sys/cpu_module.h>
#include <sys/elf_SPARC.h>
#include <sys/archsystm.h>
#include <vm/hat_sfmmu.h>
#include <sys/mem_cage.h>
#include <sys/machsystm.h>
#include <vm/seg_kmem.h>
uint_t page_colors = 0;
uint_t page_colors_mask = 0;
int consistent_coloring;
int update_proc_pgcolorbase_after_fork = 1;
/*
* A bitmask of the page sizes supported by hardware based upon szc.
* The base pagesize (p_szc == 0) must always be supported by the hardware.
*/
extern uint_t vac_colors_mask;
extern int vac_shift;
hw_pagesize_t hw_page_array[] = {
{0, 0, 0, 0}
};
/*
* Maximum page size used to map 64-bit memory segment kmem64_base..kmem64_end
*/
int max_bootlp_tteszc = TTE256M;
/*
* Maximum and default segment size tunables for user heap, stack, private
* and shared anonymous memory, and user text and initialized data.
*/
/*
* Contiguous memory allocator data structures and variables.
*
* The sun4v kernel must provide a means to allocate physically
* contiguous, non-relocatable memory. The contig_mem_arena
* and contig_mem_slab_arena exist for this purpose. Allocations
* that require physically contiguous non-relocatable memory should
* be made using contig_mem_alloc() or contig_mem_alloc_align()
* which return memory from contig_mem_arena or contig_mem_reloc_arena.
* These arenas import memory from the contig_mem_slab_arena one
* contiguous chunk at a time.
*
* When importing slabs, an attempt is made to allocate a large page
* to use as backing. As a result of the non-relocatable requirement,
* slabs are allocated from the kernel cage freelists. If the cage does
* not contain any free contiguous chunks large enough to satisfy the
* slab allocation, the slab size will be downsized and the operation
* retried. Large slab sizes are tried first to minimize cage
* fragmentation. If the slab allocation is unsuccessful still, the slab
* is allocated from outside the kernel cage. This is undesirable because,
* until slabs are freed, it results in non-relocatable chunks scattered
* throughout physical memory.
*
* Allocations from the contig_mem_arena are backed by slabs from the
* cage. Allocations from the contig_mem_reloc_arena are backed by
* slabs allocated outside the cage. Slabs are left share locked while
* in use to prevent non-cage slabs from being relocated.
*
* Since there is no guarantee that large pages will be available in
* the kernel cage, contiguous memory is reserved and added to the
* contig_mem_arena at boot time, making it available for later
* contiguous memory allocations. This reserve will be used to satisfy
* contig_mem allocations first and it is only when the reserve is
* completely allocated that new slabs will need to be imported.
*/
static vmem_t *contig_mem_slab_arena;
static vmem_t *contig_mem_arena;
static vmem_t *contig_mem_reloc_arena;
static kmutex_t contig_mem_lock;
#define CONTIG_MEM_ARENA_QUANTUM 64
/* contig_mem_arena import slab sizes, in decreasing size order */
static size_t contig_mem_import_sizes[] = {
};
#define NUM_IMPORT_SIZES \
(sizeof (contig_mem_import_sizes) / sizeof (size_t))
/* Boot-time allocated buffer to pre-populate the contig_mem_arena */
static size_t contig_mem_prealloc_size;
static void *contig_mem_prealloc_buf;
/*
* The maximum amount a randomized mapping will be slewed. We should perhaps
* arrange things so these tunables can be separate for mmap, mmapobj, and
* ld.so
*/
/*
* map_addr_proc() is the routine called when the system is to
* choose an address for the user. We will pick an address
* range which is just below the current stack limit. The
* algorithm used for cache consistency on machines with virtual
* address caches is such that offset 0 in the vnode is always
* on a shm_alignment'ed aligned address. Unfortunately, this
* means that vnodes which are demand paged will not be mapped
* cache consistently with the executable images. When the
* cache alignment for a given object is inconsistent, the
* lower level code must manage the translations so that this
* is not seen here (at the cost of efficiency, of course).
*
* Every mapping will have a redzone of a single page on either side of
* the request. This is done to leave one page unmapped between segments.
* This is not required, but it's useful for the user because if their
* program strays across a segment boundary, it will catch a fault
* immediately making debugging a little easier. Currently the redzone
* is mandatory.
*
* On input it is a hint from the user to be used in a completely
* machine dependent fashion. For MAP_ALIGN, addrp contains the
* minimal alignment, which must be some "power of two" multiple of
* pagesize.
*
* On output it is NULL if no address can be found in the current
* processes address space or else an address that is currently
* not mapped for len bytes with a page of red zone on either side.
* If vacalign is true, then the selected address will obey the alignment
* constraints of a vac machine based on the given off value.
*/
/*ARGSUSED3*/
void
{
int allow_largepage_alignment = 1;
/*
* This happens when a program wants to map something in
* a range that's accessible to a program in a smaller
* address space. For example, a 64-bit program might
* be calling mmap32(2) to guarantee that the returned
* address is below 4Gbytes.
*/
} else {
}
/* Make len be a multiple of PAGESIZE */
/*
* If the request is larger than the size of a particular
* mmu level, then we use that level to map the request.
* But this requires that both the virtual and the physical
* addresses be aligned with respect to that level, so we
* do the virtual bit of nastiness here.
*
* For 32-bit processes, only those which have specified
* MAP_ALIGN or an addr will be aligned on a page size > 4MB. Otherwise
* we can potentially waste up to 256MB of the 4G process address
* space just for alignment.
*
* XXXQ Should iterate trough hw_page_array here to catch
* all supported pagesizes
*/
}
if ((mmu_page_sizes == max_mmu_page_sizes) &&
} else if ((mmu_page_sizes == max_mmu_page_sizes) &&
} else {
/*
* Align virtual addresses on a 64K boundary to ensure
* that ELF shared libraries are mapped with the appropriate
* alignment constraints by the run-time linker.
*/
}
/*
* 64-bit processes require 1024K alignment of ELF shared libraries.
*/
if (p->p_model == DATAMODEL_LP64)
#ifdef VAC
#endif
}
/*
* Look for a large enough hole starting below the stack limit.
* After finding it, use the upper part.
*/
/*
* addr is the highest possible address to use since we have
* a PAGESIZE redzone at the beginning and end.
*/
/*
* Round address DOWN to the alignment amount and
* add the offset in.
* If addr is greater than as_addr, len would not be large
* enough to include the redzone, so we must adjust down
* by the alignment amount.
*/
addr -= align_amount;
}
/*
* If randomization is requested, slew the allocation
* backwards, within the same gap, by a random amount.
*/
if (flags & _MAP_RANDOMIZE) {
sizeof (slew));
}
} else {
}
}
/*
* Platform-dependent page scrub call.
* We call hypervisor to scrub the page.
*/
void
{
}
void
{
/* Call memory sync function */
}
{
extern int mmu_exported_pagesize_mask;
if (lpsize == 0) {
} else {
}
return (lpsize);
}
continue;
return (lpsize);
}
return (lpsize);
}
void
{
}
/*ARGSUSED*/
void
{
}
static void *
{
int pgflags;
spgcnt_t i = 0;
return (NULL);
}
/* The address should be slab-size aligned. */
return (NULL);
}
if (vmflag & VM_NORELOC)
pgflags |= PG_NORELOC;
return (NULL);
}
}
/*
* Load the locked entry. It's OK to preload the entry into
* the TSB since we now support large mappings in the kernel TSB.
*/
for (--i; i >= 0; --i) {
/*
* Leave the page share locked. For non-cage pages,
* this would prevent memory DR if it were supported
* on sun4v.
*/
page_downgrade(ppa[i]);
}
return (addr);
}
/*
* Allocates a slab by first trying to use the largest slab size
* in contig_mem_import_sizes and then falling back to smaller slab
* sizes still large enough for the allocation. The sizep argument
* is a pointer to the requested size. When a slab is successfully
* allocated, the slab size, which must be >= *sizep and <=
* contig_mem_import_size_max, is returned in the *sizep argument.
* Returns the virtual address of the new slab.
*/
static void *
{
int i;
for (i = 0; i < NUM_IMPORT_SIZES; i++) {
/*
* Check that the alignment is also less than the
* import (large page) size. In the case where the
* alignment is larger than the size, a large page
* large enough for the allocation is not necessarily
* physical-address aligned to satisfy the requested
* alignment. Since alignment is required to be a
* power-of-2, any large page >= size && >= align will
* suffice.
*/
void *addr;
continue;
return (addr);
}
return (NULL);
}
return (NULL);
}
static void *
{
}
static void *
int vmflag)
{
}
/*
* Free a span, which is always exactly one large page.
*/
static void
{
/* All slabs should be size aligned */
panic("contig_mem_span_free: page not found");
}
if (!page_tryupgrade(pp)) {
panic("contig_mem_span_free: page not found");
}
}
}
static void *
int vmflag)
{
}
/*
* contig_mem_alloc, contig_mem_alloc_align
*
* Caution: contig_mem_alloc and contig_mem_alloc_align should be
* used only when physically contiguous non-relocatable memory is
* required. Furthermore, use of these allocation routines should be
* minimized as well as should the allocation size. As described in the
* contig_mem_arena comment block above, slab allocations fall back to
* being outside of the cage. Therefore, overuse of these allocation
* routines can lead to non-relocatable large pages being allocated
* outside the cage. Such pages prevent the allocation of a larger page
* occupying overlapping pages. This can impact performance for
* applications that utilize e.g. 256M large pages.
*/
/*
* Allocates size aligned contiguous memory up to contig_mem_import_size_max.
* Size must be a power of 2.
*/
void *
{
}
/*
* contig_mem_alloc_align allocates real contiguous memory with the
* specified alignment up to contig_mem_import_size_max. The alignment must
* be a power of 2 and no greater than contig_mem_import_size_max. We assert
* the aligment is a power of 2. For non-debug, vmem_xalloc will panic
* for non power of 2 alignments.
*/
void *
{
void *buf;
if (align < CONTIG_MEM_ARENA_QUANTUM)
/*
* We take the lock here to serialize span allocations.
* We do not lose concurrency for the common case, since
* allocations that don't require new span allocations
* are serialized by vmem_xalloc. Serializing span
* allocations also prevents us from trying to allocate
* more spans than necessary.
*/
}
}
return (buf);
}
void
{
} else if (size > MMU_PAGESIZE) {
} else {
}
}
/*
* We create a set of stacked vmem arenas to enable us to
* allocate large >PAGESIZE chucks of contiguous Real Address space.
* The vmem_xcreate interface is used to create the contig_mem_arena
* allowing the import routine to downsize the requested slab size
* and return a smaller slab.
*/
void
contig_mem_init(void)
{
== NULL) {
}
}
/*
* In calculating how much memory to pre-allocate, we include a small
* amount per-CPU to account for per-CPU buffers in line with measured
* values for different size systems. contig_mem_prealloc_base_size is
* a cpu specific amount to be pre-allocated before considering per-CPU
* requirements and memory size. We always pre-allocate a minimum amount
* of memory determined by PREALLOC_MIN. Beyond that, we take the minimum
* of contig_mem_prealloc_base_size and a small percentage of physical
* memory to prevent allocating too much on smaller systems.
* contig_mem_prealloc_base_size is global, allowing for the CPU module
* to increase its value if necessary.
*/
/*
* Called at boot-time allowing pre-allocation of contiguous memory.
* The argument 'alloc_base' is the requested base address for the
* allocation and originates in startup_memlist.
*/
{
MMU_PAGESIZE4M) != alloc_base) {
/*
* Failed. This may mean the physical memory has holes in it
* and it will be more difficult to get large contiguous
* pieces of memory. Since we only guarantee contiguous
* pieces of memory contig_mem_import_size_max or smaller,
* loop, getting contig_mem_import_size_max at a time, until
* failure or contig_mem_prealloc_size is reached.
*/
for (chunkp = alloc_base;
MMU_PAGESIZE4M) != chunkp) {
break;
}
}
ASSERT(contig_mem_prealloc_size != 0);
}
if (contig_mem_prealloc_size != 0) {
} else {
}
return (alloc_base);
}