vm_pagelist.c revision ca3e8d88e8c867355e441fbc914c52e7416fc537
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * CDDL HEADER START
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The contents of this file are subject to the terms of the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Common Development and Distribution License (the "License").
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * You may not use this file except in compliance with the License.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * See the License for the specific language governing permissions
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * and limitations under the License.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * When distributing Covered Code, include this CDDL HEADER in each
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If applicable, add the following below this CDDL HEADER, with the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * fields enclosed by brackets "[]" replaced with your own identifying
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * information: Portions Copyright [yyyy] [name of copyright owner]
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * CDDL HEADER END
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Use is subject to license terms.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/* All Rights Reserved */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Portions of this source code were derived from Berkeley 4.3 BSD
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * under license from the Regents of the University of California.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This file contains common functions to access and manage the page lists.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Many of these routines originated from platform dependent modules
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * (sun4/vm/vm_dep.c, i86pc/vm/vm_machdep.c) and modified to function in
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * a platform independent manner.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * vm/vm_dep.h provides for platform specific support.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/* vm_cpu_data0 for the boot cpu before kmem is initialized */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * number of page colors equivalent to reqested color in page_get routines.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If set, keeps large pages intact longer and keeps MPO allocation
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * from the local mnode in favor of acquiring the 'correct' page color from
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * a demoted large page or from a remote mnode.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * color equivalency mask for each page size.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Mask is computed based on cpu L2$ way sizes and colorequiv global.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * High 4 bits determine the number of high order bits of the color to ignore.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Low 4 bits determines number of low order bits of color to ignore (it's only
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * relevant for hashed index based page coloring).
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * if set, specifies the percentage of large pages that are free from within
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * a large page region before attempting to lock those pages for
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page_get_contig_pages processing.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Should be turned on when kpr is available when page_trylock_contig_pages
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * can be more selective.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Limit page get contig page search based on failure cnts in pgcpfailcnt[].
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Enabled by default via pgcplimitsearch.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * pgcpfailcnt[] is bounded by PGCPFAILMAX (>= 1/2 of installed
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * memory). When reached, pgcpfailcnt[] is reset to 1/2 of this upper
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * bound. This upper bound range guarantees:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - all large page 'slots' will be searched over time
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - the minimum (1) large page candidates considered on each pgcp call
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - count doesn't wrap around to 0
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#define PGCPFAILMAX (1 << (highbit(physinstalled) - 1))
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#endif /* VM_STATS */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/* enable page_get_contig_pages */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page_freelist_split pfn flag to signify no lo or hi pfn requirement.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/* Flags involved in promotion and demotion routines */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#define PC_ALLOC 0x2 /* return page for allocation */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Flag for page_demote to be used with PC_FREE to denote that we don't care
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * what the color is as the color parameter to the function is ignored.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/* mtype value for page_promote to use when mtype does not matter */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page counters candidates info
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * See page_ctrs_cands comment below for more details.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * fields are as follows:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * pcc_pages_free: # pages which freelist coalesce can create
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * pcc_color_free: pointer to page free counts per color
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otatypedef struct pcc_info {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * On big machines it can take a long time to check page_counters
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * arrays. page_ctrs_cands is a summary array whose elements are a dynamically
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * updated sum of all elements of the corresponding page_counters arrays.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page_freelist_coalesce() searches page_counters only if an appropriate
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * element of page_ctrs_cands array is greater than 0.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page_ctrs_cands is indexed by mutex (i), region (r), mnode (m), mrange (g)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otapcc_info_t **page_ctrs_cands[NPC_MUTEX][MMU_PAGE_SIZES];
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Return in val the total number of free pages which can be created
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * for the given mnode (m), mrange (g), and region size (r)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota for (i = 0; i < NPC_MUTEX; i++) { \
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota val += page_ctrs_cands[i][(r)][(m)][(g)].pcc_pages_free; \
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Return in val the total number of free pages which can be created
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * for the given mnode (m), mrange (g), region size (r), and color (c)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#define PGCTRS_CANDS_GETVALUECOLOR(m, g, r, c, val) { \
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota for (i = 0; i < NPC_MUTEX; i++) { \
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota page_ctrs_cands[i][(r)][(m)][(g)].pcc_color_free[(c)]; \
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We can only allow a single thread to update a counter within the physical
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * range of the largest supported page size. That is the finest granularity
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * possible since the counter values are dependent on each other
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * as you move accross region sizes. PP_CTR_LOCK_INDX is used to determine the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * ctr_mutex lock index for a particular physical range.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (PAGE_BSZS_SHIFT(mmu_page_sizes - 1))) & (NPC_MUTEX - 1))
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Local functions prototypes.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid page_ctr_add_internal(int, int, page_t *, int);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid page_ctr_sub_internal(int, int, page_t *, int);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otapage_t *page_promote(int, pfn_t, uchar_t, int, int);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otapage_t *page_demote(int, pfn_t, pfn_t, uchar_t, uchar_t, int, int);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota uint_t, int, int, pfn_t, pfn_t, page_list_walker_t *);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otapage_t *page_get_mnode_cachelist(uint_t, uint_t, int, int);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The page_counters array below is used to keep track of free contiguous
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * physical memory. A hw_page_map_t will be allocated per mnode per szc.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This contains an array of counters, the size of the array, a shift value
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * used to convert a pagenum into a counter array index or vice versa, as
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * well as a cache of the last successful index to be promoted to a larger
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page size. As an optimization, we keep track of the last successful index
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * to be promoted per page color for the given size region, and this is
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * allocated dynamically based upon the number of colors for a given
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * region size.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Conceptually, the page counters are represented as:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page_counters[region_size][mnode]
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * region_size: size code of a candidate larger page made up
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * of contiguous free smaller pages.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page_counters[region_size][mnode].hpm_counters[index]:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * represents how many (region_size - 1) pages either
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * exist or can be created within the given index range.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Let's look at a sparc example:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If we want to create a free 512k page, we look at region_size 2
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * for the mnode we want. We calculate the index and look at a specific
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * hpm_counters location. If we see 8 (FULL_REGION_CNT on sparc) at
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * this location, it means that 8 64k pages either exist or can be created
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * from 8K pages in order to make a single free 512k page at the given
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * index. Note that when a region is full, it will contribute to the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * counts in the region above it. Thus we will not know what page
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * size the free pages will be which can be promoted to this new free
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page unless we look at all regions below the current region.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Note: hpmctr_t is defined in platform vm_dep.h
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * hw_page_map_t contains all the information needed for the page_counters
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * logic. The fields are as follows:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * hpm_counters: dynamically allocated array to hold counter data
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * hpm_entries: entries in hpm_counters
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * hpm_shift: shift for pnum/array index conv
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * hpm_base: PFN mapped to counter index 0
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * hpm_color_current: last index in counter array for this color at
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * which we successfully created a large page
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otatypedef struct hw_page_map {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Element zero is not used, but is allocated for convenience.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic hw_page_map_t *page_counters[MMU_PAGE_SIZES];
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Cached value of MNODE_RANGE_CNT(mnode).
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This is a function call in x86.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The following macros are convenient ways to get access to the individual
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * elements of the page_counters arrays. They can be used on both
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * the left side and right side of equations.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (page_counters[(rg_szc)][(mnode)].hpm_counters[(idx)])
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#define PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, rg_szc, g) \
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (page_counters[(rg_szc)][(mnode)].hpm_color_current[(g)])
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#define PAGE_COUNTERS_CURRENT_COLOR(mnode, rg_szc, color, mrange) \
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (((pnum) - PAGE_COUNTERS_BASE((mnode), (rg_szc))) >> \
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ((index) << PAGE_COUNTERS_SHIFT((mnode), (rg_szc))))
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Protects the hpm_counters and hpm_color_current memory from changing while
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * looking at page counters information.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Grab the write lock to modify what these fields point at.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Grab the read lock to prevent any pointers from changing.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The write lock can not be held during memory allocation due to a possible
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * recursion deadlock with trying to grab the read lock while the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * write lock is already held.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * initialize cpu_vm_data to point at cache aligned vm_cpu_data_t.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota align = (L2CACHE_ALIGN) ? L2CACHE_ALIGN : L2CACHE_ALIGN_MAX;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota sz = P2ROUNDUP(sizeof (vm_cpu_data_t), align) + align;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota cp->cpu_vm_data = (void *) P2ROUNDUP((uintptr_t)kmptr, align);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr = kmptr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize = sz;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * free cpu_vm_data
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota kmem_free(((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page size to page size code
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (i);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (-1);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * page size to page size code with the restriction that it be a supported
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * user page size. If it's not a supported user page size, -1 will be returned.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (-1);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Return how many page sizes are available for the user to use. This is
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * what the hardware supports and not based upon how the OS implements the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * support of different page sizes.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If legacy is non-zero, return the number of pagesizes available to legacy
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * applications. The number of legacy page sizes might be less than the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * exported user page sizes. This is to prevent legacy applications that
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * use the largest page size returned from getpagesizes(3c) from inadvertantly
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * using the 'new' large pagesizes.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * returns the count of the number of base pagesize pages associated with szc
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Return the size of a page based upon the index passed in. An index of
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * zero refers to the smallest page size in the system, and as index increases
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * it refers to the next larger supported page size in the system.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Note that szc and userszc may not be the same due to unsupported szc's on
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * some systems.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota panic("page_get_user_pagesize: out of range %d", szc);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota panic("page_get_pagecolors: out of range %d", szc);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * this assigns the desired equivalent color after a split
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otapage_correct_color(uchar_t szc, uchar_t nszc, uint_t color,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The interleaved_mnodes flag is set when mnodes overlap in
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * the physbase..physmax range, but have disjoint slices.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * In this case hpm_counters is shared by all mnodes.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This flag is set dynamically by the platform.
page_ctrs_sz(void)
int mnode;
int nranges;
for (i = 0; i < mmu_page_sizes; i++) {
sizeof (hpmctr_t *));
int mnode;
for (i = 0; i < mmu_page_sizes; i++) {
for (i = 0; i < NPC_MUTEX; i++) {
colors_per_szc[r];
pi++;
for (i = 0; i < NPC_MUTEX; i++) {
int r_shift;
colors_per_szc[r];
for (i = 0; i < colors_per_szc[r]; i++) {
int mrange;
idx = 0;
alloc_base +=
sizeof (hpmctr_t *));
return (alloc_base);
int lckidx;
while (r < mmu_page_sizes) {
int lckidx;
while (r < mmu_page_sizes) {
int cands_cache_nranges;
int rc = 0;
int oldmnode;
return (ENOMEM);
for (r = 0; r < mmu_page_sizes; r++) {
goto cleanup;
goto cleanup;
for (i = 0; i < NPC_MUTEX; i++) {
goto cleanup;
goto cleanup;
if (interleaved_mnodes) {
for (i = 0; i < max_mem_nodes; i++)
oldmnode = i;
r, mrange);
sizeof (hpmctr_t));
sizeof (hpmctr_t));
if (interleaved_mnodes) {
for (i = 0; i < max_mem_nodes; i++) {
if (i == mnode)
for (i = 0; i < colors_per_szc[r]; i++) {
idx = 0;
for (i = 0; i < NPC_MUTEX; i++) {
for (i = 0; i < NPC_MUTEX; i++) {
colors_per_szc[r] *
sizeof (pgcnt_t));
return (rc);
#ifdef DEBUG
while (npgs--) {
if (npgs != 0) {
for (i = 0; i < NPC_MUTEX; i++) {
for (i = 0; i < NPC_MUTEX; i++) {
int mnode;
#if defined(__sparc)
#ifdef __sparc
* It performs the function of page_list_sub/PP_SETNORELOC/page_list_add
int mnode;
int mtype;
int flags = 0;
int mnode;
#if defined(__sparc)
PC_FREE);
int bin;
int mnode;
goto try_again;
#if defined(__sparc)
#if defined(__sparc)
int mnode;
goto try_again;
#if defined(__sparc)
int mnode;
int idx;
page_t *
int which_list;
return (NULL);
return (NULL);
while (pages_left) {
goto fail_promote;
goto fail_promote;
while (npgs--) {
return (pplist);
return (NULL);
while (pplist) {
return (NULL);
page_t *
while (pplist) {
int count = 0;
n = npgs;
count++;
SE_EXCL)) {
return (ret_pp);
int mpss_coalesce_disable = 0;
page_t *
int mrange;
#if defined(__sparc)
if (mpss_coalesce_disable) {
return (NULL);
hi++;
return (NULL);
&it);
return (NULL);
if (acand) {
if (cands == 0) {
return (NULL);
#if defined(__sparc)
#if defined(__sparc)
goto wrapit;
goto wrapit;
goto next;
if (interleaved_mnodes &&
goto next;
goto next;
ret_pp =
#if defined(__sparc)
return (ret_pp);
if (--cands == 0) {
idx;
next:
wrap++;
#if defined(__sparc)
return (NULL);
if (mpss_coalesce_disable) {
if (cands != 0)
if (cands == 0) {
pfnum <
if (interleaved_mnodes)
page_t *
return (NULL);
if (pp &&
if (pp) {
if (ret_pp) {
#if defined(__sparc)
return (ret_pp);
return (ret_pp);
if (cpu_page_colors < 0) {
if (can_split) {
if (szc == 0) {
return (nbin);
nbin);
return (nbin);
return (nbin);
page_t *
int plw_initialized;
return (NULL);
plw_initialized = 0;
goto bin_empty_1;
goto bin_empty_0;
if (szc == 0) {
goto bin_empty_0;
if (szc == 0) {
#if defined(__sparc)
return (pp);
if (plw_initialized == 0) {
&plw);
return (pp);
return (pp);
if (mtype >= 0)
goto try_again;
return (NULL);
return (pgfree);
return (pgfree);
goto skipptcpcheck;
for (i = 0; i < pgcnt; i++) {
ASSERT(i == 0);
static page_t *
while (pgcnt) {
return (pp);
pp++;
pgcnt--;
if (replpp) {
while (pgcnt--) {
pp++;
while (pplist) {
return (NULL);
while (npgs--) {
return (pplist);
int decr;
int rc = 0;
return (rc);
static page_t *
return (NULL);
ceq_mask = 0;
int slotlen;
return (NULL);
memsegs_lock(0);
if (kcage_on) {
&it);
memsegs_unlock(0);
return (pp);
memsegs_unlock(0);
return (NULL);
page_t *
return (NULL);
if (pfnflag) {
return (pp);
} while (mtype >= 0);
return (NULL);
* Determine the likelihood of finding/coalescing a szc page.
page_t *
int mnode;
int mtype;
if (kcage_on) {
return (NULL);
int, mnode,
return (pp);
return (NULL);
int, mnode,
return (pp);
goto pgretry;
return (NULL);
page_t *
int mnode;
int mtype;
if (!kcage_on) {
return (NULL);
int, mnode,
return (pp);
* Try freelists/cachelists that are farther away
0, flags);
int, mnode,
return (pp);
int, mnode,
return (pp);
return (NULL);
page_t *
int plw_initialized;
return (NULL);
plw_initialized = 0;
goto bin_empty_1;
goto bin_empty_0;
mnode);
if (pp) {
#if defined(__sparc)
return (pp);
if (plw_initialized == 0) {
if (mtype >= 0)
goto try_again;
return (NULL);
#ifdef DEBUG
#define REPL_PAGE_STATS
#ifdef REPL_PAGE_STATS
struct repl_page_stats {
#define REPL_STAT_INCR(v)
int pgrppgcp;
page_t *
int szc;
int mtype;
int flags = 0;
while (npgs) {
pplist =
pplist =
if (szc == 0) {
lgrp =
(mnode =
szc = 0;
int, mnode,
npgs--;
if (npgs) {
return (NULL);
return (pl);
int mnode;
page_set_colorequiv_arr(void)
for (i = 0; i < MMU_PAGE_SIZES; i++) {
while ((colors >> a) == 0)