/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 1987, 2010, Oracle and/or its affiliates. All rights reserved.
*/
/*
* VM - Hardware Address Translation management.
*
* This file describes the contents of the sun-reference-mmu(sfmmu)-
* specific hat data structures and the sfmmu-specific hat procedures.
* The machine-independent interface is described in <vm/hat.h>.
*/
#ifndef _VM_HAT_SFMMU_H
#define _VM_HAT_SFMMU_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef _ASM
#include <sys/types.h>
#endif /* _ASM */
#ifdef _KERNEL
#include <sys/pte.h>
#include <vm/mach_sfmmu.h>
#include <sys/mmu.h>
/*
* Don't alter these without considering changes to ism_map_t.
*/
#define DEFAULT_ISM_PAGESIZE MMU_PAGESIZE4M
#define DEFAULT_ISM_PAGESZC TTE4M
#define ISM_PG_SIZE(ism_vbshift) (1 << ism_vbshift)
#define ISM_SZ_MASK(ism_vbshift) (ISM_PG_SIZE(ism_vbshift) - 1)
#define ISM_MAP_SLOTS 8 /* Change this carefully. */
#ifndef _ASM
#include <sys/t_lock.h>
#include <vm/hat.h>
#include <vm/seg.h>
#include <sys/machparam.h>
#include <sys/systm.h>
#include <sys/x_call.h>
#include <vm/page.h>
#include <sys/ksynch.h>
typedef struct hat sfmmu_t;
typedef struct sf_scd sf_scd_t;
/*
* SFMMU attributes for hat_memload/hat_devload
*/
#define SFMMU_UNCACHEPTTE 0x01000000 /* unencache in physical $ */
#define SFMMU_UNCACHEVTTE 0x02000000 /* unencache in virtual $ */
#define SFMMU_SIDEFFECT 0x04000000 /* set side effect bit */
#define SFMMU_LOAD_ALLATTR (HAT_PROT_MASK | HAT_ORDER_MASK | \
HAT_ENDIAN_MASK | HAT_NOFAULT | HAT_NOSYNC | \
SFMMU_UNCACHEPTTE | SFMMU_UNCACHEVTTE | SFMMU_SIDEFFECT)
/*
* sfmmu flags for hat_memload/hat_devload
*/
#define SFMMU_NO_TSBLOAD 0x08000000 /* do not preload tsb */
#define SFMMU_LOAD_ALLFLAG (HAT_LOAD | HAT_LOAD_LOCK | \
HAT_LOAD_ADV | HAT_LOAD_CONTIG | HAT_LOAD_NOCONSIST | \
HAT_LOAD_SHARE | HAT_LOAD_REMAP | SFMMU_NO_TSBLOAD | \
HAT_RELOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_TEXT)
/*
* sfmmu internal flag to hat_pageunload that spares locked mappings
*/
#define SFMMU_KERNEL_RELOC 0x8000
/*
* mode for sfmmu_chgattr
*/
#define SFMMU_SETATTR 0x0
#define SFMMU_CLRATTR 0x1
#define SFMMU_CHGATTR 0x2
/*
* sfmmu specific flags for page_t
*/
#define P_PNC 0x8 /* non-caching is permanent bit */
#define P_TNC 0x10 /* non-caching is temporary bit */
#define P_KPMS 0x20 /* kpm mapped small (vac alias prevention) */
#define P_KPMC 0x40 /* kpm conflict page (vac alias prevention) */
#define PP_GENERIC_ATTR(pp) ((pp)->p_nrm & (P_MOD | P_REF | P_RO))
#define PP_ISMOD(pp) ((pp)->p_nrm & P_MOD)
#define PP_ISREF(pp) ((pp)->p_nrm & P_REF)
#define PP_ISRO(pp) ((pp)->p_nrm & P_RO)
#define PP_ISNC(pp) ((pp)->p_nrm & (P_PNC|P_TNC))
#define PP_ISPNC(pp) ((pp)->p_nrm & P_PNC)
#ifdef VAC
#define PP_ISTNC(pp) ((pp)->p_nrm & P_TNC)
#endif
#define PP_ISKPMS(pp) ((pp)->p_nrm & P_KPMS)
#define PP_ISKPMC(pp) ((pp)->p_nrm & P_KPMC)
#define PP_SETMOD(pp) ((pp)->p_nrm |= P_MOD)
#define PP_SETREF(pp) ((pp)->p_nrm |= P_REF)
#define PP_SETREFMOD(pp) ((pp)->p_nrm |= (P_REF|P_MOD))
#define PP_SETRO(pp) ((pp)->p_nrm |= P_RO)
#define PP_SETREFRO(pp) ((pp)->p_nrm |= (P_REF|P_RO))
#define PP_SETPNC(pp) ((pp)->p_nrm |= P_PNC)
#ifdef VAC
#define PP_SETTNC(pp) ((pp)->p_nrm |= P_TNC)
#endif
#define PP_SETKPMS(pp) ((pp)->p_nrm |= P_KPMS)
#define PP_SETKPMC(pp) ((pp)->p_nrm |= P_KPMC)
#define PP_CLRMOD(pp) ((pp)->p_nrm &= ~P_MOD)
#define PP_CLRREF(pp) ((pp)->p_nrm &= ~P_REF)
#define PP_CLRREFMOD(pp) ((pp)->p_nrm &= ~(P_REF|P_MOD))
#define PP_CLRRO(pp) ((pp)->p_nrm &= ~P_RO)
#define PP_CLRPNC(pp) ((pp)->p_nrm &= ~P_PNC)
#ifdef VAC
#define PP_CLRTNC(pp) ((pp)->p_nrm &= ~P_TNC)
#endif
#define PP_CLRKPMS(pp) ((pp)->p_nrm &= ~P_KPMS)
#define PP_CLRKPMC(pp) ((pp)->p_nrm &= ~P_KPMC)
/*
* All shared memory segments attached with the SHM_SHARE_MMU flag (ISM)
* will be constrained to a 4M, 32M or 256M alignment. Also since every newly-
* created ISM segment is created out of a new address space at base va
* of 0 we don't need to store it.
*/
#define ISM_ALIGN(shift) (1 << shift) /* base va aligned to <n>M */
#define ISM_ALIGNED(shift, va) (((uintptr_t)va & (ISM_ALIGN(shift) - 1)) == 0)
#define ISM_SHIFT(shift, x) ((uintptr_t)x >> (shift))
/*
* Pad locks out to cache sub-block boundaries to prevent
* false sharing, so several processes don't contend for
* the same line if they aren't using the same lock. Since
* this is a typedef we also have a bit of freedom in
* changing lock implementations later if we decide it
* is necessary.
*/
typedef struct hat_lock {
kmutex_t hl_mutex;
uchar_t hl_pad[64 - sizeof (kmutex_t)];
} hatlock_t;
#define HATLOCK_MUTEXP(hatlockp) (&((hatlockp)->hl_mutex))
/*
* All segments mapped with ISM are guaranteed to be 4M, 32M or 256M aligned.
* Also size is guaranteed to be in 4M, 32M or 256M chunks.
* ism_seg consists of the following members:
* [XX..22] base address of ism segment. XX is 63 or 31 depending whether
* caddr_t is 64 bits or 32 bits.
* [21..0] size of segment.
*
* NOTE: Don't alter this structure without changing defines above and
* the tsb_miss and protection handlers.
*/
typedef struct ism_map {
uintptr_t imap_seg; /* base va + sz of ISM segment */
uchar_t imap_vb_shift; /* mmu_pageshift for ism page size */
uchar_t imap_rid; /* region id for ism */
ushort_t imap_hatflags; /* primary ism page size */
uint_t imap_sz_mask; /* mmu_pagemask for ism page size */
sfmmu_t *imap_ismhat; /* hat id of dummy ISM as */
struct ism_ment *imap_ment; /* pointer to mapping list entry */
} ism_map_t;
#define ism_start(map) ((caddr_t)((map).imap_seg & \
~ISM_SZ_MASK((map).imap_vb_shift)))
#define ism_size(map) ((map).imap_seg & ISM_SZ_MASK((map).imap_vb_shift))
#define ism_end(map) ((caddr_t)(ism_start(map) + (ism_size(map) * \
ISM_PG_SIZE((map).imap_vb_shift))))
/*
* ISM mapping entry. Used to link all hat's sharing a ism_hat.
* Same function as the p_mapping list for a page.
*/
typedef struct ism_ment {
sfmmu_t *iment_hat; /* back pointer to hat_share() hat */
caddr_t iment_base_va; /* hat's va base for this ism seg */
struct ism_ment *iment_next; /* next ism map entry */
struct ism_ment *iment_prev; /* prev ism map entry */
} ism_ment_t;
/*
* ISM segment block. One will be hung off the sfmmu structure if a
* a process uses ISM. More will be linked using ismblk_next if more
* than ISM_MAP_SLOTS segments are attached to this proc.
*
* All modifications to fields in this structure will be protected
* by the hat mutex. In order to avoid grabbing this lock in low level
* routines (tsb miss/protection handlers and vatopfn) while not
* introducing any race conditions with hat_unshare, we will set
* CTX_ISM_BUSY bit in the ctx struct. Any mmu traps that occur
* for this ctx while this bit is set will be handled in sfmmu_tsb_excption
* where it will synchronize behind the hat mutex.
*/
typedef struct ism_blk {
ism_map_t iblk_maps[ISM_MAP_SLOTS];
struct ism_blk *iblk_next;
uint64_t iblk_nextpa;
} ism_blk_t;
/*
* TSB access information. All fields are protected by the process's
* hat lock.
*/
struct tsb_info {
caddr_t tsb_va; /* tsb base virtual address */
uint64_t tsb_pa; /* tsb base physical address */
struct tsb_info *tsb_next; /* next tsb used by this process */
uint16_t tsb_szc; /* tsb size code */
uint16_t tsb_flags; /* flags for this tsb; see below */
uint_t tsb_ttesz_mask; /* page size masks; see below */
tte_t tsb_tte; /* tte to lock into DTLB */
sfmmu_t *tsb_sfmmu; /* sfmmu */
kmem_cache_t *tsb_cache; /* cache from which mem allocated */
vmem_t *tsb_vmp; /* vmem arena from which mem alloc'd */
};
/*
* Values for "tsb_ttesz_mask" bitmask.
*/
#define TSB8K (1 << TTE8K)
#define TSB64K (1 << TTE64K)
#define TSB512K (1 << TTE512K)
#define TSB4M (1 << TTE4M)
#define TSB32M (1 << TTE32M)
#define TSB256M (1 << TTE256M)
/*
* Values for "tsb_flags" field.
*/
#define TSB_RELOC_FLAG 0x1
#define TSB_FLUSH_NEEDED 0x2
#define TSB_SWAPPED 0x4
#define TSB_SHAREDCTX 0x8
#endif /* !_ASM */
/*
* Data structures for shared hmeblk support.
*/
/*
* Do not increase the maximum number of ism/hme regions without checking first
* the impact on ism_map_t, TSB miss area, hblk tag and region id type in
* sf_region structure.
* Initially, shared hmes will only be used for the main text segment
* therefore this value will be set to 64, it will be increased when shared
* libraries are included.
*/
#define SFMMU_MAX_HME_REGIONS (64)
#define SFMMU_HMERGNMAP_WORDS BT_BITOUL(SFMMU_MAX_HME_REGIONS)
#define SFMMU_PRIVATE 0
#define SFMMU_SHARED 1
#define HMEBLK_ENDPA 1
#ifndef _ASM
#define SFMMU_MAX_ISM_REGIONS (64)
#define SFMMU_ISMRGNMAP_WORDS BT_BITOUL(SFMMU_MAX_ISM_REGIONS)
#define SFMMU_RGNMAP_WORDS (SFMMU_HMERGNMAP_WORDS + SFMMU_ISMRGNMAP_WORDS)
#define SFMMU_MAX_REGION_BUCKETS (128)
#define SFMMU_MAX_SRD_BUCKETS (2048)
typedef struct sf_hmeregion_map {
ulong_t bitmap[SFMMU_HMERGNMAP_WORDS];
} sf_hmeregion_map_t;
typedef struct sf_ismregion_map {
ulong_t bitmap[SFMMU_ISMRGNMAP_WORDS];
} sf_ismregion_map_t;
typedef union sf_region_map_u {
struct _h_rmap_s {
sf_hmeregion_map_t hmeregion_map;
sf_ismregion_map_t ismregion_map;
} h_rmap_s;
ulong_t bitmap[SFMMU_RGNMAP_WORDS];
} sf_region_map_t;
#define SF_RGNMAP_ZERO(map) { \
int _i; \
for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) { \
(map).bitmap[_i] = 0; \
} \
}
/*
* Returns 1 if map1 and map2 are equal.
*/
#define SF_RGNMAP_EQUAL(map1, map2, rval) { \
int _i; \
for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) { \
if ((map1)->bitmap[_i] != (map2)->bitmap[_i]) \
break; \
} \
if (_i < SFMMU_RGNMAP_WORDS) \
rval = 0; \
else \
rval = 1; \
}
#define SF_RGNMAP_ADD(map, r) BT_SET((map).bitmap, r)
#define SF_RGNMAP_DEL(map, r) BT_CLEAR((map).bitmap, r)
#define SF_RGNMAP_TEST(map, r) BT_TEST((map).bitmap, r)
/*
* Tests whether map2 is a subset of map1, returns 1 if
* this assertion is true.
*/
#define SF_RGNMAP_IS_SUBSET(map1, map2, rval) { \
int _i; \
for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) { \
if (((map1)->bitmap[_i] & (map2)->bitmap[_i]) \
!= (map2)->bitmap[_i]) { \
break; \
} \
} \
if (_i < SFMMU_RGNMAP_WORDS) \
rval = 0; \
else \
rval = 1; \
}
#define SF_SCD_INCR_REF(scdp) { \
atomic_inc_32((volatile uint32_t *)&(scdp)->scd_refcnt); \
}
#define SF_SCD_DECR_REF(srdp, scdp) { \
sf_region_map_t _scd_rmap = (scdp)->scd_region_map; \
if (!atomic_dec_32_nv((volatile uint32_t *)&(scdp)->scd_refcnt)) {\
sfmmu_destroy_scd((srdp), (scdp), &_scd_rmap); \
} \
}
/*
* A sfmmup link in the link list of sfmmups that share the same region.
*/
typedef struct sf_rgn_link {
sfmmu_t *next;
sfmmu_t *prev;
} sf_rgn_link_t;
/*
* rgn_flags values.
*/
#define SFMMU_REGION_HME 0x1
#define SFMMU_REGION_ISM 0x2
#define SFMMU_REGION_FREE 0x8
#define SFMMU_REGION_TYPE_MASK (0x3)
/*
* sf_region defines a text or (D)ISM segment which map
* the same underlying physical object.
*/
typedef struct sf_region {
caddr_t rgn_saddr; /* base addr of attached seg */
size_t rgn_size; /* size of attached seg */
void *rgn_obj; /* the underlying object id */
u_offset_t rgn_objoff; /* offset in the object mapped */
uchar_t rgn_perm; /* PROT_READ/WRITE/EXEC */
uchar_t rgn_pgszc; /* page size of the region */
uchar_t rgn_flags; /* region type, free flag */
uchar_t rgn_id;
int rgn_refcnt; /* # of hats sharing the region */
/* callback function for hat_unload_callback */
hat_rgn_cb_func_t rgn_cb_function;
struct sf_region *rgn_hash; /* hash chain linking the rgns */
kmutex_t rgn_mutex; /* protect region sfmmu list */
/* A link list of processes attached to this region */
sfmmu_t *rgn_sfmmu_head;
ulong_t rgn_ttecnt[MMU_PAGE_SIZES];
uint16_t rgn_hmeflags; /* rgn tte size flags */
} sf_region_t;
#define rgn_next rgn_hash
/* srd */
typedef struct sf_shared_region_domain {
vnode_t *srd_evp; /* executable vnode */
/* hme region table */
sf_region_t *srd_hmergnp[SFMMU_MAX_HME_REGIONS];
/* ism region table */
sf_region_t *srd_ismrgnp[SFMMU_MAX_ISM_REGIONS];
/* hash chain linking srds */
struct sf_shared_region_domain *srd_hash;
/* pointer to the next free hme region */
sf_region_t *srd_hmergnfree;
/* pointer to the next free ism region */
sf_region_t *srd_ismrgnfree;
/* id of next ism region created */
uint16_t srd_next_ismrid;
/* id of next hme region created */
uint16_t srd_next_hmerid;
uint16_t srd_ismbusyrgns; /* # of ism rgns in use */
uint16_t srd_hmebusyrgns; /* # of hme rgns in use */
int srd_refcnt; /* # of procs in the srd */
kmutex_t srd_mutex; /* sync add/remove rgns */
kmutex_t srd_scd_mutex;
sf_scd_t *srd_scdp; /* list of scds in srd */
/* hash of regions associated with the same executable */
sf_region_t *srd_rgnhash[SFMMU_MAX_REGION_BUCKETS];
} sf_srd_t;
typedef struct sf_srd_bucket {
kmutex_t srdb_lock;
sf_srd_t *srdb_srdp;
} sf_srd_bucket_t;
/*
* The value of SFMMU_L1_HMERLINKS and SFMMU_L2_HMERLINKS will be increased
* to 16 when the use of shared hmes for shared libraries is enabled.
*/
#define SFMMU_L1_HMERLINKS (8)
#define SFMMU_L2_HMERLINKS (8)
#define SFMMU_L1_HMERLINKS_SHIFT (3)
#define SFMMU_L1_HMERLINKS_MASK (SFMMU_L1_HMERLINKS - 1)
#define SFMMU_L2_HMERLINKS_MASK (SFMMU_L2_HMERLINKS - 1)
#define SFMMU_L1_HMERLINKS_SIZE \
(SFMMU_L1_HMERLINKS * sizeof (sf_rgn_link_t *))
#define SFMMU_L2_HMERLINKS_SIZE \
(SFMMU_L2_HMERLINKS * sizeof (sf_rgn_link_t))
#if (SFMMU_L1_HMERLINKS * SFMMU_L2_HMERLINKS < SFMMU_MAX_HME_REGIONS)
#error Not Enough HMERLINKS
#endif
/*
* This macro grabs hat lock and allocates level 2 hat chain
* associated with a shme rgn. In the majority of cases, the macro
* is called with alloc = 0, and lock = 0.
* A pointer to the level 2 sf_rgn_link_t structure is returned in the lnkp
* parameter.
*/
#define SFMMU_HMERID2RLINKP(sfmmup, rid, lnkp, alloc, lock) \
{ \
int _l1ix = ((rid) >> SFMMU_L1_HMERLINKS_SHIFT) & \
SFMMU_L1_HMERLINKS_MASK; \
int _l2ix = ((rid) & SFMMU_L2_HMERLINKS_MASK); \
hatlock_t *_hatlockp; \
lnkp = (sfmmup)->sfmmu_hmeregion_links[_l1ix]; \
if (lnkp != NULL) { \
lnkp = &lnkp[_l2ix]; \
} else if (alloc && lock) { \
lnkp = kmem_zalloc(SFMMU_L2_HMERLINKS_SIZE, KM_SLEEP); \
_hatlockp = sfmmu_hat_enter(sfmmup); \
if ((sfmmup)->sfmmu_hmeregion_links[_l1ix] != NULL) { \
sfmmu_hat_exit(_hatlockp); \
kmem_free(lnkp, SFMMU_L2_HMERLINKS_SIZE); \
lnkp = (sfmmup)->sfmmu_hmeregion_links[_l1ix]; \
ASSERT(lnkp != NULL); \
} else { \
(sfmmup)->sfmmu_hmeregion_links[_l1ix] = lnkp; \
sfmmu_hat_exit(_hatlockp); \
} \
lnkp = &lnkp[_l2ix]; \
} else if (alloc) { \
lnkp = kmem_zalloc(SFMMU_L2_HMERLINKS_SIZE, KM_SLEEP); \
ASSERT((sfmmup)->sfmmu_hmeregion_links[_l1ix] == NULL); \
(sfmmup)->sfmmu_hmeregion_links[_l1ix] = lnkp; \
lnkp = &lnkp[_l2ix]; \
} \
}
/*
* Per cpu pending freelist of hmeblks.
*/
typedef struct cpu_hme_pend {
struct hme_blk *chp_listp;
kmutex_t chp_mutex;
time_t chp_timestamp;
uint_t chp_count;
uint8_t chp_pad[36]; /* pad to 64 bytes */
} cpu_hme_pend_t;
/*
* The default value of the threshold for the per cpu pending queues of hmeblks.
* The queues are flushed if either the number of hmeblks on the queue is above
* the threshold, or one second has elapsed since the last flush.
*/
#define CPU_HME_PEND_THRESH 1000
/*
* Per-MMU context domain kstats.
*
* TSB Miss Exceptions
* Number of times a TSB miss exception is handled in an MMU. See
* sfmmu_tsbmiss_exception() for more details.
* TSB Raise Exception
* Number of times the CPUs within an MMU are cross-called
* to invalidate either a specific process context (when the process
* switches MMU contexts) or the context of any process that is
* running on those CPUs (as part of the MMU context wrap-around).
* Wrap Around
* The number of times a wrap-around of MMU context happens.
*/
typedef enum mmu_ctx_stat_types {
MMU_CTX_TSB_EXCEPTIONS, /* TSB miss exceptions handled */
MMU_CTX_TSB_RAISE_EXCEPTION, /* ctx invalidation cross calls */
MMU_CTX_WRAP_AROUND, /* wraparounds */
MMU_CTX_NUM_STATS
} mmu_ctx_stat_t;
/*
* Per-MMU context domain structure. This is instantiated the first time a CPU
* belonging to the MMU context domain is configured into the system, at boot
* time or at DR time.
*
* mmu_gnum
* The current generation number for the context IDs on this MMU context
* domain. It is protected by mmu_lock.
* mmu_cnum
* The current cnum to be allocated on this MMU context domain. It
* is protected via CAS.
* mmu_nctxs
* The max number of context IDs supported on every CPU in this
* MMU context domain. This is needed here in case the system supports
* mixed type of processors/MMUs. It also helps to make ctx switch code
* access fewer cache lines i.e. no need to retrieve it from some global
* nctxs.
* mmu_lock
* The mutex spin lock used to serialize context ID wrap around
* mmu_idx
* The index for this MMU context domain structure in the global array
* mmu_ctxdoms.
* mmu_ncpus
* The actual number of CPUs that have been configured in this
* MMU context domain. This also acts as a reference count for the
* structure. When the last CPU in an MMU context domain is unconfigured,
* the structure is freed. It is protected by mmu_lock.
* mmu_cpuset
* The CPU set of configured CPUs for this MMU context domain. Used
* to cross-call all the CPUs in the MMU context domain to invalidate
* context IDs during a wraparound operation. It is protected by mmu_lock.
*/
typedef struct mmu_ctx {
uint64_t mmu_gnum;
uint_t mmu_cnum;
uint_t mmu_nctxs;
kmutex_t mmu_lock;
uint_t mmu_idx;
uint_t mmu_ncpus;
cpuset_t mmu_cpuset;
kstat_t *mmu_kstat;
kstat_named_t mmu_kstat_data[MMU_CTX_NUM_STATS];
} mmu_ctx_t;
#define mmu_tsb_exceptions \
mmu_kstat_data[MMU_CTX_TSB_EXCEPTIONS].value.ui64
#define mmu_tsb_raise_exception \
mmu_kstat_data[MMU_CTX_TSB_RAISE_EXCEPTION].value.ui64
#define mmu_wrap_around \
mmu_kstat_data[MMU_CTX_WRAP_AROUND].value.ui64
extern uint_t max_mmu_ctxdoms;
extern mmu_ctx_t **mmu_ctxs_tbl;
extern void sfmmu_cpu_init(cpu_t *);
extern void sfmmu_cpu_cleanup(cpu_t *);
extern uint_t sfmmu_ctxdom_nctxs(int);
#ifdef sun4v
extern void sfmmu_ctxdoms_remove(void);
extern void sfmmu_ctxdoms_lock(void);
extern void sfmmu_ctxdoms_unlock(void);
extern void sfmmu_ctxdoms_update(void);
#endif
/*
* The following structure is used to get MMU context domain information for
* a CPU from the platform.
*
* mmu_idx
* The MMU context domain index within the global array mmu_ctxs
* mmu_nctxs
* The number of context IDs supported in the MMU context domain
*/
typedef struct mmu_ctx_info {
uint_t mmu_idx;
uint_t mmu_nctxs;
} mmu_ctx_info_t;
#pragma weak plat_cpuid_to_mmu_ctx_info
extern void plat_cpuid_to_mmu_ctx_info(processorid_t, mmu_ctx_info_t *);
/*
* Each address space has an array of sfmmu_ctx_t structures, one structure
* per MMU context domain.
*
* cnum
* The context ID allocated for an address space on an MMU context domain
* gnum
* The generation number for the context ID in the MMU context domain.
*
* This structure needs to be a power-of-two in size.
*/
typedef struct sfmmu_ctx {
uint64_t gnum:48;
uint64_t cnum:16;
} sfmmu_ctx_t;
/*
* The platform dependent hat structure.
* tte counts should be protected by cas.
* cpuset is protected by cas.
*
* ttecnt accounting for mappings which do not use shared hme is carried out
* during pagefault handling. In the shared hme case, only the first process
* to access a mapping generates a pagefault, subsequent processes simply
* find the shared hme entry during trap handling and therefore there is no
* corresponding event to initiate ttecnt accounting. Currently, as shared
* hmes are only used for text segments, when joining a region we assume the
* worst case and add the the number of ttes required to map the entire region
* to the ttecnt corresponding to the region pagesize. However, if the region
* has a 4M pagesize, and memory is low, the allocation of 4M pages may fail
* then 8K pages will be allocated instead and the first TSB which stores 8K
* mappings will potentially be undersized. To compensate for the potential
* underaccounting in this case we always add 1/4 of the region size to the 8K
* ttecnt.
*/
struct hat {
cpuset_t sfmmu_cpusran; /* cpu bit mask for efficient xcalls */
struct as *sfmmu_as; /* as this hat provides mapping for */
/* per pgsz private ttecnt + shme rgns ttecnt for rgns not in SCD */
ulong_t sfmmu_ttecnt[MMU_PAGE_SIZES];
/* shme rgns ttecnt for rgns in SCD */
ulong_t sfmmu_scdrttecnt[MMU_PAGE_SIZES];
/* est. ism ttes that are NOT in a SCD */
ulong_t sfmmu_ismttecnt[MMU_PAGE_SIZES];
/* ttecnt for isms that are in a SCD */
ulong_t sfmmu_scdismttecnt[MMU_PAGE_SIZES];
/* inflate tsb0 to allow for large page alloc failure in region */
ulong_t sfmmu_tsb0_4minflcnt;
union _h_un {
ism_blk_t *sfmmu_iblkp; /* maps to ismhat(s) */
ism_ment_t *sfmmu_imentp; /* ism hat's mapping list */
} h_un;
uint_t sfmmu_free:1; /* hat to be freed - set on as_free */
uint_t sfmmu_ismhat:1; /* hat is dummy ism hatid */
uint_t sfmmu_scdhat:1; /* hat is dummy scd hatid */
uchar_t sfmmu_rmstat; /* refmod stats refcnt */
ushort_t sfmmu_clrstart; /* start color bin for page coloring */
ushort_t sfmmu_clrbin; /* per as phys page coloring bin */
ushort_t sfmmu_flags; /* flags */
uchar_t sfmmu_tteflags; /* pgsz flags */
uchar_t sfmmu_rtteflags; /* pgsz flags for SRD hmes */
struct tsb_info *sfmmu_tsb; /* list of per as tsbs */
uint64_t sfmmu_ismblkpa; /* pa of sfmmu_iblkp, or -1 */
lock_t sfmmu_ctx_lock; /* sync ctx alloc and invalidation */
kcondvar_t sfmmu_tsb_cv; /* signals TSB swapin or relocation */
uchar_t sfmmu_cext; /* context page size encoding */
uint8_t sfmmu_pgsz[MMU_PAGE_SIZES]; /* ranking for MMU */
sf_srd_t *sfmmu_srdp;
sf_scd_t *sfmmu_scdp; /* scd this address space belongs to */
sf_region_map_t sfmmu_region_map;
sf_rgn_link_t *sfmmu_hmeregion_links[SFMMU_L1_HMERLINKS];
sf_rgn_link_t sfmmu_scd_link; /* link to scd or pending queue */
#ifdef sun4v
struct hv_tsb_block sfmmu_hvblock;
#endif
/*
* sfmmu_ctxs is a variable length array of max_mmu_ctxdoms # of
* elements. max_mmu_ctxdoms is determined at run-time.
* sfmmu_ctxs[1] is just the fist element of an array, it always
* has to be the last field to ensure that the memory allocated
* for sfmmu_ctxs is consecutive with the memory of the rest of
* the hat data structure.
*/
sfmmu_ctx_t sfmmu_ctxs[1];
};
#define sfmmu_iblk h_un.sfmmu_iblkp
#define sfmmu_iment h_un.sfmmu_imentp
#define sfmmu_hmeregion_map sfmmu_region_map.h_rmap_s.hmeregion_map
#define sfmmu_ismregion_map sfmmu_region_map.h_rmap_s.ismregion_map
#define SF_RGNMAP_ISNULL(sfmmup) \
(sfrgnmap_isnull(&(sfmmup)->sfmmu_region_map))
#define SF_HMERGNMAP_ISNULL(sfmmup) \
(sfhmergnmap_isnull(&(sfmmup)->sfmmu_hmeregion_map))
struct sf_scd {
sfmmu_t *scd_sfmmup; /* shared context hat */
/* per pgsz ttecnt for shme rgns in SCD */
ulong_t scd_rttecnt[MMU_PAGE_SIZES];
uint_t scd_refcnt; /* address spaces attached to scd */
sf_region_map_t scd_region_map; /* bit mask of attached segments */
sf_scd_t *scd_next; /* link pointers for srd_scd list */
sf_scd_t *scd_prev;
sfmmu_t *scd_sf_list; /* list of doubly linked hat structs */
kmutex_t scd_mutex;
/*
* Link used to add an scd to the sfmmu_iment list.
*/
ism_ment_t scd_ism_links[SFMMU_MAX_ISM_REGIONS];
};
#define scd_hmeregion_map scd_region_map.h_rmap_s.hmeregion_map
#define scd_ismregion_map scd_region_map.h_rmap_s.ismregion_map
extern int disable_shctx;
extern int shctx_on;
/*
* bit mask for managing vac conflicts on large pages.
* bit 1 is for uncache flag.
* bits 2 through min(num of cache colors + 1,31) are
* for cache colors that have already been flushed.
*/
#ifdef VAC
#define CACHE_NUM_COLOR (shm_alignment >> MMU_PAGESHIFT)
#else
#define CACHE_NUM_COLOR 1
#endif
#define CACHE_VCOLOR_MASK(vcolor) (2 << (vcolor & (CACHE_NUM_COLOR - 1)))
#define CacheColor_IsFlushed(flag, vcolor) \
((flag) & CACHE_VCOLOR_MASK(vcolor))
#define CacheColor_SetFlushed(flag, vcolor) \
((flag) |= CACHE_VCOLOR_MASK(vcolor))
/*
* Flags passed to sfmmu_page_cache to flush page from vac or not.
*/
#define CACHE_FLUSH 0
#define CACHE_NO_FLUSH 1
/*
* Flags passed to sfmmu_tlbcache_demap
*/
#define FLUSH_NECESSARY_CPUS 0
#define FLUSH_ALL_CPUS 1
#ifdef DEBUG
/*
* For debugging purpose only. Maybe removed later.
*/
struct ctx_trace {
sfmmu_t *sc_sfmmu_stolen;
sfmmu_t *sc_sfmmu_stealing;
clock_t sc_time;
ushort_t sc_type;
ushort_t sc_cnum;
};
#define CTX_TRC_STEAL 0x1
#define CTX_TRC_FREE 0x0
#define TRSIZE 0x400
#define NEXT_CTXTR(ptr) (((ptr) >= ctx_trace_last) ? \
ctx_trace_first : ((ptr) + 1))
#define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) \
mutex_enter(mutex); \
(ptr)->sc_sfmmu_stolen = (stolen_sfmmu); \
(ptr)->sc_sfmmu_stealing = (stealing_sfmmu); \
(ptr)->sc_cnum = (cnum); \
(ptr)->sc_type = (type); \
(ptr)->sc_time = ddi_get_lbolt(); \
(ptr) = NEXT_CTXTR(ptr); \
num_ctx_stolen += (type); \
mutex_exit(mutex);
#else
#define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type)
#endif /* DEBUG */
#endif /* !_ASM */
/*
* Macros for sfmmup->sfmmu_flags access. The macros that change the flags
* ASSERT() that we're holding the HAT lock before changing the flags;
* however callers that read the flags may do so without acquiring the lock
* in a fast path, and then recheck the flag after acquiring the lock in
* a slow path.
*/
#define SFMMU_FLAGS_ISSET(sfmmup, flags) \
(((sfmmup)->sfmmu_flags & (flags)) == (flags))
#define SFMMU_FLAGS_CLEAR(sfmmup, flags) \
(ASSERT(sfmmu_hat_lock_held((sfmmup))), \
(sfmmup)->sfmmu_flags &= ~(flags))
#define SFMMU_FLAGS_SET(sfmmup, flags) \
(ASSERT(sfmmu_hat_lock_held((sfmmup))), \
(sfmmup)->sfmmu_flags |= (flags))
#define SFMMU_TTEFLAGS_ISSET(sfmmup, flags) \
((((sfmmup)->sfmmu_tteflags | (sfmmup)->sfmmu_rtteflags) & (flags)) == \
(flags))
/*
* sfmmu tte HAT flags, must fit in 8 bits
*/
#define HAT_CHKCTX1_FLAG 0x1
#define HAT_64K_FLAG (0x1 << TTE64K)
#define HAT_512K_FLAG (0x1 << TTE512K)
#define HAT_4M_FLAG (0x1 << TTE4M)
#define HAT_32M_FLAG (0x1 << TTE32M)
#define HAT_256M_FLAG (0x1 << TTE256M)
/*
* sfmmu HAT flags, 16 bits at the moment.
*/
#define HAT_4MTEXT_FLAG 0x01
#define HAT_32M_ISM 0x02
#define HAT_256M_ISM 0x04
#define HAT_SWAPPED 0x08 /* swapped out */
#define HAT_SWAPIN 0x10 /* swapping in */
#define HAT_BUSY 0x20 /* replacing TSB(s) */
#define HAT_ISMBUSY 0x40 /* adding/removing/traversing ISM maps */
#define HAT_CTX1_FLAG 0x100 /* ISM imap hatflag for ctx1 */
#define HAT_JOIN_SCD 0x200 /* region is joining scd */
#define HAT_ALLCTX_INVALID 0x400 /* all per-MMU ctxs are invalidated */
#define SFMMU_LGPGS_INUSE(sfmmup) \
(((sfmmup)->sfmmu_tteflags | (sfmmup)->sfmmu_rtteflags) || \
((sfmmup)->sfmmu_iblk != NULL))
/*
* Starting with context 0, the first NUM_LOCKED_CTXS contexts
* are locked so that sfmmu_getctx can't steal any of these
* contexts. At the time this software was being developed, the
* only context that needs to be locked is context 0 (the kernel
* context), and context 1 (reserved for stolen context). So this constant
* was originally defined to be 2.
*
* For sun4v only, USER_CONTEXT_TYPE represents any user context. Many
* routines only care whether the context is kernel, invalid or user.
*/
#define NUM_LOCKED_CTXS 2
#define INVALID_CONTEXT 1
#ifdef sun4v
#define USER_CONTEXT_TYPE NUM_LOCKED_CTXS
#endif
#if defined(sun4v) || defined(UTSB_PHYS)
/*
* Get the location in the 4MB base TSB of the tsbe for this fault.
* Assumes that the second TSB only contains 4M mappings.
*
* In:
* tagacc = tag access register (not clobbered)
* tsbe = 2nd TSB base register
* tmp1, tmp2 = scratch registers
* Out:
* tsbe = pointer to the tsbe in the 2nd TSB
*/
#define GET_4MBASE_TSBE_PTR(tagacc, tsbe, tmp1, tmp2) \
and tsbe, TSB_SOFTSZ_MASK, tmp2; /* tmp2=szc */ \
andn tsbe, TSB_SOFTSZ_MASK, tsbe; /* tsbbase */ \
mov TSB_ENTRIES(0), tmp1; /* nentries in TSB size 0 */ \
sllx tmp1, tmp2, tmp1; /* tmp1 = nentries in TSB */ \
sub tmp1, 1, tmp1; /* mask = nentries - 1 */ \
srlx tagacc, MMU_PAGESHIFT4M, tmp2; \
and tmp2, tmp1, tmp1; /* tsbent = virtpage & mask */ \
sllx tmp1, TSB_ENTRY_SHIFT, tmp1; /* entry num --> ptr */ \
add tsbe, tmp1, tsbe /* add entry offset to TSB base */
#define GET_2ND_TSBE_PTR(tagacc, tsbe, tmp1, tmp2) \
GET_4MBASE_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)
/*
* Get the location in the 3rd TSB of the tsbe for this fault.
* The 3rd TSB corresponds to the shared context, and is used
* for 8K - 512k pages.
*
* In:
* tagacc = tag access register (not clobbered)
* tsbe, tmp1, tmp2 = scratch registers
* Out:
* tsbe = pointer to the tsbe in the 3rd TSB
*/
#define GET_3RD_TSBE_PTR(tagacc, tsbe, tmp1, tmp2) \
and tsbe, TSB_SOFTSZ_MASK, tmp2; /* tmp2=szc */ \
andn tsbe, TSB_SOFTSZ_MASK, tsbe; /* tsbbase */ \
mov TSB_ENTRIES(0), tmp1; /* nentries in TSB size 0 */ \
sllx tmp1, tmp2, tmp1; /* tmp1 = nentries in TSB */ \
sub tmp1, 1, tmp1; /* mask = nentries - 1 */ \
srlx tagacc, MMU_PAGESHIFT, tmp2; \
and tmp2, tmp1, tmp1; /* tsbent = virtpage & mask */ \
sllx tmp1, TSB_ENTRY_SHIFT, tmp1; /* entry num --> ptr */ \
add tsbe, tmp1, tsbe /* add entry offset to TSB base */
#define GET_4TH_TSBE_PTR(tagacc, tsbe, tmp1, tmp2) \
GET_4MBASE_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)
/*
* Copy the sfmmu_region_map or scd_region_map to the tsbmiss
* shmermap or scd_shmermap, from sfmmu_load_mmustate.
*/
#define SET_REGION_MAP(rgn_map, tsbmiss_map, cnt, tmp, label) \
/* BEGIN CSTYLED */ \
label: ;\
ldx [rgn_map], tmp ;\
dec cnt ;\
add rgn_map, CLONGSIZE, rgn_map ;\
stx tmp, [tsbmiss_map] ;\
brnz,pt cnt, label ;\
add tsbmiss_map, CLONGSIZE, tsbmiss_map \
/* END CSTYLED */
/*
* If there is no scd, then zero the tsbmiss scd_shmermap,
* from sfmmu_load_mmustate.
*/
#define ZERO_REGION_MAP(tsbmiss_map, cnt, label) \
/* BEGIN CSTYLED */ \
label: ;\
dec cnt ;\
stx %g0, [tsbmiss_map] ;\
brnz,pt cnt, label ;\
add tsbmiss_map, CLONGSIZE, tsbmiss_map
/* END CSTYLED */
/*
* Set hmemisc to 1 if the shared hme is also part of an scd.
* In:
* tsbarea = tsbmiss area (not clobbered)
* hmeblkpa = hmeblkpa + hmentoff + SFHME_TTE (not clobbered)
* hmentoff = hmentoff + SFHME_TTE = tte offset(clobbered)
* Out:
* use_shctx = 1 if shme is in scd and 0 otherwise
*/
#define GET_SCDSHMERMAP(tsbarea, hmeblkpa, hmentoff, use_shctx) \
/* BEGIN CSTYLED */ \
sub hmeblkpa, hmentoff, hmentoff /* hmentofff = hmeblkpa */ ;\
add hmentoff, HMEBLK_TAG, hmentoff ;\
ldxa [hmentoff]ASI_MEM, hmentoff /* read 1st part of tag */ ;\
and hmentoff, HTAG_RID_MASK, hmentoff /* mask off rid */ ;\
and hmentoff, BT_ULMASK, use_shctx /* mask bit index */ ;\
srlx hmentoff, BT_ULSHIFT, hmentoff /* extract word */ ;\
sllx hmentoff, CLONGSHIFT, hmentoff /* index */ ;\
add tsbarea, hmentoff, hmentoff /* add to tsbarea */ ;\
ldx [hmentoff + TSBMISS_SCDSHMERMAP], hmentoff /* scdrgn */ ;\
srlx hmentoff, use_shctx, use_shctx ;\
and use_shctx, 0x1, use_shctx \
/* END CSTYLED */
/*
* Synthesize a TSB base register contents for a process.
*
* In:
* tsbinfo = TSB info pointer (ro)
* tsbreg, tmp1 = scratch registers
* Out:
* tsbreg = value to program into TSB base register
*/
#define MAKE_UTSBREG(tsbinfo, tsbreg, tmp1) \
ldx [tsbinfo + TSBINFO_PADDR], tsbreg; \
lduh [tsbinfo + TSBINFO_SZCODE], tmp1; \
and tmp1, TSB_SOFTSZ_MASK, tmp1; \
or tsbreg, tmp1, tsbreg;
/*
* Load TSB base register to TSBMISS area for privte contexts.
* This register contains utsb_pabase in bits 63:13, and TSB size
* code in bits 2:0.
*
* For private context
* In:
* tsbreg = value to load (ro)
* regnum = constant or register
* tmp1 = scratch register
* Out:
* Specified scratchpad register updated
*
*/
#define SET_UTSBREG(regnum, tsbreg, tmp1) \
mov regnum, tmp1; \
stxa tsbreg, [tmp1]ASI_SCRATCHPAD /* save tsbreg */
/*
* Get TSB base register from the scratchpad for private contexts
*
* In:
* regnum = constant or register
* tsbreg = scratch
* Out:
* tsbreg = tsbreg from the specified scratchpad register
*/
#define GET_UTSBREG(regnum, tsbreg) \
mov regnum, tsbreg; \
ldxa [tsbreg]ASI_SCRATCHPAD, tsbreg
/*
* Load TSB base register to TSBMISS area for shared contexts.
* This register contains utsb_pabase in bits 63:13, and TSB size
* code in bits 2:0.
*
* In:
* tsbmiss = pointer to tsbmiss area
* tsbmissoffset = offset to right tsb pointer
* tsbreg = value to load (ro)
* Out:
* Specified tsbmiss area updated
*
*/
#define SET_UTSBREG_SHCTX(tsbmiss, tsbmissoffset, tsbreg) \
stx tsbreg, [tsbmiss + tsbmissoffset] /* save tsbreg */
/*
* Get TSB base register from the scratchpad for
* shared contexts
*
* In:
* tsbmiss = pointer to tsbmiss area
* tsbmissoffset = offset to right tsb pointer
* tsbreg = scratch
* Out:
* tsbreg = tsbreg from the specified scratchpad register
*/
#define GET_UTSBREG_SHCTX(tsbmiss, tsbmissoffset, tsbreg) \
ldx [tsbmiss + tsbmissoffset], tsbreg
#endif /* defined(sun4v) || defined(UTSB_PHYS) */
#ifndef _ASM
/*
* Kernel page relocation stuff.
*/
struct sfmmu_callback {
int key;
int (*prehandler)(caddr_t, uint_t, uint_t, void *);
int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t);
int (*errhandler)(caddr_t, uint_t, uint_t, void *);
int capture_cpus;
};
extern int sfmmu_max_cb_id;
extern struct sfmmu_callback *sfmmu_cb_table;
struct pa_hment;
/*
* RFE: With multihat gone we gain back an int. We could use this to
* keep ref bits on a per cpu basis to eliminate xcalls.
*/
struct sf_hment {
tte_t hme_tte; /* tte for this hment */
union {
struct page *page; /* what page this maps */
struct pa_hment *data; /* pa_hment */
} sf_hment_un;
struct sf_hment *hme_next; /* next hment */
struct sf_hment *hme_prev; /* prev hment */
};
struct pa_hment {
caddr_t addr; /* va */
uint_t len; /* bytes */
ushort_t flags; /* internal flags */
ushort_t refcnt; /* reference count */
id_t cb_id; /* callback id, table index */
void *pvt; /* handler's private data */
struct sf_hment sfment; /* corresponding dummy sf_hment */
};
#define hme_page sf_hment_un.page
#define hme_data sf_hment_un.data
#define hme_size(sfhmep) ((int)(TTE_CSZ(&(sfhmep)->hme_tte)))
#define PAHME_SZ (sizeof (struct pa_hment))
#define SFHME_SZ (sizeof (struct sf_hment))
#define IS_PAHME(hme) ((hme)->hme_tte.ll == 0)
/*
* hmeblk_tag structure
* structure used to obtain a match on a hme_blk. Currently consists of
* the address of the sfmmu struct (or hatid), the base page address of the
* hme_blk, and the rehash count. The rehash count is actually only 2 bits
* and has the following meaning:
* 1 = 8k or 64k hash sequence.
* 2 = 512k hash sequence.
* 3 = 4M hash sequence.
* We require this count because we don't want to get a false hit on a 512K or
* 4M rehash with a base address corresponding to a 8k or 64k hmeblk.
* Note: The ordering and size of the hmeblk_tag members are implictly known
* by the tsb miss handlers written in assembly. Do not change this structure
* without checking those routines. See HTAG_SFMMUPSZ define.
*/
/*
* In private hmeblks hblk_rid field must be SFMMU_INVALID_RID.
*/
typedef union {
struct {
uint64_t hblk_basepg: 51, /* hme_blk base pg # */
hblk_rehash: 3, /* rehash number */
hblk_rid: 10; /* hme_blk region id */
void *hblk_id;
} hblk_tag_un;
uint64_t htag_tag[2];
} hmeblk_tag;
#define htag_id hblk_tag_un.hblk_id
#define htag_bspage hblk_tag_un.hblk_basepg
#define htag_rehash hblk_tag_un.hblk_rehash
#define htag_rid hblk_tag_un.hblk_rid
#endif /* !_ASM */
#define HTAG_REHASH_SHIFT 10
#define HTAG_MAX_RID (((0x1 << HTAG_REHASH_SHIFT) - 1))
#define HTAG_RID_MASK HTAG_MAX_RID
/* used for tagging all per sfmmu (i.e. non SRD) private hmeblks */
#define SFMMU_INVALID_SHMERID HTAG_MAX_RID
#if SFMMU_INVALID_SHMERID < SFMMU_MAX_HME_REGIONS
#error SFMMU_INVALID_SHMERID < SFMMU_MAX_HME_REGIONS
#endif
#define SFMMU_IS_SHMERID_VALID(rid) ((rid) != SFMMU_INVALID_SHMERID)
/* ISM regions */
#define SFMMU_INVALID_ISMRID 0xff
#if SFMMU_INVALID_ISMRID < SFMMU_MAX_ISM_REGIONS
#error SFMMU_INVALID_ISMRID < SFMMU_MAX_ISM_REGIONS
#endif
#define SFMMU_IS_ISMRID_VALID(rid) ((rid) != SFMMU_INVALID_ISMRID)
#define HTAGS_EQ(tag1, tag2) (((tag1.htag_tag[0] ^ tag2.htag_tag[0]) | \
(tag1.htag_tag[1] ^ tag2.htag_tag[1])) == 0)
/*
* this macro must only be used for comparing tags in shared hmeblks.
*/
#define HTAGS_EQ_SHME(hmetag, tag, hrmap) \
(((hmetag).htag_rid != SFMMU_INVALID_SHMERID) && \
(((((hmetag).htag_tag[0] ^ (tag).htag_tag[0]) & \
~HTAG_RID_MASK) | \
((hmetag).htag_tag[1] ^ (tag).htag_tag[1])) == 0) && \
SF_RGNMAP_TEST(hrmap, hmetag.htag_rid))
#define HME_REHASH(sfmmup) \
((sfmmup)->sfmmu_ttecnt[TTE512K] != 0 || \
(sfmmup)->sfmmu_ttecnt[TTE4M] != 0 || \
(sfmmup)->sfmmu_ttecnt[TTE32M] != 0 || \
(sfmmup)->sfmmu_ttecnt[TTE256M] != 0)
#define NHMENTS 8 /* # of hments in an 8k hme_blk */
/* needs to be multiple of 2 */
#ifndef _ASM
#ifdef HBLK_TRACE
#define HBLK_LOCK 1
#define HBLK_UNLOCK 0
#define HBLK_STACK_DEPTH 6
#define HBLK_AUDIT_CACHE_SIZE 16
#define HBLK_LOCK_PATTERN 0xaaaaaaaa
#define HBLK_UNLOCK_PATTERN 0xbbbbbbbb
struct hblk_lockcnt_audit {
int flag; /* lock or unlock */
kthread_id_t thread;
int depth;
pc_t stack[HBLK_STACK_DEPTH];
};
#endif /* HBLK_TRACE */
/*
* Hment block structure.
* The hme_blk is the node data structure which the hash structure
* mantains. An hme_blk can have 2 different sizes depending on the
* number of hments it implicitly contains. When dealing with 64K, 512K,
* or 4M hments there is one hment per hme_blk. When dealing with
* 8k hments we allocate an hme_blk plus an additional 7 hments to
* give us a total of 8 (NHMENTS) hments that can be referenced through a
* hme_blk.
*
* The hmeblk structure contains 2 tte reference counters used to determine if
* it is ok to free up the hmeblk. Both counters have to be zero in order
* to be able to free up hmeblk. They are protected by cas.
* hblk_hmecnt is the number of hments present on pp mapping lists.
* hblk_vcnt reflects number of valid ttes in hmeblk.
*
* The hmeblk now also has per tte lock cnts. This is required because
* the counts can be high and there are not enough bits in the tte. When
* physio is fixed to not lock the translations we should be able to move
* the lock cnt back to the tte. See bug id 1198554.
*/
struct hme_blk_misc {
uint_t notused:26;
uint_t shared_bit:1; /* set for SRD shared hmeblk */
uint_t shadow_bit:1; /* set for a shadow hme_blk */
uint_t nucleus_bit:1; /* set for a nucleus hme_blk */
uint_t ttesize:3; /* contains ttesz of hmeblk */
};
struct hme_blk {
volatile uint64_t hblk_nextpa; /* physical address for hash list */
hmeblk_tag hblk_tag; /* tag used to obtain an hmeblk match */
struct hme_blk *hblk_next; /* on free list or on hash list */
/* protected by hash lock */
struct hme_blk *hblk_shadow; /* pts to shadow hblk */
/* protected by hash lock */
uint_t hblk_span; /* span of memory hmeblk maps */
struct hme_blk_misc hblk_misc;
union {
struct {
ushort_t hblk_hmecount; /* hment on mlists counter */
ushort_t hblk_validcnt; /* valid tte reference count */
} hblk_counts;
uint_t hblk_shadow_mask;
} hblk_un;
uint_t hblk_lckcnt;
#ifdef HBLK_TRACE
kmutex_t hblk_audit_lock; /* lock to protect index */
uint_t hblk_audit_index; /* index into audit_cache */
struct hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE];
#endif /* HBLK_AUDIT */
struct sf_hment hblk_hme[1]; /* hment array */
};
#define hblk_shared hblk_misc.shared_bit
#define hblk_shw_bit hblk_misc.shadow_bit
#define hblk_nuc_bit hblk_misc.nucleus_bit
#define hblk_ttesz hblk_misc.ttesize
#define hblk_hmecnt hblk_un.hblk_counts.hblk_hmecount
#define hblk_vcnt hblk_un.hblk_counts.hblk_validcnt
#define hblk_shw_mask hblk_un.hblk_shadow_mask
#define MAX_HBLK_LCKCNT 0xFFFFFFFF
#define HMEBLK_ALIGN 0x8 /* hmeblk has to be double aligned */
#ifdef HBLK_TRACE
#define HBLK_STACK_TRACE(hmeblkp, lock) \
{ \
int flag = lock; /* to pacify lint */ \
int audit_index; \
\
mutex_enter(&hmeblkp->hblk_audit_lock); \
audit_index = hmeblkp->hblk_audit_index; \
hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) & \
(HBLK_AUDIT_CACHE_SIZE - 1)); \
mutex_exit(&hmeblkp->hblk_audit_lock); \
\
if (flag) \
hmeblkp->hblk_audit_cache[audit_index].flag = \
HBLK_LOCK_PATTERN; \
else \
hmeblkp->hblk_audit_cache[audit_index].flag = \
HBLK_UNLOCK_PATTERN; \
\
hmeblkp->hblk_audit_cache[audit_index].thread = curthread; \
hmeblkp->hblk_audit_cache[audit_index].depth = \
getpcstack(hmeblkp->hblk_audit_cache[audit_index].stack, \
HBLK_STACK_DEPTH); \
}
#else
#define HBLK_STACK_TRACE(hmeblkp, lock)
#endif /* HBLK_TRACE */
#define HMEHASH_FACTOR 16 /* used to calc # of buckets in hme hash */
/*
* A maximum number of user hmeblks is defined in order to place an upper
* limit on how much nucleus memory is required and to avoid overflowing the
* tsbmiss uhashsz and khashsz data areas. The number below corresponds to
* the number of buckets required, for an average hash chain length of 4 on
* a 16TB machine.
*/
#define MAX_UHME_BUCKETS (0x1 << 30)
#define MAX_KHME_BUCKETS (0x1 << 30)
/*
* The minimum number of kernel hash buckets.
*/
#define MIN_KHME_BUCKETS 0x800
/*
* The number of hash buckets must be a power of 2. If the initial calculated
* value is less than USER_BUCKETS_THRESHOLD we round up to the next greater
* power of 2, otherwise we round down to avoid huge over allocations.
*/
#define USER_BUCKETS_THRESHOLD (1<<22)
#define MAX_NUCUHME_BUCKETS 0x4000
#define MAX_NUCKHME_BUCKETS 0x2000
/*
* There are 2 locks in the hmehash bucket. The hmehash_mutex is
* a regular mutex used to make sure operations on a hash link are only
* done by one thread. Any operation which comes into the hat with
* a <vaddr, as> will grab the hmehash_mutex. Normally one would expect
* the tsb miss handlers to grab the hash lock to make sure the hash list
* is consistent while we traverse it. Unfortunately this can lead to
* deadlocks or recursive mutex enters since it is possible for
* someone holding the lock to take a tlb/tsb miss.
* To solve this problem we have added the hmehash_listlock. This lock
* is only grabbed by the tsb miss handlers, vatopfn, and while
* adding/removing a hmeblk from the hash list. The code is written to
* guarantee we won't take a tlb miss while holding this lock.
*/
struct hmehash_bucket {
kmutex_t hmehash_mutex;
volatile uint64_t hmeh_nextpa; /* physical address for hash list */
struct hme_blk *hmeblkp;
uint_t hmeh_listlock;
};
#endif /* !_ASM */
#define SFMMU_PGCNT_MASK 0x3f
#define SFMMU_PGCNT_SHIFT 6
#define INVALID_MMU_ID -1
#define SFMMU_MMU_GNUM_RSHIFT 16
#define SFMMU_MMU_CNUM_LSHIFT (64 - SFMMU_MMU_GNUM_RSHIFT)
#define MAX_SFMMU_CTX_VAL ((1 << 16) - 1) /* for sanity check */
#define MAX_SFMMU_GNUM_VAL ((0x1UL << 48) - 1)
/*
* The tsb miss handlers written in assembly know that sfmmup
* is a 64 bit ptr.
*
* The bspage and re-hash part is 64 bits, with the sfmmup being another 64
* bits.
*/
#define HTAG_SFMMUPSZ 0 /* Not really used for LP64 */
#define HTAG_BSPAGE_SHIFT 13
/*
* Assembly routines need to be able to get to ttesz
*/
#define HBLK_SZMASK 0x7
#ifndef _ASM
/*
* Returns the number of bytes that an hmeblk spans given its tte size
*/
#define get_hblk_span(hmeblkp) ((hmeblkp)->hblk_span)
#define get_hblk_ttesz(hmeblkp) ((hmeblkp)->hblk_ttesz)
#define get_hblk_cache(hmeblkp) (((hmeblkp)->hblk_ttesz == TTE8K) ? \
sfmmu8_cache : sfmmu1_cache)
#define HMEBLK_SPAN(ttesz) \
((ttesz == TTE8K)? (TTEBYTES(ttesz) * NHMENTS) : TTEBYTES(ttesz))
#define set_hblk_sz(hmeblkp, ttesz) \
(hmeblkp)->hblk_ttesz = (ttesz); \
(hmeblkp)->hblk_span = HMEBLK_SPAN(ttesz)
#define get_hblk_base(hmeblkp) \
((uintptr_t)(hmeblkp)->hblk_tag.htag_bspage << MMU_PAGESHIFT)
#define get_hblk_endaddr(hmeblkp) \
((caddr_t)(get_hblk_base(hmeblkp) + get_hblk_span(hmeblkp)))
#define in_hblk_range(hmeblkp, vaddr) \
(((uintptr_t)(vaddr) >= get_hblk_base(hmeblkp)) && \
((uintptr_t)(vaddr) < (get_hblk_base(hmeblkp) + \
get_hblk_span(hmeblkp))))
#define tte_to_vaddr(hmeblkp, tte) ((caddr_t)(get_hblk_base(hmeblkp) \
+ (TTEBYTES(TTE_CSZ(&tte)) * (tte).tte_hmenum)))
#define tte_to_evaddr(hmeblkp, ttep) ((caddr_t)(get_hblk_base(hmeblkp) \
+ (TTEBYTES(TTE_CSZ(ttep)) * ((ttep)->tte_hmenum + 1))))
#define vaddr_to_vshift(hblktag, vaddr, shwsz) \
((((uintptr_t)(vaddr) >> MMU_PAGESHIFT) - (hblktag.htag_bspage)) >>\
TTE_BSZS_SHIFT((shwsz) - 1))
#define HME8BLK_SZ (sizeof (struct hme_blk) + \
(NHMENTS - 1) * sizeof (struct sf_hment))
#define HME1BLK_SZ (sizeof (struct hme_blk))
#define H1MIN (2 + MAX_BIGKTSB_TTES) /* nucleus text+data, ktsb */
/*
* Hme_blk hash structure
* Active mappings are kept in a hash structure of hme_blks. The hash
* function is based on (ctx, vaddr) The size of the hash table size is a
* power of 2 such that the average hash chain lenth is HMENT_HASHAVELEN.
* The hash actually consists of 2 separate hashes. One hash is for the user
* address space and the other hash is for the kernel address space.
* The number of buckets are calculated at boot time and stored in the global
* variables "uhmehash_num" and "khmehash_num". By making the hash table size
* a power of 2 we can use a simply & function to derive an index instead of
* a divide.
*
* HME_HASH_FUNCTION(hatid, vaddr, shift) returns a pointer to a hme_hash
* bucket.
* An hme hash bucket contains a pointer to an hme_blk and the mutex that
* protects the link list.
* Spitfire supports 4 page sizes. 8k and 64K pages only need one hash.
* 512K pages need 2 hashes and 4M pages need 3 hashes.
* The 'shift' parameter controls how many bits the vaddr will be shifted in
* the hash function. It is calculated in the HME_HASH_SHIFT(ttesz) function
* and it varies depending on the page size as follows:
* 8k pages: HBLK_RANGE_SHIFT
* 64k pages: MMU_PAGESHIFT64K
* 512K pages: MMU_PAGESHIFT512K
* 4M pages: MMU_PAGESHIFT4M
* An assembly version of the hash function exists in sfmmu_ktsb_miss(). All
* changes should be reflected in both versions. This function and the TSB
* miss handlers are the only places which know about the two hashes.
*
* HBLK_RANGE_SHIFT controls range of virtual addresses that will fall
* into the same bucket for a particular process. It is currently set to
* be equivalent to 64K range or one hme_blk.
*
* The hme_blks in the hash are protected by a per hash bucket mutex
* known as SFMMU_HASH_LOCK.
* You need to acquire this lock before traversing the hash bucket link
* list, while adding/removing a hme_blk to the list, and while
* modifying an hme_blk. A possible optimization is to replace these
* mutexes by readers/writer lock but right now it is not clear whether
* this is a win or not.
*
* The HME_HASH_TABLE_SEARCH will search the hash table for the
* hme_blk that contains the hment that corresponds to the passed
* ctx and vaddr. It assumed the SFMMU_HASH_LOCK is held.
*/
#endif /* ! _ASM */
#define KHATID ksfmmup
#define UHMEHASH_SZ uhmehash_num
#define KHMEHASH_SZ khmehash_num
#define HMENT_HASHAVELEN 4
#define HBLK_RANGE_SHIFT MMU_PAGESHIFT64K /* shift for HBLK_BS_MASK */
#define HBLK_MIN_TTESZ 1
#define HBLK_MIN_BYTES MMU_PAGESIZE64K
#define HBLK_MIN_SHIFT MMU_PAGESHIFT64K
#define MAX_HASHCNT 5
#define DEFAULT_MAX_HASHCNT 3
#ifndef _ASM
#define HASHADDR_MASK(hashno) TTE_PAGEMASK(hashno)
#define HME_HASH_SHIFT(ttesz) \
((ttesz == TTE8K)? HBLK_RANGE_SHIFT : TTE_PAGE_SHIFT(ttesz))
#define HME_HASH_ADDR(vaddr, hmeshift) \
((caddr_t)(((uintptr_t)(vaddr) >> (hmeshift)) << (hmeshift)))
#define HME_HASH_BSPAGE(vaddr, hmeshift) \
(((uintptr_t)(vaddr) >> (hmeshift)) << ((hmeshift) - MMU_PAGESHIFT))
#define HME_HASH_REHASH(ttesz) \
(((ttesz) < TTE512K)? 1 : (ttesz))
#define HME_HASH_FUNCTION(hatid, vaddr, shift) \
((((void *)hatid) != ((void *)KHATID)) ? \
(&uhme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \
UHMEHASH_SZ) ]): \
(&khme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \
KHMEHASH_SZ) ]))
/*
* This macro will traverse a hmeblk hash link list looking for an hme_blk
* that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp
* will be set to NULL, otherwise it will point to the correct hme_blk.
* This macro also cleans empty hblks.
*/
#define HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, pr_hblk, listp) \
{ \
struct hme_blk *nx_hblk; \
\
ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \
hblkp = hmebp->hmeblkp; \
pr_hblk = NULL; \
while (hblkp) { \
if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \
/* found hme_blk */ \
break; \
} \
nx_hblk = hblkp->hblk_next; \
if (!hblkp->hblk_vcnt && !hblkp->hblk_hmecnt) { \
sfmmu_hblk_hash_rm(hmebp, hblkp, pr_hblk, \
listp, 0); \
} else { \
pr_hblk = hblkp; \
} \
hblkp = nx_hblk; \
} \
}
#define HME_HASH_SEARCH(hmebp, hblktag, hblkp, listp) \
{ \
struct hme_blk *pr_hblk; \
\
HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, pr_hblk, listp); \
}
/*
* This macro will traverse a hmeblk hash link list looking for an hme_blk
* that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp
* will be set to NULL, otherwise it will point to the correct hme_blk.
* It doesn't remove empty hblks.
*/
#define HME_HASH_FAST_SEARCH(hmebp, hblktag, hblkp) \
ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \
for (hblkp = hmebp->hmeblkp; hblkp; \
hblkp = hblkp->hblk_next) { \
if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \
/* found hme_blk */ \
break; \
} \
}
#define SFMMU_HASH_LOCK(hmebp) \
(mutex_enter(&hmebp->hmehash_mutex))
#define SFMMU_HASH_UNLOCK(hmebp) \
(mutex_exit(&hmebp->hmehash_mutex))
#define SFMMU_HASH_LOCK_TRYENTER(hmebp) \
(mutex_tryenter(&hmebp->hmehash_mutex))
#define SFMMU_HASH_LOCK_ISHELD(hmebp) \
(mutex_owned(&hmebp->hmehash_mutex))
#define SFMMU_XCALL_STATS(sfmmup) \
{ \
if (sfmmup == ksfmmup) { \
SFMMU_STAT(sf_kernel_xcalls); \
} else { \
SFMMU_STAT(sf_user_xcalls); \
} \
}
#define astosfmmu(as) ((as)->a_hat)
#define hblktosfmmu(hmeblkp) ((sfmmu_t *)(hmeblkp)->hblk_tag.htag_id)
#define hblktosrd(hmeblkp) ((sf_srd_t *)(hmeblkp)->hblk_tag.htag_id)
#define sfmmutoas(sfmmup) ((sfmmup)->sfmmu_as)
#define sfmmutohtagid(sfmmup, rid) \
(((rid) == SFMMU_INVALID_SHMERID) ? (void *)(sfmmup) : \
(void *)((sfmmup)->sfmmu_srdp))
/*
* We use the sfmmu data structure to keep the per as page coloring info.
*/
#define as_color_bin(as) (astosfmmu(as)->sfmmu_clrbin)
#define as_color_start(as) (astosfmmu(as)->sfmmu_clrstart)
typedef struct {
char h8[HME8BLK_SZ];
} hblk8_t;
typedef struct {
char h1[HME1BLK_SZ];
} hblk1_t;
typedef struct {
ulong_t index;
ulong_t len;
hblk8_t *list;
} nucleus_hblk8_info_t;
typedef struct {
ulong_t index;
ulong_t len;
hblk1_t *list;
} nucleus_hblk1_info_t;
/*
* This struct is used for accumlating information about a range
* of pages that are unloading so that a single xcall can flush
* the entire range from remote tlbs. A function that must demap
* a range of virtual addresses declares one of these structures
* and initializes using DEMP_RANGE_INIT(). It then passes a pointer to this
* struct to the appropriate sfmmu_hblk_* level function which does
* all the bookkeeping using the other macros. When the function has
* finished the virtual address range, it needs to call DEMAP_RANGE_FLUSH()
* macro to take care of any remaining unflushed mappings.
*
* The maximum range this struct can represent is the number of bits
* in the dmr_bitvec field times the pagesize in dmr_pgsz. Currently, only
* MMU_PAGESIZE pages are supported.
*
* Since there are now cases where it's no longer necessary to do
* flushes (e.g. when the process isn't runnable because it's swapping
* out or exiting) we allow these macros to take a NULL dmr input and do
* nothing in that case.
*/
typedef struct {
sfmmu_t *dmr_sfmmup; /* relevant hat */
caddr_t dmr_addr; /* beginning address */
caddr_t dmr_endaddr; /* ending address */
ulong_t dmr_bitvec; /* valid pages found */
ulong_t dmr_bit; /* next page to examine */
ulong_t dmr_maxbit; /* highest page in range */
ulong_t dmr_pgsz; /* page size in range */
} demap_range_t;
#define DMR_MAXBIT ((ulong_t)1<<63) /* dmr_bit high bit */
#define DEMAP_RANGE_INIT(sfmmup, dmrp) \
(dmrp)->dmr_sfmmup = (sfmmup); \
(dmrp)->dmr_bitvec = 0; \
(dmrp)->dmr_maxbit = sfmmu_dmr_maxbit; \
(dmrp)->dmr_pgsz = MMU_PAGESIZE;
#define DEMAP_RANGE_PGSZ(dmrp) ((dmrp)? (dmrp)->dmr_pgsz : MMU_PAGESIZE)
#define DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr) \
if ((dmrp) != NULL) { \
if ((dmrp)->dmr_bitvec != 0 && (dmrp)->dmr_endaddr != (addr)) \
sfmmu_tlb_range_demap(dmrp); \
(dmrp)->dmr_endaddr = (endaddr); \
}
#define DEMAP_RANGE_FLUSH(dmrp) \
if ((dmrp)->dmr_bitvec != 0) \
sfmmu_tlb_range_demap(dmrp);
#define DEMAP_RANGE_MARKPG(dmrp, addr) \
if ((dmrp) != NULL) { \
if ((dmrp)->dmr_bitvec == 0) { \
(dmrp)->dmr_addr = (addr); \
(dmrp)->dmr_bit = 1; \
} \
(dmrp)->dmr_bitvec |= (dmrp)->dmr_bit; \
}
#define DEMAP_RANGE_NEXTPG(dmrp) \
if ((dmrp) != NULL && (dmrp)->dmr_bitvec != 0) { \
if ((dmrp)->dmr_bit & (dmrp)->dmr_maxbit) { \
sfmmu_tlb_range_demap(dmrp); \
} else { \
(dmrp)->dmr_bit <<= 1; \
} \
}
/*
* TSB related structures
*
* The TSB is made up of tte entries. Both the tag and data are present
* in the TSB. The TSB locking is managed as follows:
* A software bit in the tsb tag is used to indicate that entry is locked.
* If a cpu servicing a tsb miss reads a locked entry the tag compare will
* fail forcing the cpu to go to the hat hash for the translation.
* The cpu who holds the lock can then modify the data side, and the tag side.
* The last write should be to the word containing the lock bit which will
* clear the lock and allow the tsb entry to be read. It is assumed that all
* cpus reading the tsb will do so with atomic 128-bit loads. An atomic 128
* bit load is required to prevent the following from happening:
*
* cpu 0 cpu 1 comments
*
* ldx tag tag unlocked
* ldstub lock set lock
* stx data
* stx tag unlock
* ldx tag incorrect tte!!!
*
* The software also maintains a bit in the tag to indicate an invalid
* tsb entry. The purpose of this bit is to allow the tsb invalidate code
* to invalidate a tsb entry with a single cas. See code for details.
*/
union tsb_tag {
struct {
uint32_t tag_res0:16; /* reserved - context area */
uint32_t tag_inv:1; /* sw - invalid tsb entry */
uint32_t tag_lock:1; /* sw - locked tsb entry */
uint32_t tag_res1:4; /* reserved */
uint32_t tag_va_hi:10; /* va[63:54] */
uint32_t tag_va_lo; /* va[53:22] */
} tagbits;
struct tsb_tagints {
uint32_t inthi;
uint32_t intlo;
} tagints;
};
#define tag_invalid tagbits.tag_inv
#define tag_locked tagbits.tag_lock
#define tag_vahi tagbits.tag_va_hi
#define tag_valo tagbits.tag_va_lo
#define tag_inthi tagints.inthi
#define tag_intlo tagints.intlo
struct tsbe {
union tsb_tag tte_tag;
tte_t tte_data;
};
/*
* A per cpu struct is kept that duplicates some info
* used by the tl>0 tsb miss handlers plus it provides
* a scratch area. Its purpose is to minimize cache misses
* in the tsb miss handler and is 128 bytes (2 e$ lines).
*
* There should be one allocated per cpu in nucleus memory
* and should be aligned on an ecache line boundary.
*/
struct tsbmiss {
sfmmu_t *ksfmmup; /* kernel hat id */
sfmmu_t *usfmmup; /* user hat id */
sf_srd_t *usrdp; /* user's SRD hat id */
struct tsbe *tsbptr; /* hardware computed ptr */
struct tsbe *tsbptr4m; /* hardware computed ptr */
struct tsbe *tsbscdptr; /* hardware computed ptr */
struct tsbe *tsbscdptr4m; /* hardware computed ptr */
uint64_t ismblkpa;
struct hmehash_bucket *khashstart;
struct hmehash_bucket *uhashstart;
uint_t khashsz;
uint_t uhashsz;
uint16_t dcache_line_mask; /* used to flush dcache */
uchar_t uhat_tteflags; /* private page sizes */
uchar_t uhat_rtteflags; /* SHME pagesizes */
uint32_t utsb_misses;
uint32_t ktsb_misses;
uint16_t uprot_traps;
uint16_t kprot_traps;
/*
* scratch[0] -> TSB_TAGACC
* scratch[1] -> TSBMISS_HMEBP
* scratch[2] -> TSBMISS_HATID
*/
uintptr_t scratch[3];
ulong_t shmermap[SFMMU_HMERGNMAP_WORDS]; /* 8 bytes */
ulong_t scd_shmermap[SFMMU_HMERGNMAP_WORDS]; /* 8 bytes */
uint8_t pad[48]; /* pad to 64 bytes */
};
/*
* A per cpu struct is kept for the use within the tl>0 kpm tsb
* miss handler. Some members are duplicates of common data or
* the physical addresses of common data. A few members are also
* written by the tl>0 kpm tsb miss handler. Its purpose is to
* minimize cache misses in the kpm tsb miss handler and occupies
* one ecache line. There should be one allocated per cpu in
* nucleus memory and it should be aligned on an ecache line
* boundary. It is not merged w/ struct tsbmiss since there is
* not much to share and the tsbmiss pathes are different, so
* a kpm tlbmiss/tsbmiss only touches one cacheline, except for
* (DEBUG || SFMMU_STAT_GATHER) where the dtlb_misses counter
* of struct tsbmiss is used on every dtlb miss.
*/
struct kpmtsbm {
caddr_t vbase; /* start of address kpm range */
caddr_t vend; /* end of address kpm range */
uchar_t flags; /* flags needed in TL tsbmiss handler */
uchar_t sz_shift; /* for single kpm window */
uchar_t kpmp_shift; /* hash lock shift */
uchar_t kpmp2pshft; /* kpm page to page shift */
uint_t kpmp_table_sz; /* size of kpmp_table or kpmp_stable */
uint64_t kpmp_tablepa; /* paddr of kpmp_table or kpmp_stable */
uint64_t msegphashpa; /* paddr of memseg_phash */
struct tsbe *tsbptr; /* saved ktsb pointer */
uint_t kpm_dtlb_misses; /* kpm tlbmiss counter */
uint_t kpm_tsb_misses; /* kpm tsbmiss counter */
uintptr_t pad[1];
};
extern size_t tsb_slab_size;
extern uint_t tsb_slab_shift;
extern size_t tsb_slab_mask;
#endif /* !_ASM */
/*
* Flags for TL kpm tsbmiss handler
*/
#define KPMTSBM_ENABLE_FLAG 0x01 /* bit copy of kpm_enable */
#define KPMTSBM_TLTSBM_FLAG 0x02 /* use TL tsbmiss handler */
#define KPMTSBM_TSBPHYS_FLAG 0x04 /* use ASI_MEM for TSB update */
/*
* The TSB
* All TSB sizes supported by the hardware are now supported (8K - 1M).
* For kernel TSBs we may go beyond the hardware supported sizes and support
* larger TSBs via software.
* All TTE sizes are supported in the TSB; the manner in which this is
* done is cpu dependent.
*/
#define TSB_MIN_SZCODE TSB_8K_SZCODE /* min. supported TSB size */
#define TSB_MIN_OFFSET_MASK (TSB_OFFSET_MASK(TSB_MIN_SZCODE))
#ifdef sun4v
#define UTSB_MAX_SZCODE TSB_256M_SZCODE /* max. supported TSB size */
#else /* sun4u */
#define UTSB_MAX_SZCODE TSB_1M_SZCODE /* max. supported TSB size */
#endif /* sun4v */
#define UTSB_MAX_OFFSET_MASK (TSB_OFFSET_MASK(UTSB_MAX_SZCODE))
#define TSB_FREEMEM_MIN 0x1000 /* 32 mb */
#define TSB_FREEMEM_LARGE 0x10000 /* 512 mb */
#define TSB_8K_SZCODE 0 /* 512 entries */
#define TSB_16K_SZCODE 1 /* 1k entries */
#define TSB_32K_SZCODE 2 /* 2k entries */
#define TSB_64K_SZCODE 3 /* 4k entries */
#define TSB_128K_SZCODE 4 /* 8k entries */
#define TSB_256K_SZCODE 5 /* 16k entries */
#define TSB_512K_SZCODE 6 /* 32k entries */
#define TSB_1M_SZCODE 7 /* 64k entries */
#define TSB_2M_SZCODE 8 /* 128k entries */
#define TSB_4M_SZCODE 9 /* 256k entries */
#define TSB_8M_SZCODE 10 /* 512k entries */
#define TSB_16M_SZCODE 11 /* 1M entries */
#define TSB_32M_SZCODE 12 /* 2M entries */
#define TSB_64M_SZCODE 13 /* 4M entries */
#define TSB_128M_SZCODE 14 /* 8M entries */
#define TSB_256M_SZCODE 15 /* 16M entries */
#define TSB_ENTRY_SHIFT 4 /* each entry = 128 bits = 16 bytes */
#define TSB_ENTRY_SIZE (1 << 4)
#define TSB_START_SIZE 9
#define TSB_ENTRIES(tsbsz) (1 << (TSB_START_SIZE + tsbsz))
#define TSB_BYTES(tsbsz) (TSB_ENTRIES(tsbsz) << TSB_ENTRY_SHIFT)
#define TSB_OFFSET_MASK(tsbsz) (TSB_ENTRIES(tsbsz) - 1)
#define TSB_BASEADDR_MASK ((1 << 12) - 1)
/*
* sun4u platforms
* ---------------
* We now support two user TSBs with one TSB base register.
* Hence the TSB base register is split up as follows:
*
* When only one TSB present:
* [63 62..42 41..13 12..4 3..0]
* ^ ^ ^ ^ ^
* | | | | |
* | | | | |_ TSB size code
* | | | |
* | | | |_ Reserved 0
* | | |
* | | |_ TSB VA[41..13]
* | |
* | |_ VA hole (Spitfire), zeros (Cheetah and beyond)
* |
* |_ 0
*
* When second TSB present:
* [63 62..42 41..33 32..29 28..22 21..13 12..4 3..0]
* ^ ^ ^ ^ ^ ^ ^ ^
* | | | | | | | |
* | | | | | | | |_ First TSB size code
* | | | | | | |
* | | | | | | |_ Reserved 0
* | | | | | |
* | | | | | |_ First TSB's VA[21..13]
* | | | | |
* | | | | |_ Reserved for future use
* | | | |
* | | | |_ Second TSB's size code
* | | |
* | | |_ Second TSB's VA[21..13]
* | |
* | |_ VA hole (Spitfire) / ones (Cheetah and beyond)
* |
* |_ 1
*
* Note that since we store 21..13 of each TSB's VA, TSBs and their slabs
* may be up to 4M in size. For now, only hardware supported TSB sizes
* are supported, though the slabs are usually 4M in size.
*
* sun4u platforms that define UTSB_PHYS use physical addressing to access
* the user TSBs at TL>0. The first user TSB base is in the MMU I/D TSB Base
* registers. The second TSB base uses a dedicated scratchpad register which
* requires a definition of SCRATCHPAD_UTSBREG2 in mach_sfmmu.h. The layout for
* both registers is equivalent to sun4v below, except the TSB PA range is
* [46..13] for sun4u.
*
* sun4v platforms
* ---------------
* On sun4v platforms, we use two dedicated scratchpad registers as pseudo
* hardware TSB base registers to hold up to two different user TSBs.
*
* Each register contains TSB's physical base and size code information
* as follows:
*
* [63..56 55..13 12..4 3..0]
* ^ ^ ^ ^
* | | | |
* | | | |_ TSB size code
* | | |
* | | |_ Reserved 0
* | |
* | |_ TSB PA[55..13]
* |
* |
* |
* |_ 0 for valid TSB
*
* Absence of a user TSB (primarily the second user TSB) is indicated by
* storing a negative value in the TSB base register. This allows us to
* check for presence of a user TSB by simply checking bit# 63.
*/
#define TSBREG_MSB_SHIFT 32 /* set upper bits */
#define TSBREG_MSB_CONST 0xfffff800 /* set bits 63..43 */
#define TSBREG_FIRTSB_SHIFT 42 /* to clear bits 63:22 */
#define TSBREG_SECTSB_MKSHIFT 20 /* 21:13 --> 41:33 */
#define TSBREG_SECTSB_LSHIFT 22 /* to clear bits 63:42 */
#define TSBREG_SECTSB_RSHIFT (TSBREG_SECTSB_MKSHIFT + TSBREG_SECTSB_LSHIFT)
/* sectsb va -> bits 21:13 */
/* after clearing upper bits */
#define TSBREG_SECSZ_SHIFT 29 /* to get sectsb szc to 3:0 */
#define TSBREG_VAMASK_SHIFT 13 /* set up VA mask */
#define BIGKTSB_SZ_MASK 0xf
#define TSB_SOFTSZ_MASK BIGKTSB_SZ_MASK
#define MIN_BIGKTSB_SZCODE 9 /* 256k entries */
#define MAX_BIGKTSB_SZCODE 11 /* 1024k entries */
#define MAX_BIGKTSB_TTES (TSB_BYTES(MAX_BIGKTSB_SZCODE) / MMU_PAGESIZE4M)
#define TAG_VALO_SHIFT 22 /* tag's va are bits 63-22 */
/*
* sw bits used on tsb_tag - bit masks used only in assembly
* use only a sethi for these fields.
*/
#define TSBTAG_INVALID 0x00008000 /* tsb_tag.tag_invalid */
#define TSBTAG_LOCKED 0x00004000 /* tsb_tag.tag_locked */
#ifdef _ASM
/*
* Marker to indicate that this instruction will be hot patched at runtime
* to some other value.
* This value must be zero since it fills in the imm bits of the target
* instructions to be patched
*/
#define RUNTIME_PATCH (0)
/*
* V9 defines nop instruction as the following, which we use
* at runtime to nullify some instructions we don't want to
* execute in the trap handlers on certain platforms.
*/
#define MAKE_NOP_INSTR(reg) \
sethi %hi(0x1000000), reg
/*
* This macro constructs a SPARC V9 "jmpl <source reg>, %g0"
* instruction, with the source register specified by the jump_reg_number.
* The jmp opcode [24:19] = 11 1000 and source register is bits [18:14].
* The instruction is returned in reg. The macro is used to patch in a jmpl
* instruction at runtime.
*/
#define MAKE_JMP_INSTR(jump_reg_number, reg, tmp) \
sethi %hi(0x81c00000), reg; \
mov jump_reg_number, tmp; \
sll tmp, 14, tmp; \
or reg, tmp, reg
/*
* Macro to get hat per-MMU cnum on this CPU.
* sfmmu - In, pass in "sfmmup" from the caller.
* cnum - Out, return 'cnum' to the caller
* scr - scratch
*/
#define SFMMU_CPU_CNUM(sfmmu, cnum, scr) \
CPU_ADDR(scr, cnum); /* scr = load CPU struct addr */ \
ld [scr + CPU_MMU_IDX], cnum; /* cnum = mmuid */ \
add sfmmu, SFMMU_CTXS, scr; /* scr = sfmmup->sfmmu_ctxs[] */ \
sllx cnum, SFMMU_MMU_CTX_SHIFT, cnum; \
add scr, cnum, scr; /* scr = sfmmup->sfmmu_ctxs[id] */ \
ldx [scr + SFMMU_MMU_GC_NUM], scr; /* sfmmu_ctxs[id].gcnum */ \
sllx scr, SFMMU_MMU_CNUM_LSHIFT, scr; \
srlx scr, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */
/*
* Macro to get hat gnum & cnum assocaited with sfmmu_ctx[mmuid] entry
* entry - In, pass in (&sfmmu_ctxs[mmuid] - SFMMU_CTXS) from the caller.
* gnum - Out, return sfmmu gnum
* cnum - Out, return sfmmu cnum
* reg - scratch
*/
#define SFMMU_MMUID_GNUM_CNUM(entry, gnum, cnum, reg) \
ldx [entry + SFMMU_CTXS], reg; /* reg = sfmmu (gnum | cnum) */ \
srlx reg, SFMMU_MMU_GNUM_RSHIFT, gnum; /* gnum = sfmmu gnum */ \
sllx reg, SFMMU_MMU_CNUM_LSHIFT, cnum; \
srlx cnum, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */
/*
* Macro to get this CPU's tsbmiss area.
*/
#define CPU_TSBMISS_AREA(tsbmiss, tmp1) \
CPU_INDEX(tmp1, tsbmiss); /* tmp1 = cpu idx */ \
sethi %hi(tsbmiss_area), tsbmiss; /* tsbmiss base ptr */ \
mulx tmp1, TSBMISS_SIZE, tmp1; /* byte offset */ \
or tsbmiss, %lo(tsbmiss_area), tsbmiss; \
add tsbmiss, tmp1, tsbmiss /* tsbmiss area of CPU */
/*
* Macro to set kernel context + page size codes in DMMU primary context
* register. It is only necessary for sun4u because sun4v does not need
* page size codes
*/
#ifdef sun4v
#define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3)
#else
#define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) \
sethi %hi(kcontextreg), reg0; \
ldx [reg0 + %lo(kcontextreg)], reg0; \
mov MMU_PCONTEXT, reg1; \
ldxa [reg1]ASI_MMU_CTX, reg2; \
xor reg0, reg2, reg2; \
brz reg2, label3; \
srlx reg2, CTXREG_NEXT_SHIFT, reg2; \
rdpr %pstate, reg3; /* disable interrupts */ \
btst PSTATE_IE, reg3; \
/*CSTYLED*/ \
bnz,a,pt %icc, label1; \
wrpr reg3, PSTATE_IE, %pstate; \
/*CSTYLED*/ \
label1:; \
brz reg2, label2; /* need demap if N_pgsz0/1 change */ \
sethi %hi(FLUSH_ADDR), reg4; \
mov DEMAP_ALL_TYPE, reg2; \
stxa %g0, [reg2]ASI_DTLB_DEMAP; \
stxa %g0, [reg2]ASI_ITLB_DEMAP; \
/*CSTYLED*/ \
label2:; \
stxa reg0, [reg1]ASI_MMU_CTX; \
flush reg4; \
btst PSTATE_IE, reg3; \
/*CSTYLED*/ \
bnz,a,pt %icc, label3; \
wrpr %g0, reg3, %pstate; /* restore interrupt state */ \
label3:;
#endif
/*
* Macro to setup arguments with kernel sfmmup context + page size before
* calling sfmmu_setctx_sec()
*/
#ifdef sun4v
#define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \
set KCONTEXT, arg0; \
set 0, arg1;
#else
#define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \
ldub [sfmmup + SFMMU_CEXT], arg1; \
set KCONTEXT, arg0; \
sll arg1, CTXREG_EXT_SHIFT, arg1;
#endif
#define PANIC_IF_INTR_DISABLED_PSTR(pstatereg, label, scr) \
andcc pstatereg, PSTATE_IE, %g0; /* panic if intrs */ \
/*CSTYLED*/ \
bnz,pt %icc, label; /* already disabled */ \
nop; \
\
sethi %hi(panicstr), scr; \
ldx [scr + %lo(panicstr)], scr; \
tst scr; \
/*CSTYLED*/ \
bnz,pt %xcc, label; \
nop; \
\
save %sp, -SA(MINFRAME), %sp; \
sethi %hi(sfmmu_panic1), %o0; \
call panic; \
or %o0, %lo(sfmmu_panic1), %o0; \
/*CSTYLED*/ \
label:
#define PANIC_IF_INTR_ENABLED_PSTR(label, scr) \
/* \
* The caller must have disabled interrupts. \
* If interrupts are not disabled, panic \
*/ \
rdpr %pstate, scr; \
andcc scr, PSTATE_IE, %g0; \
/*CSTYLED*/ \
bz,pt %icc, label; \
nop; \
\
sethi %hi(panicstr), scr; \
ldx [scr + %lo(panicstr)], scr; \
tst scr; \
/*CSTYLED*/ \
bnz,pt %xcc, label; \
nop; \
\
sethi %hi(sfmmu_panic6), %o0; \
call panic; \
or %o0, %lo(sfmmu_panic6), %o0; \
/*CSTYLED*/ \
label:
#endif /* _ASM */
#ifndef _ASM
#ifdef VAC
/*
* Page coloring
* The p_vcolor field of the page struct (1 byte) is used to store the
* virtual page color. This provides for 255 colors. The value zero is
* used to mean the page has no color - never been mapped or somehow
* purified.
*/
#define PP_GET_VCOLOR(pp) (((pp)->p_vcolor) - 1)
#define PP_NEWPAGE(pp) (!(pp)->p_vcolor)
#define PP_SET_VCOLOR(pp, color) \
((pp)->p_vcolor = ((color) + 1))
/*
* As mentioned p_vcolor == 0 means there is no color for this page.
* But PP_SET_VCOLOR(pp, color) expects 'color' to be real color minus
* one so we define this constant.
*/
#define NO_VCOLOR (-1)
#define addr_to_vcolor(addr) \
(((uint_t)(uintptr_t)(addr) >> MMU_PAGESHIFT) & vac_colors_mask)
#else /* VAC */
#define addr_to_vcolor(addr) (0)
#endif /* VAC */
/*
* The field p_index in the psm page structure is for large pages support.
* P_index is a bit-vector of the different mapping sizes that a given page
* is part of. An hme structure for a large mapping is only added in the
* group leader page (first page). All pages covered by a given large mapping
* have the corrosponding mapping bit set in their p_index field. This allows
* us to only store an explicit hme structure in the leading page which
* simplifies the mapping link list management. Furthermore, it provides us
* a fast mechanism for determining the largest mapping a page is part of. For
* exmaple, a page with a 64K and a 4M mappings has a p_index value of 0x0A.
*
* Implementation note: even though the first bit in p_index is reserved
* for 8K mappings, it is NOT USED by the code and SHOULD NOT be set.
* In addition, the upper four bits of the p_index field are used by the
* code as temporaries
*/
/*
* Defines for psm page struct fields and large page support
*/
#define SFMMU_INDEX_SHIFT 6
#define SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1)
/* Return the mapping index */
#define PP_MAPINDEX(pp) ((pp)->p_index & SFMMU_INDEX_MASK)
/*
* These macros rely on the following property:
* All pages constituting a large page are covered by a virtually
* contiguous set of page_t's.
*/
/* Return the leader for this mapping size */
#define PP_GROUPLEADER(pp, sz) \
(&(pp)[-(int)(pp->p_pagenum & (TTEPAGES(sz)-1))])
/* Return the root page for this page based on p_szc */
#define PP_PAGEROOT(pp) ((pp)->p_szc == 0 ? (pp) : \
PP_GROUPLEADER((pp), (pp)->p_szc))
#define PP_PAGENEXT_N(pp, n) ((pp) + (n))
#define PP_PAGENEXT(pp) PP_PAGENEXT_N((pp), 1)
#define PP_PAGEPREV_N(pp, n) ((pp) - (n))
#define PP_PAGEPREV(pp) PP_PAGEPREV_N((pp), 1)
#define PP_ISMAPPED_LARGE(pp) (PP_MAPINDEX(pp) != 0)
/* Need function to test the page mappping which takes p_index into account */
#define PP_ISMAPPED(pp) ((pp)->p_mapping || PP_ISMAPPED_LARGE(pp))
/*
* Don't call this macro with sz equal to zero. 8K mappings SHOULD NOT
* set p_index field.
*/
#define PAGESZ_TO_INDEX(sz) (1 << (sz))
/*
* prototypes for hat assembly routines. Some of these are
* known to machine dependent VM code.
*/
extern uint64_t sfmmu_make_tsbtag(caddr_t);
extern struct tsbe *
sfmmu_get_tsbe(uint64_t, caddr_t, int, int);
extern void sfmmu_load_tsbe(struct tsbe *, uint64_t, tte_t *, int);
extern void sfmmu_unload_tsbe(struct tsbe *, uint64_t, int);
extern void sfmmu_load_mmustate(sfmmu_t *);
extern void sfmmu_raise_tsb_exception(uint64_t, uint64_t);
#ifndef sun4v
extern void sfmmu_itlb_ld_kva(caddr_t, tte_t *);
extern void sfmmu_dtlb_ld_kva(caddr_t, tte_t *);
#endif /* sun4v */
extern void sfmmu_copytte(tte_t *, tte_t *);
extern int sfmmu_modifytte(tte_t *, tte_t *, tte_t *);
extern int sfmmu_modifytte_try(tte_t *, tte_t *, tte_t *);
extern pfn_t sfmmu_ttetopfn(tte_t *, caddr_t);
extern uint_t sfmmu_disable_intrs(void);
extern void sfmmu_enable_intrs(uint_t);
/*
* functions exported to machine dependent VM code
*/
extern void sfmmu_patch_ktsb(void);
#ifndef UTSB_PHYS
extern void sfmmu_patch_utsb(void);
#endif /* UTSB_PHYS */
extern pfn_t sfmmu_vatopfn(caddr_t, sfmmu_t *, tte_t *);
extern void sfmmu_vatopfn_suspended(caddr_t, sfmmu_t *, tte_t *);
extern pfn_t sfmmu_kvaszc2pfn(caddr_t, int);
#ifdef DEBUG
extern void sfmmu_check_kpfn(pfn_t);
#else
#define sfmmu_check_kpfn(pfn) /* disabled */
#endif /* DEBUG */
extern void sfmmu_memtte(tte_t *, pfn_t, uint_t, int);
extern void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, uint_t);
extern void sfmmu_tsbmiss_exception(struct regs *, uintptr_t, uint_t);
extern void sfmmu_init_tsbs(void);
extern caddr_t sfmmu_ktsb_alloc(caddr_t);
extern int sfmmu_getctx_pri(void);
extern int sfmmu_getctx_sec(void);
extern void sfmmu_setctx_sec(uint_t);
extern void sfmmu_inv_tsb(caddr_t, uint_t);
extern void sfmmu_init_ktsbinfo(void);
extern int sfmmu_setup_4lp(void);
extern void sfmmu_patch_mmu_asi(int);
extern void sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int);
extern void sfmmu_cache_flushall(void);
extern pgcnt_t sfmmu_tte_cnt(sfmmu_t *, uint_t);
extern void *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int);
extern void sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t);
extern void sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *);
extern void hat_kern_setup(void);
extern int hat_page_relocate(page_t **, page_t **, spgcnt_t *);
extern int sfmmu_get_ppvcolor(struct page *);
extern int sfmmu_get_addrvcolor(caddr_t);
extern int sfmmu_hat_lock_held(sfmmu_t *);
extern int sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *, int);
extern kmutex_t *sfmmu_mlist_enter(page_t *);
extern void sfmmu_mlist_exit(kmutex_t *);
extern int sfmmu_mlist_held(struct page *);
extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *);
/*
* MMU-specific functions optionally imported from the CPU module
*/
#pragma weak mmu_init_scd
#pragma weak mmu_large_pages_disabled
#pragma weak mmu_set_ctx_page_sizes
#pragma weak mmu_check_page_sizes
extern void mmu_init_scd(sf_scd_t *);
extern uint_t mmu_large_pages_disabled(uint_t);
extern void mmu_set_ctx_page_sizes(sfmmu_t *);
extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *);
extern sfmmu_t *ksfmmup;
extern caddr_t ktsb_base;
extern uint64_t ktsb_pbase;
extern int ktsb_sz;
extern int ktsb_szcode;
extern caddr_t ktsb4m_base;
extern uint64_t ktsb4m_pbase;
extern int ktsb4m_sz;
extern int ktsb4m_szcode;
extern uint64_t kpm_tsbbase;
extern int kpm_tsbsz;
extern int ktsb_phys;
extern int enable_bigktsb;
#ifndef sun4v
extern int utsb_dtlb_ttenum;
extern int utsb4m_dtlb_ttenum;
#endif /* sun4v */
extern int uhmehash_num;
extern int khmehash_num;
extern struct hmehash_bucket *uhme_hash;
extern struct hmehash_bucket *khme_hash;
extern uint_t hblk_alloc_dynamic;
extern struct tsbmiss tsbmiss_area[NCPU];
extern struct kpmtsbm kpmtsbm_area[NCPU];
#ifndef sun4v
extern int dtlb_resv_ttenum;
extern caddr_t utsb_vabase;
extern caddr_t utsb4m_vabase;
#endif /* sun4v */
extern vmem_t *kmem_tsb_default_arena[];
extern int tsb_lgrp_affinity;
extern uint_t disable_large_pages;
extern uint_t disable_ism_large_pages;
extern uint_t disable_auto_data_large_pages;
extern uint_t disable_auto_text_large_pages;
/* kpm externals */
extern pfn_t sfmmu_kpm_vatopfn(caddr_t);
extern void sfmmu_kpm_patch_tlbm(void);
extern void sfmmu_kpm_patch_tsbm(void);
extern void sfmmu_patch_shctx(void);
extern void sfmmu_kpm_load_tsb(caddr_t, tte_t *, int);
extern void sfmmu_kpm_unload_tsb(caddr_t, int);
extern void sfmmu_kpm_tsbmtl(short *, uint_t *, int);
extern int sfmmu_kpm_stsbmtl(uchar_t *, uint_t *, int);
extern caddr_t kpm_vbase;
extern size_t kpm_size;
extern struct memseg *memseg_hash[];
extern uint64_t memseg_phash[];
extern kpm_hlk_t *kpmp_table;
extern kpm_shlk_t *kpmp_stable;
extern uint_t kpmp_table_sz;
extern uint_t kpmp_stable_sz;
extern uchar_t kpmp_shift;
#define PP_ISMAPPED_KPM(pp) ((pp)->p_kpmref > 0)
#define IS_KPM_ALIAS_RANGE(vaddr) \
(((vaddr) - kpm_vbase) >> (uintptr_t)kpm_size_shift > 0)
#endif /* !_ASM */
/* sfmmu_kpm_tsbmtl flags */
#define KPMTSBM_STOP 0
#define KPMTSBM_START 1
/*
* For kpm_smallpages, the state about how a kpm page is mapped and whether
* it is ready to go is indicated by the two 4-bit fields defined in the
* kpm_spage structure as follows:
* kp_mapped_flag bit[0:3] - the page is mapped cacheable or not
* kp_mapped_flag bit[4:7] - the mapping is ready to go or not
* If the bit KPM_MAPPED_GO is on, it indicates that the assembly tsb miss
* handler can drop the mapping in regardless of the caching state of the
* mapping. Otherwise, we will have C handler resolve the VAC conflict no
* matter the page is currently mapped cacheable or non-cacheable.
*/
#define KPM_MAPPEDS 0x1 /* small mapping valid, no conflict */
#define KPM_MAPPEDSC 0x2 /* small mapping valid, conflict */
#define KPM_MAPPED_GO 0x10 /* the mapping is ready to go */
#define KPM_MAPPED_MASK 0xf
/* Physical memseg address NULL marker */
#define MSEG_NULLPTR_PA -1
/*
* Memseg hash defines for kpm trap level tsbmiss handler.
* Must be in sync w/ page.h .
*/
#define SFMMU_MEM_HASH_SHIFT 0x9
#define SFMMU_N_MEM_SLOTS 0x200
#define SFMMU_MEM_HASH_ENTRY_SHIFT 3
#ifndef _ASM
#if (SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT)
#error SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT
#endif
#if (SFMMU_N_MEM_SLOTS != N_MEM_SLOTS)
#error SFMMU_N_MEM_SLOTS != N_MEM_SLOTS
#endif
/* Physical memseg address NULL marker */
#define SFMMU_MEMSEG_NULLPTR_PA -1
/*
* Check KCONTEXT to be zero, asm parts depend on that assumption.
*/
#if (KCONTEXT != 0)
#error KCONTEXT != 0
#endif
#endif /* !_ASM */
#endif /* _KERNEL */
#ifndef _ASM
/*
* ctx, hmeblk, mlistlock and other stats for sfmmu
*/
struct sfmmu_global_stat {
int sf_tsb_exceptions; /* # of tsb exceptions */
int sf_tsb_raise_exception; /* # tsb exc. w/o TLB flush */
int sf_pagefaults; /* # of pagefaults */
int sf_uhash_searches; /* # of user hash searches */
int sf_uhash_links; /* # of user hash links */
int sf_khash_searches; /* # of kernel hash searches */
int sf_khash_links; /* # of kernel hash links */
int sf_swapout; /* # times hat swapped out */
int sf_tsb_alloc; /* # TSB allocations */
int sf_tsb_allocfail; /* # times TSB alloc fail */
int sf_tsb_sectsb_create; /* # times second TSB added */
int sf_scd_1sttsb_alloc; /* # SCD 1st TSB allocations */
int sf_scd_2ndtsb_alloc; /* # SCD 2nd TSB allocations */
int sf_scd_1sttsb_allocfail; /* # SCD 1st TSB alloc fail */
int sf_scd_2ndtsb_allocfail; /* # SCD 2nd TSB alloc fail */
int sf_tteload8k; /* calls to sfmmu_tteload */
int sf_tteload64k; /* calls to sfmmu_tteload */
int sf_tteload512k; /* calls to sfmmu_tteload */
int sf_tteload4m; /* calls to sfmmu_tteload */
int sf_tteload32m; /* calls to sfmmu_tteload */
int sf_tteload256m; /* calls to sfmmu_tteload */
int sf_tsb_load8k; /* # times loaded 8K tsbent */
int sf_tsb_load4m; /* # times loaded 4M tsbent */
int sf_hblk_hit; /* found hblk during tteload */
int sf_hblk8_ncreate; /* static hblk8's created */
int sf_hblk8_nalloc; /* static hblk8's allocated */
int sf_hblk1_ncreate; /* static hblk1's created */
int sf_hblk1_nalloc; /* static hblk1's allocated */
int sf_hblk_slab_cnt; /* sfmmu8_cache slab creates */
int sf_hblk_reserve_cnt; /* hblk_reserve usage */
int sf_hblk_recurse_cnt; /* hblk_reserve owner reqs */
int sf_hblk_reserve_hit; /* hblk_reserve hash hits */
int sf_get_free_success; /* reserve list allocs */
int sf_get_free_throttle; /* fails due to throttling */
int sf_get_free_fail; /* fails due to empty list */
int sf_put_free_success; /* reserve list frees */
int sf_put_free_fail; /* fails due to full list */
int sf_pgcolor_conflict; /* VAC conflict resolution */
int sf_uncache_conflict; /* VAC conflict resolution */
int sf_unload_conflict; /* VAC unload resolution */
int sf_ism_uncache; /* VAC conflict resolution */
int sf_ism_recache; /* VAC conflict resolution */
int sf_recache; /* VAC conflict resolution */
int sf_steal_count; /* # of hblks stolen */
int sf_pagesync; /* # of pagesyncs */
int sf_clrwrt; /* # of clear write perms */
int sf_pagesync_invalid; /* pagesync with inv tte */
int sf_kernel_xcalls; /* # of kernel cross calls */
int sf_user_xcalls; /* # of user cross calls */
int sf_tsb_grow; /* # of user tsb grows */
int sf_tsb_shrink; /* # of user tsb shrinks */
int sf_tsb_resize_failures; /* # of user tsb resize */
int sf_tsb_reloc; /* # of user tsb relocations */
int sf_user_vtop; /* # of user vatopfn calls */
int sf_ctx_inv; /* #times invalidate MMU ctx */
int sf_tlb_reprog_pgsz; /* # times switch TLB pgsz */
int sf_region_remap_demap; /* # times shme remap demap */
int sf_create_scd; /* # times SCD is created */
int sf_join_scd; /* # process joined scd */
int sf_leave_scd; /* # process left scd */
int sf_destroy_scd; /* # times SCD is destroyed */
};
struct sfmmu_tsbsize_stat {
int sf_tsbsz_8k;
int sf_tsbsz_16k;
int sf_tsbsz_32k;
int sf_tsbsz_64k;
int sf_tsbsz_128k;
int sf_tsbsz_256k;
int sf_tsbsz_512k;
int sf_tsbsz_1m;
int sf_tsbsz_2m;
int sf_tsbsz_4m;
int sf_tsbsz_8m;
int sf_tsbsz_16m;
int sf_tsbsz_32m;
int sf_tsbsz_64m;
int sf_tsbsz_128m;
int sf_tsbsz_256m;
};
struct sfmmu_percpu_stat {
int sf_itlb_misses; /* # of itlb misses */
int sf_dtlb_misses; /* # of dtlb misses */
int sf_utsb_misses; /* # of user tsb misses */
int sf_ktsb_misses; /* # of kernel tsb misses */
int sf_tsb_hits; /* # of tsb hits */
int sf_umod_faults; /* # of mod (prot viol) flts */
int sf_kmod_faults; /* # of mod (prot viol) flts */
};
#define SFMMU_STAT(stat) sfmmu_global_stat.stat++
#define SFMMU_STAT_ADD(stat, amount) sfmmu_global_stat.stat += (amount)
#define SFMMU_STAT_SET(stat, count) sfmmu_global_stat.stat = (count)
#define SFMMU_MMU_STAT(stat) { \
mmu_ctx_t *ctx = CPU->cpu_m.cpu_mmu_ctxp; \
if (ctx) \
ctx->stat++; \
}
#endif /* !_ASM */
#ifdef __cplusplus
}
#endif
#endif /* _VM_HAT_SFMMU_H */