sfmmu_asm.s revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* SFMMU primitives. These primitives should only be used by sfmmu
* routines.
*/
#if defined(lint)
#else /* lint */
#include "assym.h"
#endif /* lint */
#include <sys/asm_linkage.h>
#include <sys/machtrap.h>
#include <vm/hat_sfmmu.h>
#include <sys/machparam.h>
#include <sys/privregs.h>
#include <sys/machthread.h>
#include <sys/trapstat.h>
#ifdef TRAPTRACE
#include <sys/traptrace.h>
/*
* Tracing macro. Adds two instructions if TRAPTRACE is defined.
*/
#else
#endif /* TRAPTRACE */
#ifndef lint
#if (TTE_SUSPEND_SHIFT > 0)
#define TTE_SUSPEND_INT_SHIFT(reg) \
#else
#define TTE_SUSPEND_INT_SHIFT(reg)
#endif
#endif /* lint */
#ifndef lint
/*
* Assumes TSBE_TAG is 0
* Assumes TSBE_INTHI is 0
* Assumes TSBREG.split is 0
*/
#if TSBE_TAG != 0
#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
#endif
#if TSBTAG_INTHI != 0
#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
#endif
/*
* The following code assumes the tsb is not split.
*
* With TSBs no longer shared between processes, it's no longer
* necessary to hash the context bits into the tsb index to get
* tsb coloring; the new implementation treats the TSB as a
* direct-mapped, virtually-addressed cache.
*
* In:
* vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
* tsbbase = base address of TSB (clobbered)
* tagacc = tag access register (clobbered)
* szc = size code of TSB (ro)
* tmp = scratch reg
* Out:
* tsbbase = pointer to entry in TSB
*/
/*
* When the kpm TSB is used it is assumed that it is direct mapped
* using (vaddr>>vpshift)%tsbsz as the index.
*
* Note that, for now, the kpm TSB and kernel TSB are the same for
* each mapping size. However that need not always be the case. If
* the trap handlers are updated to search a different TSB for kpm
* addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
* (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
*
* In:
* vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
* vaddr = virtual address (clobbered)
* tsbp, szc, tmp = scratch
* Out:
* tsbp = pointer to entry in TSB
*/
nop ;\
/*
* Lock the TSBE at virtual address tsbep.
*
* tsbep = TSBE va (ro)
* tmp1, tmp2 = scratch registers (clobbered)
* label = label to use for branches (text)
* %asi = ASI to use for TSB access
*
* NOTE that we flush the TSB using fast VIS instructions that
* set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
* not be treated as a locked entry or we'll get stuck spinning on
* an entry that isn't locked but really invalid.
*/
#if defined(UTSB_PHYS)
label: ;\
/* tsbe lock acquired */ ;\
#else /* UTSB_PHYS */
label: ;\
/* tsbe lock acquired */ ;\
#endif /* UTSB_PHYS */
/*
* Atomically write TSBE at virtual address tsbep.
*
* tsbep = TSBE va (ro)
* tte = TSBE TTE (ro)
* tagtarget = TSBE tag (ro)
* %asi = ASI to use for TSB access
*/
#if defined(UTSB_PHYS)
membar #StoreStore ;\
#else /* UTSB_PHYS */
membar #StoreStore ;\
#endif /* UTSB_PHYS */
/*
* Load an entry into the TSB at TL > 0.
*
* tsbep = pointer to the TSBE to load as va (ro)
* tte = value of the TTE retrieved and loaded (wo)
* tagtarget = tag target register. To get TSBE tag to load,
* we need to mask off the context and leave only the va (clobbered)
* tmp1, tmp2 = scratch registers
* label = label to use for branches (text)
* %asi = ASI to use for TSB access
*/
#if defined(UTSB_PHYS)
/* ;\
* I don't need to update the TSB then check for the valid tte. ;\
* TSB invalidate will spin till the entry is unlocked. Note, ;\
* we always invalidate the hash table before we unload the TSB.;\
*/ ;\
#else /* UTSB_PHYS */
/* ;\
* I don't need to update the TSB then check for the valid tte. ;\
* TSB invalidate will spin till the entry is unlocked. Note, ;\
* we always invalidate the hash table before we unload the TSB.;\
*/ ;\
#endif /* UTSB_PHYS */
/*
* Load an entry into the TSB at TL=0.
*
* tsbep = pointer to the TSBE to load as va (ro)
* tteva = pointer to the TTE to load as va (ro)
* tagtarget = TSBE tag to load (which contains no context), synthesized
* to match va of MMU tag target register only (ro)
* tmp1, tmp2 = scratch registers (clobbered)
* label = label to use for branches (text)
* %asi = ASI to use for TSB access
*/
#if defined(UTSB_PHYS)
/* can't rd tteva after locking tsb because it can tlb miss */ ;\
#else /* UTSB_PHYS */
/* can't rd tteva after locking tsb because it can tlb miss */ ;\
#endif /* UTSB_PHYS */
/*
* Invalidate a TSB entry in the TSB.
*
* NOTE: TSBE_TAG is assumed to be zero. There is a compile time check
* about this earlier to ensure this is true. Thus when we are
* directly referencing tsbep below, we are referencing the tte_tag
* field of the TSBE. If this offset ever changes, the code below
* will need to be modified.
*
* tsbep = pointer to TSBE as va (ro)
* tag = invalidation is done if this matches the TSBE tag (ro)
* tmp1 - tmp3 = scratch registers (clobbered)
* label = label name to use for branches (text)
* %asi = ASI to use for TSB access
*/
#if defined(UTSB_PHYS)
#else /* UTSB_PHYS */
#endif /* UTSB_PHYS */
#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
#endif
/*
* An implementation of setx which will be hot patched at run time.
* since it is being hot patched, there is no value passed in.
* Thus, essentially we are implementing
* setx value, tmp, dest
* where value is RUNTIME_PATCH (aka 0) in this case.
*/
nop /* for perf reasons */ ;\
#endif (lint)
#if defined (lint)
/*
* sfmmu related subroutines
*/
/*
* Use cas, if tte has changed underneath us then reread and try again.
* In the case of a retry, it will update sttep with the new original.
*/
/* ARGSUSED */
int
{ return(0); }
/*
* Use cas, if tte has changed underneath us then return 1, else return 0
*/
/* ARGSUSED */
int
{ return(0); }
/* ARGSUSED */
void
{}
/*ARGSUSED*/
struct tsbe *
{ return(0); }
/*ARGSUSED*/
{ return(0); }
#else /* lint */
.seg ".data"
.asciz "sfmmu_asm: interrupts already disabled"
.asciz "sfmmu_asm: sfmmu_vatopfn called for user"
.asciz "sfmmu_asm: 4M tsb pointer mis-match"
.asciz "sfmmu_asm: no unlocked TTEs in TLB 0"
2:
1: retl
1:
/*
* Calculate a TSB entry pointer for the given TSB, va, pagesize.
* %o0 = TSB base address (in), pointer to TSB entry (out)
* %o1 = vaddr (in)
* %o2 = vpshift (in)
* %o3 = tsb size code (in)
* %o4 = scratch register
*/
/*
* Return a TSB tag for the given va.
* %o0 = va shifted to be in tsb tag format (with no context) (out)
*/
#endif /* lint */
/*
* Other sfmmu primitives
*/
#if defined (lint)
void
sfmmu_patch_ktsb(void)
{
}
void
sfmmu_kpm_patch_tlbm(void)
{
}
void
sfmmu_kpm_patch_tsbm(void)
{
}
/* ARGSUSED */
void
{
}
/* ARGSUSED */
void
{
}
/* ARGSUSED */
void
{
}
/* ARGSUSED */
void
{
}
#else /* lint */
#define I_SIZE 4
/*
* %o0 = start of patch area
* %o1 = size code of TSB to patch
* %o3 = scratch
*/
/* fix sll */
/* fix srl */
/*
* %o0 = start of patch area
* %o5 = kernel virtual or physical tsb base address
* %o2, %o3 are used as scratch registers.
*/
/* fixup sethi instruction */
/*
* %o0 = start of patch area
* %o4 = 64 bit value to patch
* %o2, %o3 are used as scratch registers.
*
* Note: Assuming that all parts of the instructions which need to be
* patched correspond to RUNTIME_PATCH (aka 0)
*
* Note the implementation of setx which is being patched is as follows:
*
* sethi %hh(RUNTIME_PATCH), tmp
* sethi %lm(RUNTIME_PATCH), dest
* or tmp, %hm(RUNTIME_PATCH), tmp
* or dest, %lo(RUNTIME_PATCH), dest
* sllx tmp, 32, tmp
* nop
* or tmp, dest, dest
*
* which differs from the implementation in the
* "SPARC Architecture Manual"
*/
/* fixup sethi instruction */
/* fixup sethi instruction */
/* fixup or instruction */
/* fixup or instruction */
/*
* %o0 = start of patch area
* %o4 = 32 bit value to patch
* %o2, %o3 are used as scratch registers.
* Note: Assuming that all parts of the instructions which need to be
* patched correspond to RUNTIME_PATCH (aka 0)
*/
/*
* %o0 = start of patch area
* %o2, %o3 are used as scratch registers.
*
* so we do a simple add. The caller must be careful to prevent
* overflow, which could easily occur if the initial value is nonzero!
*/
/*
* Patch imm_asi of all ldda instructions in the MMU
* trap handlers. We search MMU_PATCH_INSTR instructions
* starting from the itlb miss handler (trap 0x64).
* %o0 = address of tt[0,1]_itlbmiss
* %o1 = imm_asi to setup, shifted by appropriate offset.
* %o3 = number of instructions to search
* %o4 = reserved by caller: called from leaf routine
*/
add %o0, I_SIZE, %o0
retl
flush %o0
SET_SIZE(sfmmu_fixup_mmu_asi)
/*
* Patch immediate ASI used to access the TSB in the
* trap table.
* inputs: %o0 = value of ktsb_phys
*/
ENTRY_NP(sfmmu_patch_mmu_asi)
mov %o7, %o4 ! save return pc in %o4
movrnz %o0, ASI_QUAD_LDD_PHYS, %o3
movrz %o0, ASI_NQUAD_LD, %o3
sll %o3, 5, %o1 ! imm_asi offset
mov 6, %o3 ! number of instructions
sethi %hi(dktsb), %o0 ! to search
call sfmmu_fixup_mmu_asi ! patch kdtlb miss
or %o0, %lo(dktsb), %o0
mov 6, %o3 ! number of instructions
sethi %hi(dktsb4m), %o0 ! to search
call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss
or %o0, %lo(dktsb4m), %o0
mov 6, %o3 ! number of instructions
sethi %hi(iktsb), %o0 ! to search
call sfmmu_fixup_mmu_asi ! patch kitlb miss
or %o0, %lo(iktsb), %o0
mov %o4, %o7 ! retore return pc -- leaf
retl
nop
SET_SIZE(sfmmu_patch_mmu_asi)
ENTRY_NP(sfmmu_patch_ktsb)
/*
* We need to fix iktsb, dktsb, et. al.
*/
save %sp, -SA(MINFRAME), %sp
set ktsb_phys, %o1
ld [%o1], %o4
set ktsb_base, %o5
set ktsb4m_base, %l1
brz,pt %o4, 1f
nop
set ktsb_pbase, %o5
set ktsb4m_pbase, %l1
1:
sethi %hi(ktsb_szcode), %o1
ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */
sethi %hi(iktsb), %o0
call sfmmu_fix_ktlb_traptable
or %o0, %lo(iktsb), %o0
sethi %hi(dktsb), %o0
call sfmmu_fix_ktlb_traptable
or %o0, %lo(dktsb), %o0
sethi %hi(ktsb4m_szcode), %o1
ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */
sethi %hi(dktsb4m), %o0
call sfmmu_fix_ktlb_traptable
or %o0, %lo(dktsb4m), %o0
#ifndef sun4v
mov ASI_N, %o2
movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch
mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4
sethi %hi(tsb_kernel_patch_asi), %o0
call sfmmu_fixup_or
or %o0, %lo(tsb_kernel_patch_asi), %o0
#endif
ldx [%o5], %o4 ! load ktsb base addr (VA or PA)
sethi %hi(dktsbbase), %o0
call sfmmu_fixup_setx ! patch value of ktsb base addr
or %o0, %lo(dktsbbase), %o0
sethi %hi(iktsbbase), %o0
call sfmmu_fixup_setx ! patch value of ktsb base addr
or %o0, %lo(iktsbbase), %o0
sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0
call sfmmu_fixup_setx ! patch value of ktsb base addr
or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
#ifdef sun4v
sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0
call sfmmu_fixup_setx ! patch value of ktsb base addr
or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
#endif /* sun4v */
ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA)
sethi %hi(dktsb4mbase), %o0
call sfmmu_fixup_setx ! patch value of ktsb4m base addr
or %o0, %lo(dktsb4mbase), %o0
sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0
call sfmmu_fixup_setx ! patch value of ktsb4m base addr
or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
#ifdef sun4v
sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0
call sfmmu_fixup_setx ! patch value of ktsb4m base addr
or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
#endif /* sun4v */
set ktsb_szcode, %o4
ld [%o4], %o4
sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0
call sfmmu_fixup_or ! patch value of ktsb_szcode
or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
#ifdef sun4v
sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0
call sfmmu_fixup_or ! patch value of ktsb_szcode
or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
#endif /* sun4v */
set ktsb4m_szcode, %o4
ld [%o4], %o4
sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
call sfmmu_fixup_or ! patch value of ktsb4m_szcode
or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
#ifdef sun4v
sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
call sfmmu_fixup_or ! patch value of ktsb4m_szcode
or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
#endif /* sun4v */
ret
restore
SET_SIZE(sfmmu_patch_ktsb)
ENTRY_NP(sfmmu_kpm_patch_tlbm)
/*
* Fixup trap handlers in common segkpm case. This is reserved
* for future use should kpm TSB be changed to be other than the
* kernel TSB.
*/
retl
nop
SET_SIZE(sfmmu_kpm_patch_tlbm)
ENTRY_NP(sfmmu_kpm_patch_tsbm)
/*
* nop the branch to sfmmu_kpm_dtsb_miss_small
* in the case where we are using large pages for
* seg_kpm (and hence must probe the second TSB for
* seg_kpm VAs)
*/
set dktsb4m_kpmcheck_small, %o0
MAKE_NOP_INSTR(%o1)
st %o1, [%o0]
flush %o0
retl
nop
SET_SIZE(sfmmu_kpm_patch_tsbm)
ENTRY_NP(sfmmu_patch_utsb)
#ifdef sun4v
retl
nop
#else /* sun4v */
/*
* We need to hot patch utsb_vabase and utsb4m_vabase
*/
save %sp, -SA(MINFRAME), %sp
/* patch value of utsb_vabase */
set utsb_vabase, %o1
ldx [%o1], %o4
sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
call sfmmu_fixup_setx
or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
call sfmmu_fixup_setx
or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
call sfmmu_fixup_setx
or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
/* patch value of utsb4m_vabase */
set utsb4m_vabase, %o1
ldx [%o1], %o4
sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0
call sfmmu_fixup_setx
or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
call sfmmu_fixup_setx
or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
call sfmmu_fixup_setx
or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
/*
* Patch TSB base register masks and shifts if needed.
* By default the TSB base register contents are set up for 4M slab.
*/
/* delay slot safe */
/* patch reserved VA range size if needed. */
1:
/* patch TSBREG_VAMASK used to set up TSB base register */
#endif /* sun4v */
/*
* Routine that loads an entry into a tsb using virtual addresses.
* Locking is required since all cpus can use the same TSB.
* Note that it is no longer required to have a valid context
* when calling this function.
*/
/*
* %o0 = pointer to tsbe to load
* %o1 = tsb tag
* %o2 = virtual pointer to TTE
* %o3 = 1 if physical address in %o0 else 0
*/
#ifdef DEBUG
1:
#endif /* DEBUG */
/*
* Flush TSB of a given entry if the tag matches.
*/
/*
* %o0 = pointer to tsbe to be flushed
* %o1 = tag to match
* %o2 = 1 if physical address in %o0 else 0
*/
/*
* Routine that loads a TTE into the kpm TSB from C code.
* Locking is required since kpm TSB is shared among all CPUs.
*/
/*
* %o0 = vaddr
* %o1 = ttep
* %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
*/
#ifdef DEBUG
1:
#endif /* DEBUG */
#ifndef sun4v
#endif
/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
/* %g2 = tsbep, %g1 clobbered */
/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
/*
* Routine that shoots down a TTE in the kpm TSB or in the
* kernel TSB depending on virtpg. Locking is required since
*/
/*
* %o0 = vaddr
* %o1 = virtpg to TSB index shift (e.g. TTE page shift)
*/
#ifndef sun4v
#endif
/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
/* %g2 = tsbep, %g1 clobbered */
/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
#endif /* lint */
#if defined (lint)
/*ARGSUSED*/
{ return(0); }
#else /* lint */
/*
* g1 = pfn
*/
#endif /* !lint */
#if defined (lint)
/*
* The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
* the hash list.
*/
/* ARGSUSED */
void
{
}
/*
* The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
* hash list.
*/
/* ARGSUSED */
void
{
}
#else /* lint */
/*
* instead of the whole int because eventually we might want to
* put some counters on the other bytes (of course, these routines would
* have to change). The code that grab this lock should execute
* with interrupts disabled and hold the lock for the least amount of time
* possible.
*/
/*
* Even though hmeh_listlock is updated using pa there's no need to flush
* dcache since hmeh_listlock will be restored to the original value (0)
* before interrupts are reenabled.
*/
/*
* For sparcv9 hme hash buckets may not be in the nucleus. hme hash update
* routines still use virtual addresses to update the bucket fields. But they
* must not cause a TLB miss after grabbing the low level bucket lock. To
* achieve this we must make sure the bucket structure is completely within an
* 8K page.
*/
#endif
label1: ;\
.seg ".data"
.ascii "sfmmu_hblk_hash_add: interrupts disabled"
.byte 0
.ascii "sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
.byte 0
.align 4
.seg ".text"
/*
* %o0 = hmebp
* %o1 = hmeblkp
* %o2 = hblkpa
*/
#ifdef DEBUG
3:
#endif /* DEBUG */
/*
* g1 = hblkpa
*/
#ifdef DEBUG
1:
#endif /* DEBUG */
/*
* We update hmeblks entries before grabbing lock because the stores
* could take a tlb miss and require the hash lock. The buckets
* are part of the nucleus so we are cool with those stores.
*
* if buckets are not part of the nucleus our game is to
* not touch any other page via va until we drop the lock.
* This guarantees we won't get a tlb miss before the lock release
* since interrupts are disabled.
*/
/*
* This function removes an hmeblk from the hash chain.
* It is written to guarantee we don't take a tlb miss
* by using physical addresses to update the list.
*
* %o0 = hmebp
* %o1 = hmeblkp
* %o2 = hmeblkp previous pa
* %o3 = hmeblkp previous
*/
#ifdef DEBUG
3:
#endif /* DEBUG */
/*
* disable interrupts, clear Address Mask to access 64 bit physaddr
*/
#ifndef sun4v
#endif /* sun4v */
/*
* if buckets are not part of the nucleus our game is to
* not touch any other page via va until we drop the lock.
* This guarantees we won't get a tlb miss before the lock release
* since interrupts are disabled.
*/
/*
* hmeblk is first on list
*/
1:
/* hmeblk is not first on list */
#ifndef sun4v
#endif /* sun4v */
2:
#endif /* lint */
/*
* These macros are used to update global sfmmu hme hash statistics
* in perf critical paths. It is only enabled in debug kernels or
* if SFMMU_STAT_GATHER is defined
*/
#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
#else /* DEBUG || SFMMU_STAT_GATHER */
#endif /* DEBUG || SFMMU_STAT_GATHER */
/*
* This macro is used to update global sfmmu kstas in non
* perf critical areas so they are enabled all the time
*/
/*
* These macros are used to update per cpu stats in non perf
* critical areas so they are enabled all the time
*/
/*
* These macros are used to update per cpu stats in non perf
* critical areas so they are enabled all the time
*/
#if defined(KPM_TLBMISS_STATS_GATHER)
/*
* Count kpm dtlb misses separately to allow a different
* evaluation of hme and kpm tlbmisses. kpm tsb hits can
* be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
*/
nop ;\
/* VA range check */ ;\
#else
#endif /* KPM_TLBMISS_STATS_GATHER */
#if defined (lint)
/*
* The following routines are jumped to from the mmu trap handlers to do
* the setting up to call systrap. They are separate routines instead of
* being part of the handlers because the handlers would exceed 32
* instructions and since this is part of the slow path the jump
* cost is irrelevant.
*/
void
sfmmu_pagefault(void)
{
}
void
sfmmu_mmu_trap(void)
{
}
void
sfmmu_window_trap(void)
{
}
void
sfmmu_kpm_exception(void)
{
}
#else /* lint */
#ifdef PTL1_PANIC_DEBUG
.seg ".data"
.word 0
.align 8
.seg ".text"
.align 4
#endif /* PTL1_PANIC_DEBUG */
#ifdef PTL1_PANIC_DEBUG
/* check if we want to test the tl1 panic */
#endif /* PTL1_PANIC_DEBUG */
1:
/*
* g2 = tag access reg
* g3.l = type
* g3.h = 0
*/
2:
1:
/*
* g2 = tag access reg
* g3 = type
*/
/*NOTREACHED*/
1:
/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
/*NOTREACHED*/
/*
* No %g registers in use at this point.
*/
#ifdef sun4v
#ifdef DEBUG
/* We assume previous %gl was 1 */
#endif /* DEBUG */
/* user miss at tl>1. better be the window handler or user_rtt */
/* in user_rtt? */
ba,a 7f
6:
#else /* sun4v */
/* user miss at tl>1. better be the window handler */
#endif /* sun4v */
/* tpc should be in the trap table */
7:
/*
* some wbuf handlers will call systrap to resolve the fault
* we pass the trap type so they figure out the correct parameters.
* g5 = trap type, g6 = tag access reg
*/
/*
* only use g5, g6, g7 registers after we have switched to alternate
* globals.
*/
SET_GL_REG(1)
1:
/*
* We have accessed an unmapped segkpm address or a legal segkpm
* address which is involved in a VAC alias conflict prevention.
* Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
* set. If it is, we will instead note that a fault has occurred
* by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
* a "retry"). This will step over the faulting instruction.
* Note that this means that a legal segkpm address involved in
* a VAC alias conflict prevention (a rare case to begin with)
* cannot be used in DTrace.
*/
bz 0f
0:
1:
/*
* g2=tagacc g3.l=type g3.h=0
*/
#endif /* lint */
#if defined (lint)
void
sfmmu_tsb_miss(void)
{
}
void
sfmmu_kpm_dtsb_miss(void)
{
}
void
{
}
#else /* lint */
#endif
#if (IMAP_SEG != 0)
#endif
/*
* Copies ism mapping for this ctx in param "ism" if this is a ISM
* tlb miss and branches to label "ismhit". If this is not an ISM
* process or an ISM tlb miss it falls thru.
*
* Checks to see if the vaddr passed in via tagacc is in an ISM segment for
* this process.
* If so, it will branch to label "ismhit". If not, it will fall through.
*
* Also hat_unshare() will set the context for this process to INVALID_CONTEXT
* so that any other threads of this process will not try and walk the ism
* maps while they are being changed.
*
* NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
* will make sure of that. This means we can terminate our search on
* the first zero mapping we find.
*
* Parameters:
* tagacc = tag access register (vaddr + ctx) (in)
* tsbmiss = address of tsb miss area (in)
* ismseg = contents of ism_seg for this ism map (out)
* ismhat = physical address of imap_ismhat for this ism map (out)
* tmp1 = scratch reg (CLOBBERED)
* tmp2 = scratch reg (CLOBBERED)
* tmp3 = scratch reg (CLOBBERED)
* label: temporary labels
* ismhit: label where to jump to if an ism dtlb miss
* exitlabel:label where to jump if hat is busy due to hat_unshare.
*/
;\
;\
/*
* Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
* It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
* Parameters:
* vaddr = reg containing virtual address
* hatid = reg containing sfmmu pointer
* hmebp = register where bucket pointer will be stored
* vapg = register where virtual page will be stored
* tmp1, tmp2 = tmp registers
*/
/*
* hashtag includes bspage + hashno (64 bits).
*/
/*
* Function to traverse hmeblk hash link list and find corresponding match.
* The search is done using physical pointers. It returns the physical address
* and virtual address pointers to the hmeblk that matches with the tag
* provided.
* Parameters:
* hmebp = register that points to hme hash bucket, also used as
* tmp reg (clobbered)
* hmeblktag = register with hmeblk tag match
* hatid = register with hatid
* hmeblkpa = register where physical ptr will be stored
* hmeblkva = register where virtual ptr will be stored
* tmp1 = tmp reg
* label: temporary label
*/
#endif
/*
* HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
* he offset for the corresponding hment.
* Parameters:
* vaddr = register with virtual address
* hmeblkpa = physical pointer to hme_blk
* hment = register where address of hment will be stored
* hmentoff = register where hment offset will be stored
* label1 = temporary label
*/
/*
* GET_TTE is a macro that returns a TTE given a tag and hatid.
*
* tagacc = tag access register (vaddr + ctx) (in)
* hatid = sfmmu pointer for TSB miss (in)
* tte = tte for TLB miss if found, otherwise clobbered (out)
* hmeblkpa = PA of hment if found, otherwise clobbered (out)
* hmeblkva = VA of hment if found, otherwise clobbered (out)
* tsbarea = pointer to the tsbmiss area for this cpu. (in)
* hmentoff = temporarily stores hment offset (clobbered)
* for this page size.
* label = temporary label for branching within macro.
* foundlabel = label to jump to when tte is found.
* suspendlabel= label to jump to when tte is suspended.
* exitlabel = label to jump to when tte is not found. The hmebp lock
* is still held at this time.
*
* The caller should set up the tsbmiss->scratch[2] field correctly before
* calling this funciton (aka TSBMISS_SCRATCH + TSBMISS_HATID)
*/
;\
;\
/* ;\
* tagacc = tagacc ;\
* hatid = hatid ;\
* tsbarea = tsbarea ;\
* tte = hmebp (hme bucket pointer) ;\
* hmeblkpa = vapg (virtual page) ;\
* hmentoff, hmeblkva = scratch ;\
*/ ;\
;\
/* ;\
* tagacc = tagacc ;\
* hatid = hatid ;\
* tte = hmebp ;\
* hmeblkpa = CLOBBERED ;\
* hmentoff = htag_bspage & hashno ;\
* hmeblkva = scratch ;\
*/ ;\
/* ;\
* tagacc = CLOBBERED ;\
* tte = CLOBBERED ;\
* hmeblkpa = hmeblkpa ;\
* hmeblkva = hmeblkva ;\
*/ ;\
nop ;\
/* ;\
* We have found the hmeblk containing the hment. ;\
* Now we calculate the corresponding tte. ;\
* ;\
* tagacc = tagacc ;\
* hatid = clobbered ;\
* tte = hmebp ;\
* hmeblkpa = hmeblkpa ;\
* hmentoff = hblktag ;\
* hmeblkva = hmeblkva ;\
*/ ;\
;\
nop ;\
;\
/* ;\
* Mapping is suspended, so goto suspend label. ;\
*/ ;\
/*
* KERNEL PROTECTION HANDLER
*
* g1 = tsb8k pointer register (clobbered)
* g2 = tag access register (ro)
* g3 - g7 = scratch registers
*
* Note: This function is patched at runtime for performance reasons.
* Any changes here require sfmmu_patch_ktsb fixed.
*/
/* %g1 = contents of ktsb_base or ktsb_pbase */
/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
/*
* USER PROTECTION HANDLER
*
* g1 = tsb8k pointer register (ro)
* g2 = tag access register (ro)
* g3 = faulting context (clobbered, currently not used)
* g4 - g7 = scratch registers
*/
#ifdef sun4v
/* %g1 = first TSB entry ptr now, %g2 preserved */
/* %g3 = second TSB entry ptr now, %g2 preserved */
#else /* sun4v */
/* %g3 = second TSB entry ptr now, %g7 clobbered */
#endif /* sun4v */
9:
/*
* Kernel 8K page iTLB miss. We also get here if we took a
* fast instruction access mmu miss trap while running in
* invalid context.
*
* %g1 = 8K TSB pointer register (not used, clobbered)
* %g2 = tag access register (used)
* %g3 = faulting context id (used)
* %g7 = 4M virtual page number for tag matching (used)
*/
.align 64
/* kernel miss */
/* get kernel tsb pointer */
/* we patch the next set of instructions at run time */
/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
/* %g4 = contents of ktsb_base or ktsb_pbase */
/*
* Kernel dTLB miss. We also get here if we took a fast data
* access mmu miss trap while running in invalid context.
*
* Note: for now we store kpm TTEs in the kernel TSB as usual.
* We select the TSB miss handler to branch to depending on
* the virtual address of the access. In the future it may
* be desirable to separate kpm TTEs into their own TSB,
* in which case all that needs to be done is to set
* kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
* early in the miss if we detect a kpm VA to a new handler.
*
* %g1 = 8K TSB pointer register (not used, clobbered)
* %g2 = tag access register (used)
* %g3 = faulting context id (used)
*/
.align 64
/* Gather some stats for kpm misses in the TLB. */
/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
/*
* Get first TSB offset and look for 8K/64K/512K mapping
* using the 8K virtual page as the index.
*
* We patch the next set of instructions at run time;
* any changes here require sfmmu_patch_ktsb changes too.
*/
/* %g7 = contents of ktsb_base or ktsb_pbase */
/*
* At this point %g1 is our index into the TSB.
* We just masked off enough bits of the VA depending
* on our TSB size code.
*/
/* trapstat expects tte in %g5 */
/*
* If kpm is using large pages, the following instruction needs
* to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
* so that we will probe the 4M TSB regardless of the VA. In
* the case kpm is using small pages, we know no large kernel
* mappings are located above 0x80000000.00000000 so we skip the
* probe as an optimization.
*/
/* delay slot safe, below */
/*
* Get second TSB offset and look for 4M mapping
* using 4M virtual page as the TSB index.
*
* Here:
* %g1 = 8K TSB pointer. Don't squash it.
* %g2 = tag access register (we still need it)
*/
/*
* We patch the next set of instructions at run time;
* any changes here require sfmmu_patch_ktsb changes too.
*/
/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
/*
* At this point %g3 is our index into the TSB.
* We just masked off enough bits of the VA depending
* on our TSB size code.
*/
/* we don't check TTE size here since we assume 4M TSB is separate */
/* trapstat expects tte in %g5 */
/*
* So, we failed to find a valid TTE to match the faulting
* address in either TSB. There are a few cases that could land
* us here:
*
* 1) This is a kernel VA below 0x80000000.00000000. We branch
* to sfmmu_tsb_miss_tt to handle the miss.
* 2) We missed on a kpm VA, and we didn't find the mapping in the
* 4M TSB. Let segkpm handle it.
*
* Note that we shouldn't land here in the case of a kpm VA when
* kpm_smallpages is active -- we handled that case earlier at
* dktsb4m_kpmcheck_small.
*
* At this point:
* g1 = 8K-indexed primary TSB pointer
* g2 = tag access register
* g3 = 4M-indexed secondary TSB pointer
*/
#ifdef sun4v
/*
* User instruction miss w/ single TSB.
* The first probe covers 8K, 64K, and 512K page sizes,
* because 64K and 512K mappings are replicated off 8K
* pointer.
*
* g1 = tsb8k pointer register
* g2 = tag access register
* g3 - g6 = scratch registers
* g7 = TSB tag to match
*/
.align 64
/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
/*
* User data miss w/ single TSB.
* The first probe covers 8K, 64K, and 512K page sizes,
* because 64K and 512K mappings are replicated off 8K
* pointer.
*
* g1 = tsb8k pointer register
* g2 = tag access register
* g3 - g6 = scratch registers
* g7 = TSB tag to match
*/
.align 64
/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
#endif /* sun4v */
/*
* User instruction miss w/ multiple TSBs.
* The first probe covers 8K, 64K, and 512K page sizes,
* because 64K and 512K mappings are replicated off 8K
* pointer. Second probe covers 4M page size only.
*
* Just like sfmmu_udtlb_slowpath, except:
* o Uses ASI_ITLB_IN
* o checks for execute permission
* o No ISM prediction.
*
* g1 = tsb8k pointer register
* g2 = tag access register
* g3 - g6 = scratch registers
* g7 = TSB tag to match
*/
.align 64
#ifdef sun4v
/* g4 - g5 = clobbered here */
/* g1 = first TSB pointer, g3 = second TSB pointer */
/* NOT REACHED */
#else /* sun4v */
/* g4 - g5 = clobbered here */
/* g1 = first TSB pointer, g3 = second TSB pointer */
/* NOT REACHED */
#endif /* sun4v */
/*
* User data miss w/ multiple TSBs.
* The first probe covers 8K, 64K, and 512K page sizes,
* because 64K and 512K mappings are replicated off 8K
* pointer. Second probe covers 4M page size only.
*
* We consider probing for 4M pages first if the VA falls
* in a range that's likely to be ISM.
*
* g1 = tsb8k pointer register
* g2 = tag access register
* g3 - g6 = scratch registers
* g7 = TSB tag to match
*/
.align 64
/*
* Check for ISM. If it exists, look for 4M mappings in the second TSB
* first, then probe for other mappings in the first TSB if that fails.
*/
/*
* g1 = 8K TSB pointer register
* g2 = tag access register
* g3 = (potentially) second TSB entry ptr
* g6 = ism pred.
* g7 = vpg_4m
*/
#ifdef sun4v
/*
* Here:
* g1 = first TSB pointer
* g2 = tag access reg
* g3 = second TSB ptr IFF ISM pred. (else don't care)
*/
#else /* sun4v */
/*
* Here:
* g1 = first TSB pointer
* g2 = tag access reg
* g3 = second TSB ptr IFF ISM pred. (else don't care)
*/
/* fall through in 8K->4M probe order */
#endif /* sun4v */
/*
* Look in the second TSB for the TTE
* g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
* g2 = tag access reg
* g3 = 8K TSB pointer register
* g6 = ism pred.
* g7 = vpg_4m
*/
#ifdef sun4v
/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
/* tagacc (%g2) not destroyed */
/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
#else
/* %g2 clobbered, %g3 =second tsbe ptr */
#endif
/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
/* fall through to sfmmu_tsb_miss_tt */
/*
* We get here if there is a TSB miss OR a write protect trap.
*
* g1 = First TSB entry pointer
* g2 = tag access register
* g3 = 4M TSB entry pointer; NULL if no 2nd TSB
* g4 - g7 = scratch registers
*/
/*
* If trapstat is running, we need to shift the %tpc and %tnpc to
* point to trapstat's TSB miss return code (note that trapstat
* itself will patch the correct offset to add).
*/
/* delay slot safe */
0:
/*
* The miss wasn't in an ISM segment.
*
* %g1 %g3, %g4, %g5, %g7 all clobbered
* %g2 = tag access (vaddr + ctx)
*/
1:
/*
* 8K and 64K hash.
*/
2:
/* NOT REACHED */
/*
* Note that there is a small window here where we may have
* a 512k page in the hash list but have not set the HAT_512K_FLAG
* flag yet, so we will skip searching the 512k hash list.
* In this case we will end up in pagefault which will find
* the mapping and return. So, in this instance we will end up
* spending a bit more time resolving this TSB miss, but it can
* only happen once per process and even then, the chances of that
* are very small, so it's not worth the extra overhead it would
* take to close this window.
*/
3:
/*
* 512K hash
*/
/* NOT REACHED */
4:
/*
* 4M hash
*/
/* NOT REACHED */
#ifndef sun4v
#endif
#ifdef sun4v
#else
#endif
5:
/*
* 32M hash
*/
/* NOT REACHED */
6:
/*
* 256M hash
*/
/* NOT REACHED */
/*
* g3 = tte
* g4 = tte pa
* g5 = tte va
* g6 = tsbmiss area
*/
/*
*/
4:
/*
* If ITLB miss check exec bit.
* If not set treat as invalid TTE.
*/
5:
3:
/*
* Set reference bit if not already set
*/
/*
* g3 = tte
* g4 = patte
* g6 = tsbmiss area
*/
#ifdef sun4v
9:
#else
#endif
#ifdef sun4v
#else
#endif
#ifdef sun4v
#else
#endif
#ifndef sun4v
#endif /* sun4v */
#ifdef sun4v
#endif /* sun4v */
9:
#ifndef sun4v
#endif
5:
#ifdef sun4v
#endif /* sun4v */
9:
#ifdef sun4v
#else
#endif
5:
6:
#ifndef sun4v
#endif
3:
#ifdef sun4v
#endif /* sun4v */
1:
/*
* This is an ISM [i|d]tlb miss. We optimize for largest
* page size down to smallest.
*
* g2 = vaddr + ctx aka tag access register
* g3 = ismmap->ism_seg
* g4 = physical address of ismmap->ism_sfmmu
* g6 = tsbmiss area
*/
/* g5 = pa of imap_vb_shift */
/*
* ISM pages are always locked down.
* If we can't find the tte then pagefault
* and let the spt segment driver resovle it.
*
* g2 = ISM vaddr (offset in ISM seg)
* g6 = tsb miss area
* g7 = ISM hatid
*/
/*
* 32M hash.
*/
/* NOT REACHED */
/*
* 256M hash.
*/
/*
* 4M hash.
*/
/* NOT REACHED */
/*
* 8K and 64K hash.
*/
/* NOT REACHED */
/*
* we get here if we couldn't find a valid tte in the hash.
*
* If user and we are at tl>1 we go to window handling code.
*
* If kernel and the fault is on the same page as our stack
* pointer, then we know the stack is bad and the trap handler
* will fail, so we call ptl1_panic with PTL1_BAD_STACK.
*
* If this is a kernel trap and tl>1, panic.
*
* Otherwise we call pagefault.
*/
#ifdef sun4v
#else
#endif
2:
/*
* We are taking a pagefault in the kernel on a kernel address. If
* CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
* want to call sfmmu_pagefault -- we will instead note that a fault
* has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
* (instead of a "retry"). This will step over the faulting
* instruction.
*/
3:
4:
/*
* We are taking a pagefault on a non-kernel address. If we are in
* the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
* and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
*/
/*
* Be sure that we're actually taking this miss from the kernel --
* otherwise we have managed to return to user-level with
* CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
*/
bz,a ptl1_panic
/*
* If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
* if it is, indicated that we have faulted and issue a done.
*/
bz 1f
/*
* Be sure that we're actually taking this miss from the kernel --
* otherwise we have managed to return to user-level with
* CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
*/
bz,a ptl1_panic
1:
#ifdef sun4v
2:
#else
#endif
#endif
#endif /* lint */
#if defined (lint)
/*
* This routine will look for a user or kernel vaddr in the hash
* structure. It returns a valid pfn or PFN_INVALID. It doesn't
* grab any locks. It should only be used by other sfmmu routines.
*/
/* ARGSUSED */
{
return(0);
}
#else /* lint */
/*
* disable interrupts
*/
#ifdef DEBUG
1:
#endif
/*
* disable interrupts to protect the TSBMISS area
*/
/*
* o0 = vaddr
* o1 = sfmmup
* o2 = ttep
*/
/*
* o0 = vaddr
* o1 & o4 = hatid
* o2 = ttep
* o5 = tsbmiss area
*/
1:
/*
* o0 = vaddr
* o1 = sfmmup
* o2 = ttep
* o3 = old %pstate
* o4 = hatid
* o5 = tsbmiss
* g5 = rehash #
* g6 = hmeshift
*
* The first arg to GET_TTE is actually tagaccess register
* not just vaddr. Since this call is for kernel we need to clear
* any lower vaddr bits that would be interpreted as ctx bits.
*/
/*
* o0 = vaddr
* o1 = sfmmup
* o2 = ttep
* g1 = tte
* g2 = tte pa
* g3 = tte va
* o2 = tsbmiss area
* o1 = hat id
*/
/*
* o0 = vaddr
* o1 = sfmmup
* o2 = ttep
* g1 = pfn
*/
/*
* we get here if we couldn't find valid hblk in hash. We rehash
* if neccesary.
*/
#ifdef sun4v
#else
#endif
#ifdef sun4v
#else
#endif
6:
/*
* o0 = vaddr
* o1 = sfmmup
* o2 = ttep
* g1 = tte
* g2 = tte pa
* g3 = tte va
* o2 = tsbmiss area use o5 instead of o2 for tsbmiss
*/
/*
* o0 = PFN return value PFN_INVALID, PFN_SUSPENDED, or pfn#
* o1 = sfmmup
* o2 = ttep
* g1 = pfn
*/
8:
/*
* This routine does NOT support user addresses
* There is a routine in C that supports this.
* The only reason why we don't have the C routine
* support kernel addresses as well is because
* we do va_to_pa while holding the hashlock.
*/
#endif /* lint */
#if !defined(lint)
/*
* kpm lock used between trap level tsbmiss handler and kpm C level.
*/
label1: ;\
/*
* Lookup a memseg for a given pfn and if found, return the physical
* address of the corresponding struct memseg in mseg, otherwise
* return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
* tsbmp, %asi is assumed to be ASI_MEM.
* This lookup is done by strictly traversing only the physical memseg
* linkage. The more generic approach, to check the virtual linkage
* before using the physical (used e.g. with hmehash buckets), cannot
* be used here. Memory DR operations can run in parallel to this
* lookup w/o any locks and updates of the physical and virtual linkage
* cannot be done atomically wrt. to each other. Because physical
* address zero can be valid physical address, MSEG_NULLPTR_PA acts
* as "physical NULL" pointer.
*/
nop ;\
/* brute force lookup */ ;\
nop ;\
/*
* kpm tsb miss handler large pages
* g1 = 8K kpm TSB entry pointer
* g2 = tag access register
* g3 = 4M kpm TSB entry pointer
*/
/* check enable flag */
/* VA range check */
/*
* check TL tsbmiss handling flag
* bump tsbmiss counter
*/
#ifdef DEBUG
#else
#endif
/*
* At this point:
* g1 = 8K kpm TSB pointer (not used)
* g2 = tag access register
* g3 = clobbered
* g6 = per-CPU kpm tsbmiss area
* g7 = kpm_vbase
*/
/* vaddr2pfn */
/*
* Setup %asi
* mseg_pa = page_numtomemseg_nolock(pfn)
* if (mseg_pa == NULL) sfmmu_kpm_exception
* g2=pfn
*/
/*
* inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
* g2=pfn g3=mseg_pa
*/
/*
* Validate inx value
* g2=pfn g3=mseg_pa g4=inx
*/
#ifdef DEBUG
#else
#endif
/*
* kp = &mseg_pa->kpm_pages[inx]
*/
/*
* KPMP_HASH(kp)
* g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
*/
/*
* Calculate physical kpm_page pointer
* g2=pfn g3=mseg_pa g4=offset g5=hashinx
*/
/*
* Calculate physical hash lock address
* g1=kp_refcntc_pa g2=pfn g5=hashinx
*/
/*
* Assemble tte
* g1=kp_pa g2=pfn g3=hlck_pa
*/
#ifdef sun4v
#else
#endif
/*
* tsb dropin
* g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
*/
/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
/* use C-handler if there's no go for dropin */
#ifdef DEBUG
/* double check refcnt */
#endif
#ifndef sun4v
#endif
/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
/* KPMLOCK_EXIT(kpmlckp, asi) */
/*
* If trapstat is running, we need to shift the %tpc and %tnpc to
* point to trapstat's TSB miss return code (note that trapstat
* itself will patch the correct offset to add).
* Note: TTE is expected in %g5 (allows per pagesize reporting).
*/
0:
5:
/* g3=hlck_pa */
/*
* kpm tsbmiss handler for smallpages
* g1 = 8K kpm TSB pointer
* g2 = tag access register
* g3 = 4M kpm TSB pointer
*/
/* check enable flag */
/*
* VA range check
* On fail: goto sfmmu_tsb_miss
*/
/*
* check TL tsbmiss handling flag
* bump tsbmiss counter
*/
#ifdef DEBUG
#else
#endif
/*
* At this point:
* g1 = clobbered
* g2 = tag access register
* g3 = 4M kpm TSB pointer (not used)
* g6 = per-CPU kpm tsbmiss area
* g7 = kpm_vbase
*/
/* vaddr2pfn */
/*
* Setup %asi
* mseg_pa = page_numtomemseg_nolock_pa(pfn)
* if (mseg not found) sfmmu_kpm_exception
* g2=pfn
*/
/*
* inx = pfn - mseg_pa->kpm_pbase
* g2=pfn g3=mseg_pa
*/
#ifdef DEBUG
/*
* Validate inx value
* g2=pfn g3=mseg_pa g4=inx
*/
#else
#endif
/* ksp = &mseg_pa->kpm_spages[inx] */
/*
* KPMP_SHASH(kp)
* g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz
*/
/*
* Calculate physical kpm_spage pointer
* g2=pfn g3=mseg_pa g4=offset g5=hashinx
*/
/*
* Calculate physical hash lock address.
* Note: Changes in kpm_shlk_t must be reflected here.
* g1=ksp_pa g2=pfn g5=hashinx
*/
/*
* Assemble tte
* g1=ksp_pa g2=pfn g3=hlck_pa
*/
/*
* tsb dropin
* g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte
*/
/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
/* use C-handler if there's no go for dropin */
#ifndef sun4v
#endif
/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
/* KPMLOCK_EXIT(kpmlckp, asi) */
/*
* If trapstat is running, we need to shift the %tpc and %tnpc to
* point to trapstat's TSB miss return code (note that trapstat
* itself will patch the correct offset to add).
* Note: TTE is expected in %g5 (allows per pagesize reporting).
*/
0:
5:
/* g3=hlck_pa */
#endif
#endif /* lint */
#ifdef lint
/*
* khl_lock is a low level spin lock to protect the kp_tsbmtl field.
* Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
* Assumes khl_mutex is held when called from C-level.
*/
/* ARGSUSED */
void
{
}
/*
* kpm_smallpages: stores val to byte at address mapped within
* low level lock brackets. The old value is returned.
* Called from C-level.
*/
/* ARGSUSED */
int
{
return (0);
}
#else /* lint */
.seg ".data"
.ascii "sfmmu_kpm_tsbmtl: interrupts disabled"
.byte 0
.ascii "sfmmu_kpm_stsbmtl: interrupts disabled"
.byte 0
.align 4
.seg ".text"
/*
* %o0 = &kp_refcntc
* %o1 = &khl_lock
* %o3 = pstate save
*/
#ifdef DEBUG
1:
#endif /* DEBUG */
2:
/*
* %o0 = &mapped
* %o1 = &kshl_lock
* %o2 = val
* %o3 = pstate save
*/
#ifdef DEBUG
1:
#endif /* DEBUG */
#endif /* lint */
#ifndef lint
#ifdef sun4v
/*
* The first probe covers 8K, 64K, and 512K page sizes,
* because 64K and 512K mappings are replicated off 8K
* pointer. Second probe covers 4M page size only.
*
* MMU fault area contains miss address and context.
*/
/*
* %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
* %g3 = ctx (cannot be INVALID_CONTEXT)
*/
/*
* Kernel miss
* Get 8K and 4M TSB pointers in %g1 and %g3 and
* branch to sfmmu_tsb_miss_tt to handle it.
*/
8:
/*
* User miss
* Get first TSB pointer in %g1
* Get second TSB pointer (or NULL if no second TSB) in %g3
* Branch to sfmmu_tsb_miss_tt to handle it
*/
/* %g1 = first TSB entry ptr now, %g2 preserved */
/* %g3 = second TSB entry ptr now, %g2 preserved */
9:
/*
* The first probe covers 8K, 64K, and 512K page sizes,
* because 64K and 512K mappings are replicated off 8K
* pointer. Second probe covers 4M page size only.
*
* MMU fault area contains miss address and context.
*/
#endif /* sun4v */
#endif /* lint */
#ifndef lint
/*
* Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
*/
.seg ".data"
.align 64
.align 64
#endif /* lint */