us3_cheetahplus_asm.s revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*
* Assembly code support for the Cheetah+ module
*/
#pragma ident "%Z%%M% %I% %E% SMI"
#if !defined(lint)
#include "assym.h"
#endif /* lint */
#include <sys/asm_linkage.h>
#include <vm/hat_sfmmu.h>
#include <sys/machparam.h>
#include <sys/machcpuvar.h>
#include <sys/machthread.h>
#include <sys/machtrap.h>
#include <sys/privregs.h>
#include <sys/asm_linkage.h>
#include <sys/cheetahregs.h>
#include <sys/us3_module.h>
#include <sys/cheetahasm.h>
#ifdef TRAPTRACE
#include <sys/traptrace.h>
#endif /* TRAPTRACE */
#if !defined(lint)
/* BEGIN CSTYLED */
/*
* Cheetah+ version to reflush an Ecache line by index.
*
* By default we assume the Ecache is 2-way so we flush both
* ways. Even if the cache is direct-mapped no harm will come
* from performing the flush twice, apart from perhaps a performance
* penalty.
*
* XXX - scr2 not used.
*/
/*
* Cheetah+ version of ecache_flush_line. Uses Cheetah+ Ecache Displacement
* Flush feature.
*/
/* END CSTYLED */
/*
* Panther version to reflush a line from both the L2 cache and L3
* cache by the respective indexes. Flushes all ways of the line from
* each cache.
*
* l2_index Index into the L2$ of the line to be flushed. This
* register will not be modified by this routine.
* l3_index Index into the L3$ of the line to be flushed. This
* register will not be modified by this routine.
* scr2 scratch register.
* scr3 scratch register.
*
*/
1: \
bg,a 1b; \
2: \
bg,a 2b; \
/*
* Panther version of ecache_flush_line. Flushes the line corresponding
* to physaddr from both the L2 cache and the L3 cache.
*
* physaddr Input: Physical address to flush.
* Output: Physical address to flush (preserved).
* l2_idx_out Input: scratch register.
* Output: Index into the L2$ of the line to be flushed.
* l3_idx_out Input: scratch register.
* Output: Index into the L3$ of the line to be flushed.
* scr3 scratch register.
* scr4 scratch register.
*
*/
#endif /* !lint */
/*
* Fast ECC error at TL>0 handler
* We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter
* this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
* For a complete description of the Fast ECC at TL>0 handling see the
*/
#if defined(lint)
void
fast_ecc_tl1_err(void)
{}
#else /* lint */
.section ".text"
.align 64
/*
* This macro turns off the D$/I$ if they are on and saves their
* original state in ch_err_tl1_tmp, saves all the %g registers in the
* ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
* the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will
* point to the ch_err_tl1_data structure and the original D$/I$ state
* will be saved in ch_err_tl1_tmp. All %g registers except for %g1
* will be available.
*/
/*
* Get the diagnostic logout data. %g4 must be initialized to
* current CEEN state, %g5 must point to logout structure in
* ch_err_tl1_data_t. %g3 will contain the nesting count upon
* return.
*/
/*
* If the logout nesting count is exceeded, we're probably
* not making any progress, try to panic instead.
*/
/*
* Save the current CEEN and NCEEN state in %g7 and turn them off
* before flushing the Ecache.
*/
/*
* Flush the Ecache, using the largest possible cache size with the
* smallest possible line size since we can't get the actual sizes
* from the cpu_node due to DTLB misses.
*/
2:
/*
* Restore CEEN and NCEEN to the previous state.
*/
/*
* If we turned off the D$, then flush it and turn it back on.
*/
/*
* Flush the D$.
*/
/*
* Turn the D$ back on.
*/
3:
/*
* If we turned off the I$, then flush it and turn it back on.
*/
/*
* Flush the I$. Panther has different I$ parameters, and we
* can't access the logout I$ params without possibly generating
* a MMU miss.
*/
/*
* Turn the I$ back on. Changing DCU_IC requires flush.
*/
4:
#ifdef TRAPTRACE
/*
* Get current trap trace entry physical pointer.
*/
/*
* Create trap trace entry.
*/
/*
* Advance trap trace pointer.
*/
#endif /* TRAPTRACE */
/*
* handling and just do the necessary cache-flushing.
*/
/*
* If a UCU or L3_UCU followed by a WDU has occurred go ahead
* and panic since a UE will occur (on the retry) before the
* UCU and WDU messages are enqueued.
*/
5:
6:
/*
* We fall into this macro if we've successfully logged the error in
* the ch_err_tl1_data structure and want the PIL15 softint to pick
* it up and log it. %g1 must point to the ch_err_tl1_data structure.
* Restores the %g registers and issues retry.
*/
/*
* Establish panic exit label.
*/
#endif /* lint */
#if defined(lint)
/*
* scrubphys - Pass in the aligned physical memory address
* that you want to scrub, along with the ecache set size.
*
* 1) Displacement flush the E$ line corresponding to %addr.
* The first ldxa guarantees that the %addr is no longer in
* M, O, or E (goes to I or S (if instruction fetch also happens).
* 2) "Write" the data using a CAS %addr,%g0,%g0.
* The casxa guarantees a transition from I to M or S to M.
* 3) Displacement flush the E$ line corresponding to %addr.
* The second ldxa pushes the M line out of the ecache, into the
* writeback buffers, on the way to memory.
* 4) The "membar #Sync" pushes the cache line out of the writeback
* buffers onto the bus, on the way to dram finally.
*
* This is a modified version of the algorithm suggested by Gary Lauterbach.
* In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
* as modified, but then we found out that for spitfire, if it misses in the
* E$ it will probably install as an M, but if it hits in the E$, then it
* will stay E, if the store doesn't happen. So the first displacement flush
* should ensure that the CAS will miss in the E$. Arrgh.
*/
/* ARGSUSED */
void
{}
#else /* lint */
#endif /* lint */
#if defined(lint)
/*
* clearphys - Pass in the aligned physical memory address
* that you want to push out, as a ecache_linesize byte block of zeros,
* from the ecache zero-filled.
*/
/* ARGSUSED */
void
{
}
#else /* lint */
/* turn off IE, AM bits */
/* turn off NCEEN */
/* zero the E$ line */
/* clear the AFSR */
/* turn NCEEN back on */
/* return and re-enable IE and AM */
#endif /* lint */
#if defined(lint)
/*
* Cheetah+ Ecache displacement flush the specified line from the E$
*
* For Panther, this means flushing the specified line from both the
* L2 cache and L3 cache.
*
* Register usage:
* %o0 - 64 bit physical address for flushing
* %o1 - Ecache set size
*/
/*ARGSUSED*/
void
{
}
#else /* lint */
#endif /* lint */
#if defined(lint)
void
{
}
#else /* lint */
#endif /* lint */
#if defined(lint)
/*
* The CPU jumps here from the MMU exception handler if an ITLB parity
* error is detected and we are running on Panther.
*
* In this routine we collect diagnostic information and write it to our
* logout structure (if possible) and clear all ITLB entries that may have
* caused our parity trap.
* Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
* and log any error messages. As for parameters to cpu_tlb_parity_error, we
* send two:
*
* %g2 - Contains the VA whose lookup in the ITLB caused the parity error
* %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct,
* regardless of whether or not we actually used the logout struct.
*
* In the TL0 handler (cpu_tlb_parity_error) we will compare those two
* parameters to the data contained in the logout structure in order to
* determine whether the logout information is valid for this particular
* error or not.
*/
void
itlb_parity_trap(void)
{}
#else /* lint */
/*
* Collect important information about the trap which will be
* used as a parameter to the TL0 handler.
*/
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
*
* Next, we calculate the TLB index value for the failing VA.
*/
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
* %g4 - contains the TLB access index value for the
*
* Check to see if the logout structure is available.
*/
cmp %g6, %g5 ! information before clearing
bne itlb_parity_trap_1 ! and logging the error.
nop
/*
* Record the logout information. %g4 contains our index + TLB ID
* for use in ASI_ITLB_ACCESS and ASI_ITLB_TAGREAD. %g1 contains
* the pointer to our logout struct.
*/
stx %g3, [%g1 + PN_TLO_INFO]
stx %g2, [%g1 + PN_TLO_ADDR]
stx %g2, [%g1 + PN_TLO_PC] ! %tpc == fault addr for IMMU
add %g1, PN_TLO_ITLB_TTE, %g1 ! move up the pointer
ldxa [%g4]ASI_ITLB_ACCESS, %g5 ! read the data
stx %g5, [%g1 + CH_TLO_TTE_DATA] ! store it away
ldxa [%g4]ASI_ITLB_TAGREAD, %g5 ! read the tag
stx %g5, [%g1 + CH_TLO_TTE_TAG] ! store it away
set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1
or %g4, %g6, %g4
add %g1, CH_TLO_TTE_SIZE, %g1 ! move up the pointer
ldxa [%g4]ASI_ITLB_ACCESS, %g5 ! read the data
stx %g5, [%g1 + CH_TLO_TTE_DATA] ! store it away
ldxa [%g4]ASI_ITLB_TAGREAD, %g5 ! read the tag
stx %g5, [%g1 + CH_TLO_TTE_TAG] ! store it away
andn %g4, %g6, %g4 ! back to way 0
itlb_parity_trap_1:
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
* %g4 - contains the TLB access index value for the
*
* Here we will clear the errors from the TLB.
*/
set MMU_TAG_ACCESS, %g5 ! We write a TTE tag value of
stxa %g0, [%g5]ASI_IMMU ! 0 as it will be invalid.
stxa %g0, [%g4]ASI_ITLB_ACCESS ! Write the data and tag
membar #Sync
set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1
or %g4, %g6, %g4
stxa %g0, [%g4]ASI_ITLB_ACCESS ! Write same data and tag
membar #Sync
sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a
flush %g6 ! flush after writing MMU regs
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
*
* Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
#endif /* lint */
#if defined(lint)
/*
* The CPU jumps here from the MMU exception handler if a DTLB parity
* error is detected and we are running on Panther.
*
* In this routine we collect diagnostic information and write it to our
* logout structure (if possible) and clear all DTLB entries that may have
* caused our parity trap.
* Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0
* and log any error messages. As for parameters to cpu_tlb_parity_error, we
* send two:
*
* %g2 - Contains the VA whose lookup in the DTLB caused the parity error
* %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct,
* regardless of whether or not we actually used the logout struct.
*
* In the TL0 handler (cpu_tlb_parity_error) we will compare those two
* parameters to the data contained in the logout structure in order to
* determine whether the logout information is valid for this particular
* error or not.
*/
void
dtlb_parity_trap(void)
{}
#else /* lint */
/*
* Collect important information about the trap which will be
* used as a parameter to the TL0 handler.
*/
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
*
* Calculate the TLB index values for the failing VA. Since the T512
* TLBs can be configured for different page sizes, we need to find
* the index into each one separately.
*/
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
* %g4 - contains the T512_0 access index value for the
* %g7 - contains the T512_1 access index value for the
*
* If this trap happened at TL>0, then we don't want to mess
* with the normal logout struct since that could caused a TLB
* miss.
*/
/*
* If we are here, then the trap happened at TL>1. Simply
* update our tlo_info field and then skip to the TLB flush
* code.
*/
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
* %g4 - contains the T512_0 access index value for the
* %g7 - contains the T512_1 access index value for the
*
* Check to see if the logout structure is available.
*/
cmp %g6, %g5 ! information before clearing
bne dtlb_parity_trap_2 ! and logging the error.
nop
/*
* Record the logout information. %g4 contains our DTLB_0
* index + TLB ID and %g7 contains our DTLB_1 index + TLB ID
* both of which will be used for ASI_DTLB_ACCESS and
* ASI_DTLB_TAGREAD. %g1 contains the pointer to our logout
* struct.
*/
stx %g3, [%g1 + PN_TLO_INFO]
stx %g2, [%g1 + PN_TLO_ADDR]
rdpr %tpc, %g5
stx %g5, [%g1 + PN_TLO_PC]
add %g1, PN_TLO_DTLB_TTE, %g1 ! move up the pointer
ldxa [%g4]ASI_DTLB_ACCESS, %g5 ! read the data from DTLB_0
stx %g5, [%g1 + CH_TLO_TTE_DATA] ! way 0 and store it away
ldxa [%g4]ASI_DTLB_TAGREAD, %g5 ! read the tag from DTLB_0
stx %g5, [%g1 + CH_TLO_TTE_TAG] ! way 0 and store it away
ldxa [%g7]ASI_DTLB_ACCESS, %g5 ! now repeat for DTLB_1 way 0
stx %g5, [%g1 + (CH_TLO_TTE_DATA + (CH_TLO_TTE_SIZE * 2))]
ldxa [%g7]ASI_DTLB_TAGREAD, %g5
stx %g5, [%g1 + (CH_TLO_TTE_TAG + (CH_TLO_TTE_SIZE * 2))]
set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1
or %g4, %g6, %g4 ! of each TLB.
or %g7, %g6, %g7
add %g1, CH_TLO_TTE_SIZE, %g1 ! move up the pointer
ldxa [%g4]ASI_DTLB_ACCESS, %g5 ! read the data from DTLB_0
stx %g5, [%g1 + CH_TLO_TTE_DATA] ! way 1 and store it away
ldxa [%g4]ASI_DTLB_TAGREAD, %g5 ! read the tag from DTLB_0
stx %g5, [%g1 + CH_TLO_TTE_TAG] ! way 1 and store it away
ldxa [%g7]ASI_DTLB_ACCESS, %g5 ! now repeat for DTLB_1 way 1
stx %g5, [%g1 + (CH_TLO_TTE_DATA + (CH_TLO_TTE_SIZE * 2))]
ldxa [%g7]ASI_DTLB_TAGREAD, %g5
stx %g5, [%g1 + (CH_TLO_TTE_TAG + (CH_TLO_TTE_SIZE * 2))]
andn %g4, %g6, %g4 ! back to way 0
andn %g7, %g6, %g7 ! back to way 0
dtlb_parity_trap_2:
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
* %g4 - contains the T512_0 access index value for the
* %g7 - contains the T512_1 access index value for the
*
* Here we will clear the errors from the DTLB.
*/
set MMU_TAG_ACCESS, %g5 ! We write a TTE tag value of
stxa %g0, [%g5]ASI_DMMU ! 0 as it will be invalid.
stxa %g0, [%g4]ASI_DTLB_ACCESS ! Write the data and tag.
stxa %g0, [%g7]ASI_DTLB_ACCESS ! Now repeat for DTLB_1 way 0
membar #Sync
set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1
or %g4, %g6, %g4
or %g7, %g6, %g7
stxa %g0, [%g4]ASI_DTLB_ACCESS ! Write same data and tag.
stxa %g0, [%g7]ASI_DTLB_ACCESS ! Now repeat for DTLB_1 way 0
membar #Sync
sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a
flush %g6 ! flush after writing MMU regs
/*
* at this point:
* %g2 - contains the VA whose lookup caused the trap
* %g3 - contains the tlo_info field
*
* Call cpu_tlb_parity_error via systrap at PIL 14 unless we're
*/
#endif /* lint */
#if defined(lint)
/*
* Calculates the Panther TLB index based on a virtual address and page size
*
* Register usage:
* %o0 - virtual address whose index we want
* %o1 - Page Size of the TLB in question as encoded in the
* ASI_[D|I]MMU_TAG_ACCESS_EXT register.
*/
{
}
#else /* lint */
#endif /* lint */
#if defined(lint)
/*
* For Panther CPUs we need to flush the IPB after any I$ or D$
* parity errors are detected.
*/
void
flush_ipb(void)
{ return; }
#else /* lint */
#endif /* lint */