drmach_asm.s revision 1e2e7a75ddb1eedcefa449ce98fd5862749b72ee
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* This file is through cpp before being used as
* an inline. It contains support routines used
* only by DR.
*/
#if defined(lint)
#else
#include "assym.h"
#endif /* lint */
#include <sys/asm_linkage.h>
#include <sys/privregs.h>
#include <sys/machthread.h>
#include <sys/cheetahregs.h>
#include <sys/cheetahasm.h>
#if defined(lint)
/*ARGSUSED*/
void
{}
/*ARGSUSED*/
void
{}
void
drmach_rename_end(void)
{}
/*ARGSUSED*/
void
{
}
/*ARGSUSED*/
void
{
}
/*ARGSUSED*/
void
{
}
/*ARGSUSED*/
{
return (0x0ull);
}
/* ARGSUSED */
{
return (0x0);
}
#else /* lint */
1: ;\
ba,a 1b ;\
nop ;\
2:
#define LPA_MASK 0x7ff8
bz 2f ;\
nop ;\
bnz,a 1f ;\
1: ;\
2: ;\
! While running at trap level > 0, all memory accesses are
! performed using NUCLEUS context, which is always 0.
! Since the cross trap handler does not force PRIMARY context
! to be zero, the following casxa instruction must specify
! NUCLEUS ASI.
! This ASI must be specified explicitly (via casxa), rather
! than using casx. This is because of the fact that the
! default casx specifies ASI_PRIMARY, which if non-zero, can
! prevent the cpu from translating the address, leading to panic
! on bad trap following repetitive dtlb misses. This behavior
! was encountered on MCPUs when using casx instruction.
#define ATOMIC_ADD_LONG(label, simm, reg1, reg2, reg3) \
set label, reg1 ;\
ldx [reg1], reg2 ;\
1: ;\
add reg2, simm, reg3 ;\
casxa [reg1]ASI_N, reg2, reg3 ;\
cmp reg2, reg3 ;\
bne,a,pn %xcc, 1b ;\
ldx [reg1], reg2
#define HERE(reg1, simm, reg2) \
rdpr %tick, reg2 ;\
stx reg2, [reg1 + simm]
!
! Returns processor icache size and linesize in reg1 and
! reg2, respectively.
!
! Panther has a larger icache compared to Cheetahplus and
! Jaguar.
!
#define GET_ICACHE_PARAMS(reg1, reg2) \
GET_CPU_IMPL(reg1) ;\
cmp reg1, PANTHER_IMPL ;\
bne %xcc, 1f ;\
nop ;\
set PN_ICACHE_SIZE, reg1 ;\
set PN_ICACHE_LSIZE, reg2 ;\
ba 2f ;\
nop ;\
1: ;\
set CH_ICACHE_SIZE, reg1 ;\
set CH_ICACHE_LSIZE, reg2 ;\
2:
#define DRMACH_MCU_IDLE_READS 3
! Macro to check if a Panther MC is idle. The EMU Activity
! Status register is first read to clear the MCU status bit.
! The MCU status is then checked DRMACH_MCU_IDLE_READS times
! to verify the MCU is indeed idle. A single non-idle status
! will fail the idle check. This could be made more lenient
! by adding a retry loop.
! addr: Panther EMU Activity Status register read address.
! Assumed to be 0x18 for local ASI access or else
! FIREPLANE_ADDRESS_REG + 0x400050 for PIO access.
! 0 is returned in this register if MCU is idle and
! queues are empty. Otherwise, -1 is returned in this
! register.
! asi: Immediate asi value. Assumed to be ASI_SAFARI_CONFIG
! for local ASI or ASI_IO for PIO access.
! scr1: Scratch
! scr2: Scratch
!
#define CHECK_MCU_IDLE(addr, asi, scr1, scr2) \
ldxa [addr]asi, %g0 ;\
ba 1f ;\
clr scr2 ;\
0: ;\
btst MCU_ACT_STATUS, scr1 ;\
bne,a 2f ;\
sub %g0, 1, addr ;\
inc scr2 ;\
1: ;\
cmp scr2, DRMACH_MCU_IDLE_READS ;\
ble,a 0b ;\
ldxa [addr]asi, scr1 ;\
clr addr ;\
2:
! drmach_shutdown_asm
!
! inputs:
! %o0 = stack pointer
! %o1 = ecache flush address (ignored if cheetah+ processor)
! %o2 = ecache size
! %o3 = ecache line size
! %o4 = phys addr of byte to clear when finished
!
! output:
! Stores a zero at [%o4]ASI_MEM when the processor
! is ready to be removed from domain coherency.
!
ENTRY_NP(drmach_shutdown_asm)
membar #LoadStore ! parsley.
! Calculate pointer to data area. Determine size of
! drmach_shutdown_asm, add to base address and align
! to next 16 byte boundary. Leave result in %g6.
set drmach_shutdown_asm_end, %g6
set drmach_shutdown_asm, %g1
set drmach_cpu_sram_va, %g2
ldx [%g2], %g2
sub %g6, %g1, %g6
add %g6, %g2, %g6
add %g6, 15, %g6
andn %g6, 15, %g6
! Save parameters
stx %o0, [%g6 + 0] ! save stack pointer
stx %o1, [%g6 + 24] ! save E$ flush PA
st %o2, [%g6 + 32] ! save E$ size
st %o3, [%g6 + 36] ! save E$ linesize
stx %o4, [%g6 + 40] ! save phys addr of signal byte
set dcache_size, %g1
ld [%g1], %g1
st %g1, [%g6 + 8] ! save dcache_size
set dcache_linesize, %g1
ld [%g1], %g1
st %g1, [%g6 + 12] ! save dcache_linesize
GET_ICACHE_PARAMS(%g1, %g2)
st %g1, [%g6 + 16] ! save icache_size
st %g2, [%g6 + 20] ! save icache_linesize
! Flushes all active windows except the current one.
! Can cause spill traps to occur.
flushw
! Make sure all asynchronous processing is complete.
! Note: has no implications on pending bus transactions.
membar #Sync
! Move stack. Algorithm copied from t0stacktop setup of
! Replaces SWITCH_STACK() macro used in Starfire DR.
ldx [%g6 + 0], %g1
sub %g1, SA(KFPUSIZE+GSR_SIZE), %g2
and %g2, 0x3f, %g3
sub %g2, %g3, %o2
sub %o2, SA(MPCBSIZE) + STACK_BIAS, %sp
stx %sp, [%g6 + 48] ! for debug
HERE(%g6, 128, %g1) ! initialization complete (for debug)
! Panther needs to flush the L2 cache before the L3
! cache is flushed by the ecache flushall macro.
PN_L2_FLUSHALL(%g1, %g2, %g3)
! Flush E$. The purpose of this flush is to rid the E$ of
! lines in states O or Os. Implicitly flushes W$.
ldx [%g6 + 24], %g1 ! *ecache_flushaddr
ld [%g6 + 32], %g2 ! ecache_size
ld [%g6 + 36], %g3 ! ecache_linesize
ECACHE_FLUSHALL(%g2, %g3, %g1, %g4)
! Since the bus sync list read below does not guarantee
! transaction completion on Panther domains, as an
! optimization Panther skips the read and subsequent
! E$ flush.
GET_CPU_IMPL(%g1)
cmp %g1, PANTHER_IMPL
be %xcc, drmach_shutdown_ecache_flushed
nop
!
! Ensure all outstanding writebacks have retired. Following this
! sync, all writes must be strictly managed.
!
set drmach_bus_sync_list, %g1
BUS_SYNC(%g1, %g2)
! Flush E$ again to victimize references to drmach_bus_sync_list.
ldx [%g6 + 24], %g1 ! *ecache_flushaddr
ld [%g6 + 32], %g2 ! ecache_size
ld [%g6 + 36], %g3 ! ecache_linesize
ECACHE_FLUSHALL(%g2, %g3, %g1, %g4)
drmach_shutdown_ecache_flushed:
ld [%g6 + 8], %g1 ! flush dcache
ld [%g6 + 12], %g2
CH_DCACHE_FLUSHALL(%g1, %g2, %g3)
ld [%g6 + 16], %g1 ! flush icache
ld [%g6 + 20], %g2
CH_ICACHE_FLUSHALL(%g1, %g2, %g3, %g4)
PCACHE_FLUSHALL(%g1, %g2, %g3) ! flush pcache (no parameters)
!
! Flush all unlocked dtlb and itlb entries.
! Replaces TLB_FLUSH_UNLOCKED macro used in Starfire DR.
!
sethi %hi(FLUSH_ADDR), %g1
set DEMAP_ALL_TYPE, %g2
stxa %g0, [%g2]ASI_DTLB_DEMAP
stxa %g0, [%g2]ASI_ITLB_DEMAP
flush %g1
!
! Zero LPA by clearing CBASE and CBND. Following
! this, all transactions to cachable address space
! will be of the remote flavor.
!
SET_NULL_LPA(%g1, %g2)
HERE(%g6, 136, %g1) ! preparation complete (for debug)
!
! Clear byte to signal finished.
! NOTE: This store will allocate in the E$. It is
! vitally important that this line is demoted to
! state I before removing this processor from the
! coherency. The demotion is ensured by a synchronous
! "steal back" that takes place in drmach_cpu_poweroff.
ldx [%g6 + 40], %g1
stba %g0, [%g1]ASI_MEM
5:
HERE(%g6, 144, %g1) ! spin indicator (for debug)
ba 5b
nop
.asciz "drmach_shutdown_asm" ! for debug
.align 4
.global drmach_shutdown_asm_end
drmach_shutdown_asm_end:
SET_SIZE(drmach_shutdown_asm)
! lddsafconfig
!
! input:
! nothing
!
! output:
! %o0 content of this processor's SCR
!
! Configuration Register.
!
ENTRY(lddsafconfig)
retl
ldxa [%g0]ASI_SAFARI_CONFIG, %o0
SET_SIZE(lddsafconfig)
! drmach_rename
!
! input:
! %o1 address for setting error code if rename did not
! complete. Unmodified if no error.
! %o2 address for returning opaque memory controller id
! in case of error. Unmodified if no error.
! Global drmach_xt_mb[cpuid] is expected to be the new LPA.
!
! output:
! [%o1] = 1 if failed to idle memory controller, otherwise unmodified.
! [%o2] = id of failed memory controller, otherwise unmodified.
!
! Perform HW register reprogramming. This is the "rename" step for
! the copy-rename process. drmach_rename is copied to a cpu's sram
!
!
!
3:
4:
5:
b 5b
6:
b 6b
7:
b 7b
8:
.align 4
!
! input:
! nothing
!
! output:
! nothing
!
! cpu's execution out of coherent space while a copy-rename
! operation is in progress.
!
! In each CPU SRAM exists an area (16KB on Cheetah+ boards,
! logically divided by DR into 8KB pages, one page per CPU (or
! core) in a port pair. (Two Safari ports share HW resources on
!
! This routine begins by mapping the appropriate SRAM page,
! transferring the machine code (between the labels
! drmach_rename_wait_asm and drmach_rename_wait_asm_end), then
! jumping to SRAM. After returning from SRAM, the page is
! demapped before the cross-call is exited (sic).
!
! The machine code flushes all caches, waits for a special
!
!
!
!
1:
!
!
0:
bne 0b
! from there.
! Input:
! %g5 is cpuid
! %g6 is data area (follows text)
! %g7 is link address back to caller
!
st %g5, [%g6 + 4] ! save cpuid (for debug)
set dcache_size, %g1
ld [%g1], %g1
st %g1, [%g6 + 8] ! save dcache_size
set dcache_linesize, %g1
ld [%g1], %g1
st %g1, [%g6 + 12] ! save dcache_linesize
GET_ICACHE_PARAMS(%g1, %g2)
st %g1, [%g6 + 16] ! save icache_size
st %g2, [%g6 + 20] ! save icache_linesize
set drmach_iocage_paddr, %g1
ldx [%g1], %g1
stx %g1, [%g6 + 24] ! save *ecache_flushadr
mulx %g5, CPU_NODE_SIZE, %g1 ! %g4 = &cpunodes[cpuid]
set cpunodes, %g4
add %g4, %g1, %g4
ld [%g4 + ECACHE_SIZE], %g1
st %g1, [%g6 + 32] ! save ecache_size
ld [%g4 + ECACHE_LINESIZE], %g1
st %g1, [%g6 + 36] ! save ecache_linesize
LOAD_MB(%g5, %g1, %g2) ! save mailbox data
stb %g1, [%g6 + 40]
membar #Sync ! Complete any pending processing.
! Flush E$. The purpose of this flush is to rid the E$ of
! lines in states O or Os. Implicitly flushes W$.
! NOTE: Reading the bus sync list and r/w ops on drmach_xt_ready
! will disturb the E$. The lines of the bus sync list will be
! in state S. The line containing drmach_xt_ready will be in
! state O. Before proceeding with the copy-rename, the master
! processor will "steal back" the drmach_xt_ready (sic) line.
! This will demote the state of the line in E$ to I.
! However, the lines containing the bus sync list must be
! victimized before returning to the OS. This is vital because
! following copy-rename the corresponding lines in the new home
! memory will be in state gM. The resulting S,gM state pair is
! invalid and does represent a loss of coherency. Flushing the
! E$ after the bus sync list is read will be sufficient to
! avoid the invalid condition.
!
! For Panther, there is redundancy as both cores flush the shared
! L2 and L3 caches. As an optimization, only one core could do the
! flush of the shared caches, however care must be taken that the
! sibling core does not install owned lines once the flush begins.
PN_L2_FLUSHALL(%g1, %g2, %g3)
ldx [%g6 + 24], %g1 ! *ecache_flushaddr
ld [%g6 + 32], %g2 ! ecache_size
ld [%g6 + 36], %g3 ! ecache_linesize
ECACHE_FLUSHALL(%g2, %g3, %g1, %g4)
! Make sure all outstanding transactions for this processor
! have retired. See E$ note above.
set drmach_bus_sync_list, %g1
BUS_SYNC(%g1, %g2)
HERE(%g6, 128, %g4) ! preparation complete (for debug)
! Signal this processor is ready for rename operation to begin.
! See E$ note above.
ATOMIC_ADD_LONG(drmach_xt_ready, 1, %g2, %g3, %g4)
! Loop on IRSR waiting for interrupt. The expected interrupt
! is a cross-trap to drmach_wait_done. It is sent by the master
! processor when the copy-rename operation is complete. The
! received cross-trap is used only as a signal. It is not executed.
2:
HERE(%g6, 136, %g4) ! last poll tick (for debug)
ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g4 ! wait for xt
btst IRSR_BUSY, %g4
bz 2b
nop
stx %g4, [%g6 + 64] ! save status and payload
set IRDR_0, %g2
ldxa [%g2]ASI_INTR_RECEIVE, %g2
stx %g2, [%g6 + 72]
set IRDR_1, %g2
ldxa [%g2]ASI_INTR_RECEIVE, %g2
stx %g2, [%g6 + 80]
set IRDR_2, %g2
ldxa [%g2]ASI_INTR_RECEIVE, %g2
stx %g2, [%g6 + 88]
! clear rcv status
stxa %g0, [%g0]ASI_INTR_RECEIVE_STATUS
membar #Sync
HERE(%g6, 144, %g4) ! signal rcvd tick (for debug)
! Check for copy-rename abort signal. If this signal is received,
! the LPA change is skipped since the rename step was not done.
! The cache flushes are still done as paranoia.
set drmach_rename_abort, %g1
ldx [%g6 + 72], %g2
cmp %g1, %g2
be 3f
nop
! Resume waiting if this is not drmach_rename_done.
set drmach_rename_done, %g1
cmp %g1, %g2
bne 2b
nop
ldub [%g6 + 40], %g1 ! get saved mailbox data
SET_LPA(%g1, %g2, %g3) ! set LPA as indicated by the mb data
3:
! Flush all caches (E, D, I and P) to ensure each is resynchronized
! with the corresponding states in the new home memory. (W$ is
! implicitly flushed when the E$ is flushed.)
!
! Panther needs to flush the L2 cache before the L3
! cache is flushed by the ecache flushall macro.
PN_L2_FLUSHALL(%g1, %g2, %g3)
ldx [%g6 + 24], %g1 ! *ecache_flushaddr
ld [%g6 + 32], %g2 ! ecache_size
ld [%g6 + 36], %g3 ! ecache_linesize
ECACHE_FLUSHALL(%g2, %g3, %g1, %g4)
ld [%g6 + 8], %g1 ! flush dcache
ld [%g6 + 12], %g2
CH_DCACHE_FLUSHALL(%g1, %g2, %g3)
ld [%g6 + 16], %g1 ! flush icache
ld [%g6 + 20], %g2
CH_ICACHE_FLUSHALL(%g1, %g2, %g3, %g4)
PCACHE_FLUSHALL(%g1, %g2, %g3) ! flush pcache (no parameters)
HERE(%g6, 152, %g4) ! done tick (for debug)
jmpl %g7+8, %g0
nop
.asciz "drmach_rename_wait" ! for debug
.align 4
drmach_rename_wait_asm_end:
SET_SIZE(drmach_rename_wait)
! drmach_rename_done
!
! input:
! nothing
!
! output:
! nothing
!
! Used as signal data. See drmach_rename_wait.
!
ENTRY_NP(drmach_rename_done)
retry
SET_SIZE(drmach_rename_done)
! drmach_rename_abort
!
! input:
! nothing
!
! output:
! nothing
!
! Used as signal data. See drmach_rename_wait.
!
ENTRY_NP(drmach_rename_abort)
retry
SET_SIZE(drmach_rename_abort)
! drmach_set_lpa
!
! input:
! Globals: drmach_xt_mb[cpuid] contains new LPA data
!
! output:
! nothing
!
!
!
!
CPU_INDEX(%g1, %g2)
!
! Get LPA message from mailbox, leave in %g5.
!
LOAD_MB(%g1, %g5, %g2)
!
! Set LPA, mailbox data in %g5.
!
SET_LPA(%g5, %g1, %g2)
!
! Signal work is done.
!
ATOMIC_ADD_LONG(drmach_xt_ready, 1, %g1, %g2, %g3)
retry
SET_SIZE(drmach_set_lpa)
!
! drmach_bc_bzero
!
! inputs:
! %o0 = base vaddr of area to clear (must be 64-byte aligned)
! %o1 = size of area to clear (must be multiple of 256 bytes)
!
! outputs:
! %o0 =
! 0 (success)
! 1 (size too small or not modulo 256)
! 2 (vaddr not 64-byte aligned)
!
! Zero a block of storage using block commit stores.
!
.bz_block:
.bz_done:
#endif /* lint */